From e61fb42cbc6b4199f0d8d67195b028d72d9d1f7d Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 17 Feb 2024 12:30:47 +0100 Subject: [PATCH] remove metadata plugin Signed-off-by: Nicola Murino --- docker/scripts/download-plugins.sh | 2 +- docs/eventmanager.md | 1 - docs/full-configuration.md | 4 +- docs/google-cloud-storage.md | 2 +- docs/plugins.md | 1 - docs/rest-api.md | 1 - docs/s3.md | 2 +- go.mod | 10 +-- go.sum | 20 ++--- internal/common/common.go | 79 +----------------- internal/common/common_test.go | 32 -------- internal/common/connection_test.go | 11 ++- internal/common/eventmanager.go | 53 ------------ internal/common/eventmanager_test.go | 57 ++----------- internal/common/protocol_test.go | 3 +- internal/dataprovider/admin.go | 3 +- internal/dataprovider/eventrule.go | 10 +-- internal/dataprovider/user.go | 21 ----- internal/httpd/api_metadata.go | 71 ---------------- internal/httpd/httpd.go | 2 - internal/httpd/httpd_test.go | 47 ----------- internal/httpd/internal_test.go | 56 ------------- internal/httpd/server.go | 3 - internal/plugin/metadata.go | 94 ---------------------- internal/plugin/plugin.go | 116 --------------------------- internal/util/i18n.go | 1 - internal/vfs/azblobfs.go | 101 ++--------------------- internal/vfs/folder.go | 12 --- internal/vfs/gcsfs.go | 108 ++----------------------- internal/vfs/s3fs.go | 97 ++-------------------- internal/vfs/vfs.go | 94 ---------------------- openapi/openapi.yaml | 74 ----------------- static/locales/en/translation.json | 4 +- static/locales/it/translation.json | 4 +- templates/webadmin/eventactions.html | 3 +- 35 files changed, 71 insertions(+), 1128 deletions(-) delete mode 100644 internal/httpd/api_metadata.go delete mode 100644 internal/plugin/metadata.go diff --git a/docker/scripts/download-plugins.sh b/docker/scripts/download-plugins.sh index 29562427..6d9cbdd0 100755 --- a/docker/scripts/download-plugins.sh +++ b/docker/scripts/download-plugins.sh @@ -17,7 +17,7 @@ esac echo "download plugins for arch ${SUFFIX}" -for PLUGIN in geoipfilter kms pubsub eventstore eventsearch metadata auth +for PLUGIN in geoipfilter kms pubsub eventstore eventsearch auth do echo "download plugin from https://github.com/sftpgo/sftpgo-plugin-${PLUGIN}/releases/latest/download/sftpgo-plugin-${PLUGIN}-linux-${SUFFIX}" curl -L "https://github.com/sftpgo/sftpgo-plugin-${PLUGIN}/releases/latest/download/sftpgo-plugin-${PLUGIN}-linux-${SUFFIX}" --output "/usr/local/bin/sftpgo-plugin-${PLUGIN}" diff --git a/docs/eventmanager.md b/docs/eventmanager.md index f1d2a69b..d32c69fc 100644 --- a/docs/eventmanager.md +++ b/docs/eventmanager.md @@ -12,7 +12,6 @@ The following actions are supported: - `Folder quota reset`. The quota used by virtual folders will be updated based on current usage. - `Transfer quota reset`. The transfer quota values will be reset to `0`. - `Data retention check`. You can define per-folder retention policies. -- `Metadata check`. A metadata check requires a metadata plugin such as [this one](https://github.com/sftpgo/sftpgo-plugin-metadata) and removes the metadata associated to missing items (for example objects deleted outside SFTPGo). A metadata check does nothing is no metadata plugin is installed or external metadata are not supported for a filesystem. - `Password expiration check`. You can send an email notification to users whose password is about to expire. - `User expiration check`. You can receive notifications with expired users. - `Identity Provider account check`. You can create/update accounts for users/admins logging in using an Identity Provider. diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 2750a812..8663fdd8 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -67,7 +67,7 @@ The configuration file contains the following sections: - `execute_on`, list of strings. Valid values are `pre-download`, `download`, `first-download`, `pre-upload`, `upload`, `first-upload`, `pre-delete`, `delete`, `rename`, `mkdir`, `rmdir`, `ssh_cmd`, `copy`. Leave empty to disable actions. - `execute_sync`, list of strings. Actions, defined in the `execute_on` list above, to be performed synchronously. The `pre-*` actions are always executed synchronously while the other ones are asynchronous. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. Leave empty to execute only the defined `pre-*` hook synchronously - `hook`, string. Absolute path to the command to execute or HTTP URL to notify. - - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode if not supported": requests for changing permissions and owner/group are silently ignored for cloud filesystems and executed for local/SFTP filesystem. Requests for changing modification times are always executed for local/SFTP filesystems and are executed for cloud based filesystems if the target is a file and there is a metadata plugin available. A metadata plugin can be found [here](https://github.com/sftpgo/sftpgo-plugin-metadata). + - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode if not supported": requests for changing permissions and owner/group are silently ignored for cloud filesystems and executed for local/SFTP filesystem. - `rename_mode`, integer. By default (`0`), renaming of non-empty directories is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to `1` to enable recursive renames for these providers, they may be slow, there is no atomic rename API like for local filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry (partial renaming and incorrect disk quota updates are possible in error cases). Default `0`. - `resume_max_size`, integer. defines the maximum size allowed, in bytes, to resume uploads on storage backends with immutable objects. By default, resuming uploads is not allowed for cloud storage providers (S3, GCS, Azure Blob) because SFTPGo must rewrite the entire file. Set to a value greater than 0 to allow resuming uploads of files smaller than or equal to the defined size. Please note that uploads for these backends are still atomic, the client must intentionally upload a portion of the target file and then resume uploading.. Default `0`. - `temp_path`, string. Defines the path for temporary files such as those used for atomic uploads or file pipes. If you set this option you must make sure that the defined path exists, is accessible for writing by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise the renaming for atomic uploads will become a copy and therefore may take a long time. The temporary files are not namespaced. The default is generally fine. Leave empty for the default. @@ -478,7 +478,7 @@ The configuration file contains the following sections:
Plugins - **plugins**, list of external plugins. :warning: Please note that the plugin system is experimental, the configuration parameters and interfaces may change in a backward incompatible way in future. Each plugin is configured using a struct with the following fields: - - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`, `metadata`, `eventsearcher`, `ipfilter`. + - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`, `eventsearcher`, `ipfilter`. - `notifier_options`, struct. Defines the options for notifier plugins. - `fs_events`, list of strings. Defines the filesystem events that will be notified to this plugin. - `provider_events`, list of strings. Defines the provider events that will be notified to this plugin. diff --git a/docs/google-cloud-storage.md b/docs/google-cloud-storage.md index 6038c93f..681bee7c 100644 --- a/docs/google-cloud-storage.md +++ b/docs/google-cloud-storage.md @@ -8,4 +8,4 @@ You can optionally specify a [storage class](https://cloud.google.com/storage/do The configured bucket must exist. -This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations. As with S3 `chtime` will fail with the default configuration, you can install the [metadata plugin](https://github.com/sftpgo/sftpgo-plugin-metadata) to make it work and thus be able to preserve/change file modification times. +This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations. diff --git a/docs/plugins.md b/docs/plugins.md index bbea847e..907c1855 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -13,7 +13,6 @@ The following plugin types are supported: - `auth`, allows to authenticate users. - `notifier`, allows to receive notifications for supported filesystem events such as file uploads, downloads etc. and provider events such as objects add, update, delete. - `kms`, allows to support additional KMS providers. -- `metadata`, allows to store metadata, such as the last modification time, for storage backends that does not support them (S3, Google Cloud Storage, Azure Blob). - `ipfilter`, allows to allow/deny access based on client IP. Full configuration details can be found [here](./full-configuration.md). diff --git a/docs/rest-api.md b/docs/rest-api.md index 437e39c5..c922ad6e 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -39,7 +39,6 @@ You can create other administrator and assign them the following permissions: - manage admins - manage groups - manage data retention -- manage metadata - view events - manage event rules diff --git a/docs/s3.md b/docs/s3.md index 7cba3671..86ff3760 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -36,4 +36,4 @@ Other notes: - For server side encryption, you have to configure the mapped bucket to automatically encrypt objects. - A local home directory is still required to store temporary files. - Clients that require advanced filesystem-like features such as `sshfs` are not supported. -- `chtime` will fail with the default configuration, you can install the [metadata plugin](https://github.com/sftpgo/sftpgo-plugin-metadata) to make it work and thus be able to preserve/change file modification times. +- `chtime` not supported. diff --git a/go.mod b/go.mod index 5d495f8f..02598674 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.0 github.com/aws/aws-sdk-go-v2/credentials v1.17.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.1 github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.20.1 - github.com/aws/aws-sdk-go-v2/service/s3 v1.49.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.1 github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 github.com/bmatcuk/doublestar/v4 v4.6.1 @@ -26,7 +26,7 @@ require ( github.com/fclairamb/ftpserverlib v0.22.0 github.com/fclairamb/go-log v0.4.1 github.com/go-acme/lego/v4 v4.15.0 - github.com/go-chi/chi/v5 v5.0.11 + github.com/go-chi/chi/v5 v5.0.12 github.com/go-chi/jwtauth/v5 v5.3.0 github.com/go-chi/render v1.0.3 github.com/go-sql-driver/mysql v1.7.1 @@ -53,7 +53,7 @@ require ( github.com/rs/cors v1.10.1 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.32.0 - github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c + github.com/sftpgo/sdk v0.1.6-0.20240216180841-c13afec62842 github.com/shirou/gopsutil/v3 v3.24.1 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 @@ -145,7 +145,7 @@ require ( github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.47.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index ae4a24b2..5a3d330d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.0 h1:lMW2x6sKBsiAJrpi1doOXqWFyEPo github.com/aws/aws-sdk-go-v2/credentials v1.17.0/go.mod h1:uT41FIH8cCIxOdUYIL0PYyHlL1NoneDuDSCwg5VE/5o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 h1:xWCwjjvVz2ojYTP4kBKUuUh9ZrXfcAXpflhOUUeXg1k= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0/go.mod h1:j3fACuqXg4oMTQOR2yY7m0NmJY0yBK4L4sLsRXq1Ins= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.0 h1:FHVyVIJpOeQZCnYj9EVKTWahb4WDNFEUOKCx/dOUPcM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.0/go.mod h1:SL/aJzGL0LsQPQ1y2HMNbJGrm/Xh6aVCGq6ki+DLGEw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.1 h1:FqtJUSBgT2yfZ8kZhTi9AO131qMLOzb4MiH4riAM8XM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.1/go.mod h1:G3V4qNUPMHKrXW/l149QXmHjf1vlMWBO4UuGPCK4a/c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 h1:NPs/EqVO+ajwOoq56EfcGKa3L3ruWuazkIw1BqxwOPw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0/go.mod h1:D+duLy2ylgatV+yTlQ8JTuLfDD0BnFvnQRc+o6tbZ4M= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0 h1:ks7KGMVUMoDzcxNWUlEdI+/lokMFD136EL6DWmUOV80= @@ -63,8 +63,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0 h1:l5puwOHr7IxECu github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0/go.mod h1:Oov79flWa/n7Ni+lQC3z+VM7PoRM47omRqbJU9B5Y7E= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.20.1 h1:eHNChn4Sp+g1hdz4rkx96n1l/LpJEQLDuFB0V+fA/yg= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.20.1/go.mod h1:9ev55pJx9xNX3UAOKzZmbmaTbwwuLTCemOJPsd7rUz8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.49.0 h1:VfU15izXQjz4m9y1DkbY79iylIiuPwWtrram4cSpWEI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.49.0/go.mod h1:1o/W6JFUuREj2ExoQ21vHJgO7wakvjhol91M9eknFgs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0 h1:jZAdMD1ioZdqirzzVVRhpHHWJmcGGCn8JqDYBs5nmYA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0/go.mod h1:1o/W6JFUuREj2ExoQ21vHJgO7wakvjhol91M9eknFgs= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.1 h1:ss/HbHbONu0uscM549++4YanT6MnjNN0BGhE5pZRfG4= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.1/go.mod h1:JsJDZFHwLGZu6dxhV9EV1gJrMnCeE4GEXubSZA59xdA= github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 h1:u6OkVDxtBPnxPkZ9/63ynEe+8kHbtS5IfaC4PzVxzWM= @@ -142,8 +142,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-acme/lego/v4 v4.15.0 h1:A7MHEU3b+TDFqhC/HmzMJnzPbyeaYvMZQBbqgvbThhU= github.com/go-acme/lego/v4 v4.15.0/go.mod h1:eeGhjW4zWT7Ccqa3sY7ayEqFLCAICx+mXgkMHKIkLxg= -github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= -github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/jwtauth/v5 v5.3.0 h1:X7RKGks1lrVeIe2omGyz47pNaNjG2YmwlRN5UKhN8qg= github.com/go-chi/jwtauth/v5 v5.3.0/go.mod h1:2PoGm/KbnzRN9ILY6HFZAI6fTnb1gEZAKogAyqkd6fY= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= @@ -331,8 +331,8 @@ github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -355,8 +355,8 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c h1:07TYPvNbOnmKsBxjNsUr+gsILIUWflw1UYwjn1jognM= -github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c/go.mod h1:AWoY2YYe/P1ymfTlRER/meERQjCcZZTbgVPGcPQgaqc= +github.com/sftpgo/sdk v0.1.6-0.20240216180841-c13afec62842 h1:Rqh/TYkMX6UmUWvgXrsOBoG7ee2GH1AJXBFlszIzKT0= +github.com/sftpgo/sdk v0.1.6-0.20240216180841-c13afec62842/go.mod h1:AWoY2YYe/P1ymfTlRER/meERQjCcZZTbgVPGcPQgaqc= github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= diff --git a/internal/common/common.go b/internal/common/common.go index dd09e26c..9c5135d6 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -153,11 +153,9 @@ var ( // Connections is the list of active connections Connections ActiveConnections // QuotaScans is the list of active quota scans - QuotaScans ActiveScans - // ActiveMetadataChecks holds the active metadata checks - ActiveMetadataChecks MetadataChecks - transfersChecker TransfersChecker - supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV, + QuotaScans ActiveScans + transfersChecker TransfersChecker + supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV, ProtocolHTTP, ProtocolHTTPShare, ProtocolOIDC} disconnHookProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP} // the map key is the protocol, for each protocol we can have multiple rate limiters @@ -1397,74 +1395,3 @@ func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool { return false } - -// MetadataCheck defines an active metadata check -type MetadataCheck struct { - // Username to which the metadata check refers - Username string `json:"username"` - // check start time as unix timestamp in milliseconds - StartTime int64 `json:"start_time"` - Role string `json:"-"` -} - -// MetadataChecks holds the active metadata checks -type MetadataChecks struct { - sync.RWMutex - checks []MetadataCheck -} - -// Get returns the active metadata checks -func (c *MetadataChecks) Get(role string) []MetadataCheck { - c.RLock() - defer c.RUnlock() - - checks := make([]MetadataCheck, 0, len(c.checks)) - for _, check := range c.checks { - if role == "" || role == check.Role { - checks = append(checks, MetadataCheck{ - Username: check.Username, - StartTime: check.StartTime, - }) - } - } - - return checks -} - -// Add adds a user to the ones with active metadata checks. -// Return false if a metadata check is already active for the specified user -func (c *MetadataChecks) Add(username, role string) bool { - c.Lock() - defer c.Unlock() - - for idx := range c.checks { - if c.checks[idx].Username == username { - return false - } - } - - c.checks = append(c.checks, MetadataCheck{ - Username: username, - StartTime: util.GetTimeAsMsSinceEpoch(time.Now()), - Role: role, - }) - - return true -} - -// Remove removes a user from the ones with active metadata checks -func (c *MetadataChecks) Remove(username string) bool { - c.Lock() - defer c.Unlock() - - for idx := range c.checks { - if c.checks[idx].Username == username { - lastIdx := len(c.checks) - 1 - c.checks[idx] = c.checks[lastIdx] - c.checks = c.checks[:lastIdx] - return true - } - } - - return false -} diff --git a/internal/common/common_test.go b/internal/common/common_test.go index c6bb3684..1f846459 100644 --- a/internal/common/common_test.go +++ b/internal/common/common_test.go @@ -1486,38 +1486,6 @@ func TestUpdateTransferTimestamps(t *testing.T) { assert.NoError(t, err) } -func TestMetadataAPI(t *testing.T) { - username := "metadatauser" - require.False(t, ActiveMetadataChecks.Remove(username)) - require.True(t, ActiveMetadataChecks.Add(username, "")) - require.False(t, ActiveMetadataChecks.Add(username, "")) - checks := ActiveMetadataChecks.Get("") - require.Len(t, checks, 1) - checks[0].Username = username + "a" - checks = ActiveMetadataChecks.Get("") - require.Len(t, checks, 1) - require.Equal(t, username, checks[0].Username) - require.True(t, ActiveMetadataChecks.Remove(username)) - require.Len(t, ActiveMetadataChecks.Get(""), 0) -} - -func TestMetadataAPIRole(t *testing.T) { - username := "muser" - role1 := "r1" - role2 := "r2" - require.True(t, ActiveMetadataChecks.Add(username, role2)) - require.False(t, ActiveMetadataChecks.Add(username, "")) - checks := ActiveMetadataChecks.Get("") - require.Len(t, checks, 1) - assert.Empty(t, checks[0].Role) - checks = ActiveMetadataChecks.Get(role1) - require.Len(t, checks, 0) - checks = ActiveMetadataChecks.Get(role2) - require.Len(t, checks, 1) - require.True(t, ActiveMetadataChecks.Remove(username)) - require.Len(t, ActiveMetadataChecks.Get(""), 0) -} - func TestIPList(t *testing.T) { type test struct { ip string diff --git a/internal/common/connection_test.go b/internal/common/connection_test.go index d7e1b904..6797c17c 100644 --- a/internal/common/connection_test.go +++ b/internal/common/connection_test.go @@ -607,6 +607,8 @@ func TestErrorResolvePath(t *testing.T) { assert.Error(t, err) err = conn.doRecursiveRemove(nil, "/fspath", "/vpath", vfs.NewFileInfo("vpath", true, 0, time.Now(), false), 2000) assert.Error(t, err, util.ErrRecursionTooDeep) + err = conn.doRecursiveCopy("/src", "/dst", vfs.NewFileInfo("src", true, 0, time.Now(), false), false, 2000) + assert.Error(t, err, util.ErrRecursionTooDeep) err = conn.checkCopy(vfs.NewFileInfo("name", true, 0, time.Unix(0, 0), false), nil, "/source", "/target") assert.Error(t, err) sourceFile := filepath.Join(os.TempDir(), "f", "source") @@ -1144,13 +1146,16 @@ func TestListerAt(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, n) assert.Equal(t, "..", files[0].Name()) - assert.Equal(t, "p3", files[1].Name()) + vfolders := []string{files[1].Name()} files = make([]os.FileInfo, 200) n, err = lister.ListAt(files, 0) require.NoError(t, err) require.Equal(t, 102, n) - assert.Equal(t, "p2", files[0].Name()) - assert.Equal(t, "p1", files[1].Name()) + vfolders = append(vfolders, files[0].Name()) + vfolders = append(vfolders, files[1].Name()) + assert.Contains(t, vfolders, "p1") + assert.Contains(t, vfolders, "p2") + assert.Contains(t, vfolders, "p3") err = lister.Close() require.NoError(t, err) } diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index be4695d6..137b6ffa 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -2354,57 +2354,6 @@ func executeUserExpirationCheckRuleAction(conditions dataprovider.ConditionOptio return nil } -func executeMetadataCheckForUser(user *dataprovider.User) error { - if err := user.LoadAndApplyGroupSettings(); err != nil { - eventManagerLog(logger.LevelError, "skipping scheduled quota reset for user %s, cannot apply group settings: %v", - user.Username, err) - return err - } - if !ActiveMetadataChecks.Add(user.Username, user.Role) { - eventManagerLog(logger.LevelError, "another metadata check is already in progress for user %q", user.Username) - return fmt.Errorf("another metadata check is in progress for user %q", user.Username) - } - defer ActiveMetadataChecks.Remove(user.Username) - - if err := user.CheckMetadataConsistency(); err != nil { - eventManagerLog(logger.LevelError, "error checking metadata consistence for user %q: %v", user.Username, err) - return fmt.Errorf("error checking metadata consistence for user %q: %w", user.Username, err) - } - return nil -} - -func executeMetadataCheckRuleAction(conditions dataprovider.ConditionOptions, params *EventParams) error { - users, err := params.getUsers() - if err != nil { - return fmt.Errorf("unable to get users: %w", err) - } - var failures []string - var executed int - for _, user := range users { - // if sender is set, the conditions have already been evaluated - if params.sender == "" { - if !checkUserConditionOptions(&user, &conditions) { - eventManagerLog(logger.LevelDebug, "skipping metadata check for user %q, condition options don't match", - user.Username) - continue - } - } - executed++ - if err = executeMetadataCheckForUser(&user); err != nil { - params.AddError(err) - failures = append(failures, user.Username) - } - } - if len(failures) > 0 { - return fmt.Errorf("metadata check failed for users: %s", strings.Join(failures, ", ")) - } - if executed == 0 { - eventManagerLog(logger.LevelError, "no metadata check executed") - return errors.New("no metadata check executed") - } - return nil -} - func executePwdExpirationCheckForUser(user *dataprovider.User, config dataprovider.EventActionPasswordExpiration) error { if err := user.LoadAndApplyGroupSettings(); err != nil { eventManagerLog(logger.LevelError, "skipping password expiration check for user %q, cannot apply group settings: %v", @@ -2568,8 +2517,6 @@ func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams, err = executeTransferQuotaResetRuleAction(conditions, params) case dataprovider.ActionTypeDataRetentionCheck: err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params, action.Name) - case dataprovider.ActionTypeMetadataCheck: - err = executeMetadataCheckRuleAction(conditions, params) case dataprovider.ActionTypeFilesystem: err = executeFsRuleAction(action.Options.FsConfig, conditions, params) case dataprovider.ActionTypePasswordExpirationCheck: diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go index 2b5c7863..c9a32b2e 100644 --- a/internal/common/eventmanager_test.go +++ b/internal/common/eventmanager_test.go @@ -605,8 +605,6 @@ func TestEventManagerErrors(t *testing.T) { assert.Error(t, err) err = executeTransferQuotaResetRuleAction(dataprovider.ConditionOptions{}, &EventParams{}) assert.Error(t, err) - err = executeMetadataCheckRuleAction(dataprovider.ConditionOptions{}, &EventParams{}) - assert.Error(t, err) err = executeUserExpirationCheckRuleAction(dataprovider.ConditionOptions{}, &EventParams{}) assert.Error(t, err) err = executeDeleteFsRuleAction(nil, nil, dataprovider.ConditionOptions{}, &EventParams{}) @@ -639,15 +637,6 @@ func TestEventManagerErrors(t *testing.T) { }, }) assert.Error(t, err) - err = executeMetadataCheckForUser(&dataprovider.User{ - Groups: []sdk.GroupMapping{ - { - Name: groupName, - Type: sdk.GroupTypePrimary, - }, - }, - }) - assert.Error(t, err) err = executeDataRetentionCheckForUser(dataprovider.User{ Groups: []sdk.GroupMapping{ { @@ -987,40 +976,6 @@ func TestEventRuleActions(t *testing.T) { assert.Error(t, err) assert.Contains(t, getErrorString(err), "no user quota reset executed") - action = dataprovider.BaseEventAction{ - Type: dataprovider.ActionTypeMetadataCheck, - } - - err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ - Names: []dataprovider.ConditionPattern{ - { - Pattern: "don't match", - }, - }, - }) - assert.Error(t, err) - assert.Contains(t, getErrorString(err), "no metadata check executed") - - err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ - Names: []dataprovider.ConditionPattern{ - { - Pattern: username1, - }, - }, - }) - assert.NoError(t, err) - // simulate another metadata check in progress - assert.True(t, ActiveMetadataChecks.Add(username1, "")) - err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ - Names: []dataprovider.ConditionPattern{ - { - Pattern: username1, - }, - }, - }) - assert.Error(t, err) - assert.True(t, ActiveMetadataChecks.Remove(username1)) - action = dataprovider.BaseEventAction{ Type: dataprovider.ActionTypeUserExpirationCheck, } @@ -1551,10 +1506,6 @@ func TestEventRuleActionsNoGroupMatching(t *testing.T) { if assert.Error(t, err) { assert.Contains(t, err.Error(), "no user quota reset executed") } - err = executeMetadataCheckRuleAction(conditions, &EventParams{}) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "no metadata check executed") - } err = executeTransferQuotaResetRuleAction(conditions, &EventParams{}) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no transfer quota reset executed") @@ -1840,6 +1791,14 @@ func TestFilesystemActionErrors(t *testing.T) { assert.Contains(t, getErrorString(err), "is outside base dir") } + wr := &zipWriterWrapper{ + Name: xid.New().String() + ".zip", + Writer: zip.NewWriter(bytes.NewBuffer(nil)), + Entries: map[string]bool{}, + } + err = addZipEntry(wr, conn, "/p1", "/", 2000) + assert.ErrorIs(t, err, util.ErrRecursionTooDeep) + err = dataprovider.DeleteUser(username, "", "", "") assert.NoError(t, err) err = os.RemoveAll(user.GetHomeDir()) diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 1b2589ac..6d34b65d 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -8077,10 +8077,9 @@ func TestSFTPLoopError(t *testing.T) { assert.NoError(t, err, string(resp)) user2, resp, err = httpdtest.AddUser(user2, http.StatusCreated) assert.NoError(t, err, string(resp)) - // test metadata check event error a1 := dataprovider.BaseEventAction{ Name: "a1", - Type: dataprovider.ActionTypeMetadataCheck, + Type: dataprovider.ActionTypeUserQuotaReset, } action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated) assert.NoError(t, err) diff --git a/internal/dataprovider/admin.go b/internal/dataprovider/admin.go index 2bc2b125..2d994987 100644 --- a/internal/dataprovider/admin.go +++ b/internal/dataprovider/admin.go @@ -54,7 +54,6 @@ const ( PermAdminManageDefender = "manage_defender" PermAdminViewDefender = "view_defender" PermAdminRetentionChecks = "retention_checks" - PermAdminMetadataChecks = "metadata_checks" PermAdminViewEvents = "view_events" PermAdminManageEventRules = "manage_event_rules" PermAdminManageRoles = "manage_roles" @@ -76,7 +75,7 @@ var ( PermAdminCloseConnections, PermAdminViewServerStatus, PermAdminManageAdmins, PermAdminManageRoles, PermAdminManageEventRules, PermAdminManageAPIKeys, PermAdminQuotaScans, PermAdminManageSystem, PermAdminManageDefender, PermAdminViewDefender, PermAdminManageIPLists, PermAdminRetentionChecks, - PermAdminMetadataChecks, PermAdminViewEvents} + PermAdminViewEvents} forbiddenPermsForRoleAdmins = []string{PermAdminAny, PermAdminManageAdmins, PermAdminManageSystem, PermAdminManageEventRules, PermAdminManageIPLists, PermAdminManageRoles} ) diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go index e3852da5..1563cc6c 100644 --- a/internal/dataprovider/eventrule.go +++ b/internal/dataprovider/eventrule.go @@ -44,7 +44,7 @@ const ( ActionTypeTransferQuotaReset ActionTypeDataRetentionCheck ActionTypeFilesystem - ActionTypeMetadataCheck + actionTypeReserved ActionTypePasswordExpirationCheck ActionTypeUserExpirationCheck ActionTypeIDPAccountCheck @@ -53,7 +53,7 @@ const ( var ( supportedEventActions = []int{ActionTypeHTTP, ActionTypeCommand, ActionTypeEmail, ActionTypeFilesystem, ActionTypeBackup, ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypePasswordExpirationCheck, + ActionTypeDataRetentionCheck, ActionTypePasswordExpirationCheck, ActionTypeUserExpirationCheck, ActionTypeIDPAccountCheck} ) @@ -77,8 +77,6 @@ func getActionTypeAsString(action int) string { return util.I18nActionTypeTransferQuotaReset case ActionTypeDataRetentionCheck: return util.I18nActionTypeDataRetentionCheck - case ActionTypeMetadataCheck: - return util.I18nActionTypeMetadataCheck case ActionTypeFilesystem: return util.I18nActionTypeFilesystem case ActionTypePasswordExpirationCheck: @@ -1652,7 +1650,7 @@ func (r *EventRule) validateMandatorySyncActions() error { func (r *EventRule) checkIPBlockedAndCertificateActions() error { unavailableActions := []int{ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypeFilesystem, ActionTypePasswordExpirationCheck, + ActionTypeDataRetentionCheck, ActionTypeFilesystem, ActionTypePasswordExpirationCheck, ActionTypeUserExpirationCheck} for _, action := range r.Actions { if util.Contains(unavailableActions, action.Type) { @@ -1668,7 +1666,7 @@ func (r *EventRule) checkProviderEventActions(providerObjectType string) error { // can be executed only if we modify a user. They will be executed for the // affected user. Folder quota reset can be executed only for folders. userSpecificActions := []int{ActionTypeUserQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypeFilesystem, + ActionTypeDataRetentionCheck, ActionTypeFilesystem, ActionTypePasswordExpirationCheck, ActionTypeUserExpirationCheck} for _, action := range r.Actions { if util.Contains(userSpecificActions, action.Type) && providerObjectType != actionObjectUser { diff --git a/internal/dataprovider/user.go b/internal/dataprovider/user.go index a604876a..d1c40513 100644 --- a/internal/dataprovider/user.go +++ b/internal/dataprovider/user.go @@ -620,27 +620,6 @@ func (u *User) GetVirtualFolderForPath(virtualPath string) (vfs.VirtualFolder, e return folder, errNoMatchingVirtualFolder } -// CheckMetadataConsistency checks the consistency between the metadata stored -// in the configured metadata plugin and the filesystem -func (u *User) CheckMetadataConsistency() error { - fs, err := u.getRootFs(xid.New().String()) - if err != nil { - return err - } - defer fs.Close() - - if err = fs.CheckMetadata(); err != nil { - return err - } - for idx := range u.VirtualFolders { - v := &u.VirtualFolders[idx] - if err = v.CheckMetadataConsistency(); err != nil { - return err - } - } - return nil -} - // ScanQuota scans the user home dir and virtual folders, included in its quota, // and returns the number of files and their size func (u *User) ScanQuota() (int, int64, error) { diff --git a/internal/httpd/api_metadata.go b/internal/httpd/api_metadata.go deleted file mode 100644 index eb2b7f6c..00000000 --- a/internal/httpd/api_metadata.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2019 Nicola Murino -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, version 3. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package httpd - -import ( - "fmt" - "net/http" - - "github.com/go-chi/render" - - "github.com/drakkan/sftpgo/v2/internal/common" - "github.com/drakkan/sftpgo/v2/internal/dataprovider" - "github.com/drakkan/sftpgo/v2/internal/logger" -) - -func getMetadataChecks(w http.ResponseWriter, r *http.Request) { - r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - claims, err := getTokenClaims(r) - if err != nil || claims.Username == "" { - sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) - return - } - render.JSON(w, r, common.ActiveMetadataChecks.Get(claims.Role)) -} - -func startMetadataCheck(w http.ResponseWriter, r *http.Request) { - r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - claims, err := getTokenClaims(r) - if err != nil || claims.Username == "" { - sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) - return - } - - user, err := dataprovider.GetUserWithGroupSettings(getURLParam(r, "username"), claims.Role) - if err != nil { - sendAPIResponse(w, r, err, "", getRespStatus(err)) - return - } - if !common.ActiveMetadataChecks.Add(user.Username, user.Role) { - sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %q", user.Username), - http.StatusConflict) - return - } - go doMetadataCheck(user) //nolint:errcheck - - sendAPIResponse(w, r, err, "Check started", http.StatusAccepted) -} - -func doMetadataCheck(user dataprovider.User) error { - defer common.ActiveMetadataChecks.Remove(user.Username) - - err := user.CheckMetadataConsistency() - if err != nil { - logger.Warn(logSender, "", "error checking metadata for user %q: %v", user.Username, err) - return err - } - logger.Debug(logSender, "", "metadata check completed for user: %q", user.Username) - return nil -} diff --git a/internal/httpd/httpd.go b/internal/httpd/httpd.go index d3fc44d3..c9f3e0ac 100644 --- a/internal/httpd/httpd.go +++ b/internal/httpd/httpd.go @@ -86,8 +86,6 @@ const ( userSharesPath = "/api/v2/user/shares" retentionBasePath = "/api/v2/retention/users" retentionChecksPath = "/api/v2/retention/users/checks" - metadataBasePath = "/api/v2/metadata/users" - metadataChecksPath = "/api/v2/metadata/users/checks" fsEventsPath = "/api/v2/events/fs" providerEventsPath = "/api/v2/events/provider" logEventsPath = "/api/v2/events/logs" diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 26ce3e2d..f42d8f11 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -122,7 +122,6 @@ const ( userProfilePath = "/api/v2/user/profile" userSharesPath = "/api/v2/user/shares" retentionBasePath = "/api/v2/retention/users" - metadataBasePath = "/api/v2/metadata/users" fsEventsPath = "/api/v2/events/fs" providerEventsPath = "/api/v2/events/provider" logEventsPath = "/api/v2/events/logs" @@ -4624,52 +4623,6 @@ func TestUserType(t *testing.T) { assert.NoError(t, err) } -func TestMetadataAPIMock(t *testing.T) { - user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) - assert.NoError(t, err) - - token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) - assert.NoError(t, err) - req, err := http.NewRequest(http.MethodGet, path.Join(metadataBasePath, "/checks"), nil) - assert.NoError(t, err) - setBearerForReq(req, token) - rr := executeRequest(req) - checkResponseCode(t, http.StatusOK, rr) - var resp []any - err = json.Unmarshal(rr.Body.Bytes(), &resp) - assert.NoError(t, err) - assert.Len(t, resp, 0) - - req, err = http.NewRequest(http.MethodPost, path.Join(metadataBasePath, user.Username, "/check"), nil) - assert.NoError(t, err) - setBearerForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusAccepted, rr) - - assert.Eventually(t, func() bool { - req, err := http.NewRequest(http.MethodGet, path.Join(metadataBasePath, "/checks"), nil) - assert.NoError(t, err) - setBearerForReq(req, token) - rr := executeRequest(req) - checkResponseCode(t, http.StatusOK, rr) - var resp []any - err = json.Unmarshal(rr.Body.Bytes(), &resp) - assert.NoError(t, err) - return len(resp) == 0 - }, 1000*time.Millisecond, 50*time.Millisecond) - - _, err = httpdtest.RemoveUser(user, http.StatusOK) - assert.NoError(t, err) - err = os.RemoveAll(user.GetHomeDir()) - assert.NoError(t, err) - - req, err = http.NewRequest(http.MethodPost, path.Join(metadataBasePath, user.Username, "/check"), nil) - assert.NoError(t, err) - setBearerForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusNotFound, rr) -} - func TestRetentionAPI(t *testing.T) { user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) assert.NoError(t, err) diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go index 449266d8..5be25ca1 100644 --- a/internal/httpd/internal_test.go +++ b/internal/httpd/internal_test.go @@ -731,16 +731,6 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") - rr = httptest.NewRecorder() - getMetadataChecks(rr, req) - assert.Equal(t, http.StatusBadRequest, rr.Code) - assert.Contains(t, rr.Body.String(), "Invalid token claims") - - rr = httptest.NewRecorder() - startMetadataCheck(rr, req) - assert.Equal(t, http.StatusBadRequest, rr.Code) - assert.Contains(t, rr.Body.String(), "Invalid token claims") - rr = httptest.NewRecorder() getUsersQuotaScans(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -2852,52 +2842,6 @@ func TestUserCanResetPassword(t *testing.T) { assert.False(t, isUserAllowedToResetPassword(req, &u)) } -func TestMetadataAPI(t *testing.T) { - username := "metadatauser" - assert.False(t, common.ActiveMetadataChecks.Remove(username)) - - user := dataprovider.User{ - BaseUser: sdk.BaseUser{ - Username: username, - Password: "metadata_pwd", - HomeDir: filepath.Join(os.TempDir(), username), - Status: 1, - }, - } - user.Permissions = make(map[string][]string) - user.Permissions["/"] = []string{dataprovider.PermAny} - err := dataprovider.AddUser(&user, "", "", "") - assert.NoError(t, err) - - assert.True(t, common.ActiveMetadataChecks.Add(username, "")) - - tokenAuth := jwtauth.New(jwa.HS256.String(), util.GenerateRandomBytes(32), nil) - claims := make(map[string]any) - claims["username"] = defaultAdminUsername - claims[jwt.ExpirationKey] = time.Now().UTC().Add(1 * time.Hour) - token, _, err := tokenAuth.Encode(claims) - assert.NoError(t, err) - req, err := http.NewRequest(http.MethodPost, path.Join(metadataBasePath, username, "check"), nil) - assert.NoError(t, err) - rctx := chi.NewRouteContext() - rctx.URLParams.Add("username", username) - req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) - req = req.WithContext(context.WithValue(req.Context(), jwtauth.TokenCtxKey, token)) - - rr := httptest.NewRecorder() - startMetadataCheck(rr, req) - assert.Equal(t, http.StatusConflict, rr.Code, rr.Body.String()) - - assert.True(t, common.ActiveMetadataChecks.Remove(username)) - assert.Len(t, common.ActiveMetadataChecks.Get(""), 0) - err = dataprovider.DeleteUser(username, "", "", "") - assert.NoError(t, err) - - user.FsConfig.Provider = sdk.AzureBlobFilesystemProvider - err = doMetadataCheck(user) - assert.Error(t, err) -} - func TestBrowsableSharePaths(t *testing.T) { share := dataprovider.Share{ Paths: []string{"/"}, diff --git a/internal/httpd/server.go b/internal/httpd/server.go index 1557bdd6..4a77b24b 100644 --- a/internal/httpd/server.go +++ b/internal/httpd/server.go @@ -1378,9 +1378,6 @@ func (s *httpdServer) initializeRouter() { router.With(s.checkPerm(dataprovider.PermAdminRetentionChecks)).Get(retentionChecksPath, getRetentionChecks) router.With(s.checkPerm(dataprovider.PermAdminRetentionChecks)).Post(retentionBasePath+"/{username}/check", startRetentionCheck) - router.With(s.checkPerm(dataprovider.PermAdminMetadataChecks)).Get(metadataChecksPath, getMetadataChecks) - router.With(s.checkPerm(dataprovider.PermAdminMetadataChecks)).Post(metadataBasePath+"/{username}/check", - startMetadataCheck) router.With(s.checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler). Get(fsEventsPath, searchFsEvents) router.With(s.checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler). diff --git a/internal/plugin/metadata.go b/internal/plugin/metadata.go deleted file mode 100644 index 19436cbd..00000000 --- a/internal/plugin/metadata.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) 2019 Nicola Murino -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, version 3. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package plugin - -import ( - "fmt" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/go-plugin" - "github.com/sftpgo/sdk/plugin/metadata" - - "github.com/drakkan/sftpgo/v2/internal/logger" -) - -type metadataPlugin struct { - config Config - metadater metadata.Metadater - client *plugin.Client -} - -func newMetadaterPlugin(config Config) (*metadataPlugin, error) { - p := &metadataPlugin{ - config: config, - } - if err := p.initialize(); err != nil { - logger.Warn(logSender, "", "unable to create metadata plugin: %v, config %+v", err, config) - return nil, err - } - return p, nil -} - -func (p *metadataPlugin) exited() bool { - return p.client.Exited() -} - -func (p *metadataPlugin) cleanup() { - p.client.Kill() -} - -func (p *metadataPlugin) initialize() error { - killProcess(p.config.Cmd) - logger.Debug(logSender, "", "create new metadata plugin %q", p.config.Cmd) - secureConfig, err := p.config.getSecureConfig() - if err != nil { - return err - } - client := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: metadata.Handshake, - Plugins: metadata.PluginMap, - Cmd: p.config.getCommand(), - SkipHostEnv: true, - AllowedProtocols: []plugin.Protocol{ - plugin.ProtocolGRPC, - }, - Managed: false, - AutoMTLS: p.config.AutoMTLS, - SecureConfig: secureConfig, - Logger: &logger.HCLogAdapter{ - Logger: hclog.New(&hclog.LoggerOptions{ - Name: fmt.Sprintf("%v.%v", logSender, metadata.PluginName), - Level: pluginsLogLevel, - DisableTime: true, - }), - }, - }) - rpcClient, err := client.Client() - if err != nil { - logger.Debug(logSender, "", "unable to get rpc client for plugin %q: %v", p.config.Cmd, err) - return err - } - raw, err := rpcClient.Dispense(metadata.PluginName) - if err != nil { - logger.Debug(logSender, "", "unable to get plugin %v from rpc client for command %q: %v", - metadata.PluginName, p.config.Cmd, err) - return err - } - - p.client = client - p.metadater = raw.(metadata.Metadater) - - return nil -} diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index f17dc5e5..6b51b937 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -35,7 +35,6 @@ import ( "github.com/sftpgo/sdk/plugin/eventsearcher" "github.com/sftpgo/sdk/plugin/ipfilter" kmsplugin "github.com/sftpgo/sdk/plugin/kms" - "github.com/sftpgo/sdk/plugin/metadata" "github.com/sftpgo/sdk/plugin/notifier" "github.com/drakkan/sftpgo/v2/internal/kms" @@ -53,8 +52,6 @@ var ( pluginsLogLevel = hclog.Debug // ErrNoSearcher defines the error to return for events searches if no plugin is configured ErrNoSearcher = errors.New("no events searcher plugin defined") - // ErrNoMetadater returns the error to return for metadata methods if no plugin is configured - ErrNoMetadater = errors.New("no metadata plugin defined") ) // Renderer defines the interface for generic objects rendering @@ -172,13 +169,10 @@ type Manager struct { auths []*authPlugin searcherLock sync.RWMutex searcher *searcherPlugin - metadaterLock sync.RWMutex - metadater *metadataPlugin ipFilterLock sync.RWMutex filter *ipFilterPlugin authScopes int hasSearcher bool - hasMetadater bool hasNotifiers bool hasAuths bool hasIPFilter bool @@ -250,12 +244,6 @@ func initializePlugins() error { return err } Handler.searcher = plugin - case metadata.PluginName: - plugin, err := newMetadaterPlugin(config) - if err != nil { - return err - } - Handler.metadater = plugin case ipfilter.PluginName: plugin, err := newIPFilterPlugin(config) if err != nil { @@ -274,7 +262,6 @@ func (m *Manager) validateConfigs() error { kmsSchemes := make(map[string]bool) kmsEncryptions := make(map[string]bool) m.hasSearcher = false - m.hasMetadater = false m.hasNotifiers = false m.hasAuths = false m.hasIPFilter = false @@ -295,11 +282,6 @@ func (m *Manager) validateConfigs() error { return errors.New("only one eventsearcher plugin can be defined") } m.hasSearcher = true - case metadata.PluginName: - if m.hasMetadater { - return errors.New("only one metadata plugin can be defined") - } - m.hasMetadater = true case notifier.PluginName: m.hasNotifiers = true case auth.PluginName: @@ -405,71 +387,6 @@ func (m *Manager) SearchLogEvents(searchFilters *eventsearcher.LogEventSearch) ( return plugin.searchear.SearchLogEvents(searchFilters) } -// HasMetadater returns true if a metadata plugin is defined -func (m *Manager) HasMetadater() bool { - return m.hasMetadater -} - -// SetModificationTime sets the modification time for the specified object -func (m *Manager) SetModificationTime(storageID, objectPath string, mTime int64) error { - if !m.hasMetadater { - return ErrNoMetadater - } - m.metadaterLock.RLock() - plugin := m.metadater - m.metadaterLock.RUnlock() - - return plugin.metadater.SetModificationTime(storageID, objectPath, mTime) -} - -// GetModificationTime returns the modification time for the specified path -func (m *Manager) GetModificationTime(storageID, objectPath string, _ bool) (int64, error) { - if !m.hasMetadater { - return 0, ErrNoMetadater - } - m.metadaterLock.RLock() - plugin := m.metadater - m.metadaterLock.RUnlock() - - return plugin.metadater.GetModificationTime(storageID, objectPath) -} - -// GetModificationTimes returns the modification times for all the files within the specified folder -func (m *Manager) GetModificationTimes(storageID, objectPath string) (map[string]int64, error) { - if !m.hasMetadater { - return nil, ErrNoMetadater - } - m.metadaterLock.RLock() - plugin := m.metadater - m.metadaterLock.RUnlock() - - return plugin.metadater.GetModificationTimes(storageID, objectPath) -} - -// RemoveMetadata deletes the metadata stored for the specified object -func (m *Manager) RemoveMetadata(storageID, objectPath string) error { - if !m.hasMetadater { - return ErrNoMetadater - } - m.metadaterLock.RLock() - plugin := m.metadater - m.metadaterLock.RUnlock() - - return plugin.metadater.RemoveMetadata(storageID, objectPath) -} - -// GetMetadataFolders returns the folders that metadata is associated with -func (m *Manager) GetMetadataFolders(storageID, from string, limit int) ([]string, error) { - if !m.hasMetadater { - return nil, ErrNoMetadater - } - m.metadaterLock.RLock() - plugin := m.metadater - m.metadaterLock.RUnlock() - - return plugin.metadater.GetFolders(storageID, limit, from) -} - // IsIPBanned returns true if the IP filter plugin does not allow the specified ip. // If no IP filter plugin is defined this method returns false func (m *Manager) IsIPBanned(ip, protocol string) bool { @@ -689,16 +606,6 @@ func (m *Manager) checkCrashedPlugins() { m.searcherLock.RUnlock() } - if m.hasMetadater { - m.metadaterLock.RLock() - if m.metadater.exited() { - defer func(cfg Config) { - Handler.restartMetadaterPlugin(cfg) - }(m.metadater.config) - } - m.metadaterLock.RUnlock() - } - if m.hasIPFilter { m.ipFilterLock.RLock() if m.filter.exited() { @@ -776,22 +683,6 @@ func (m *Manager) restartSearcherPlugin(config Config) { m.searcherLock.Unlock() } -func (m *Manager) restartMetadaterPlugin(config Config) { - if m.closed.Load() { - return - } - logger.Info(logSender, "", "try to restart crashed metadater plugin %q", config.Cmd) - plugin, err := newMetadaterPlugin(config) - if err != nil { - logger.Error(logSender, "", "unable to restart metadater plugin %q, err: %v", config.Cmd, err) - return - } - - m.metadaterLock.Lock() - m.metadater = plugin - m.metadaterLock.Unlock() -} - func (m *Manager) restartIPFilterPlugin(config Config) { if m.closed.Load() { return @@ -851,13 +742,6 @@ func (m *Manager) Cleanup() { m.searcherLock.Unlock() } - if m.hasMetadater { - m.metadaterLock.Lock() - logger.Debug(logSender, "", "cleanup metadater plugin %v", m.metadater.config.Cmd) - m.metadater.cleanup() - m.metadaterLock.Unlock() - } - if m.hasIPFilter { m.ipFilterLock.Lock() logger.Debug(logSender, "", "cleanup IP filter plugin %v", m.filter.config.Cmd) diff --git a/internal/util/i18n.go b/internal/util/i18n.go index ac28a874..540b1a20 100644 --- a/internal/util/i18n.go +++ b/internal/util/i18n.go @@ -269,7 +269,6 @@ const ( I18nActionTypeFolderQuotaReset = "actions.types.folder_quota_reset" I18nActionTypeTransferQuotaReset = "actions.types.transfer_quota_reset" I18nActionTypeDataRetentionCheck = "actions.types.data_retention_check" - I18nActionTypeMetadataCheck = "actions.types.metadata_check" I18nActionTypeFilesystem = "actions.types.filesystem" I18nActionTypePwdExpirationCheck = "actions.types.password_expiration_check" I18nActionTypeUserExpirationCheck = "actions.types.user_expiration_check" diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index a11aa870..7c439325 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -46,7 +46,6 @@ import ( "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/metric" - "github.com/drakkan/sftpgo/v2/internal/plugin" "github.com/drakkan/sftpgo/v2/internal/util" "github.com/drakkan/sftpgo/v2/internal/version" ) @@ -171,10 +170,10 @@ func (fs *AzureBlobFs) ConnectionID() string { // Stat returns a FileInfo describing the named file func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { if name == "" || name == "/" || name == "." { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } if fs.config.KeyPrefix == name+"/" { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } attrs, err := fs.headObject(name) @@ -182,9 +181,7 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { contentType := util.GetStringFromPointer(attrs.ContentType) isDir := checkDirectoryMarkers(contentType, attrs.Metadata) metric.AZListObjectsCompleted(nil) - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, - util.GetIntFromPointer(attrs.ContentLength), - util.GetTimeFromPointer(attrs.LastModified), false)) + return NewFileInfo(name, isDir, util.GetIntFromPointer(attrs.ContentLength), util.GetTimeFromPointer(attrs.LastModified), false), nil } if !fs.IsNotExist(err) { return nil, err @@ -195,7 +192,7 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { return nil, err } if hasContents { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } return nil, os.ErrNotExist } @@ -347,11 +344,6 @@ func (fs *AzureBlobFs) Remove(name string, isDir bool) error { } } metric.AZDeleteObjectCompleted(err) - if plugin.Handler.HasMetadater() && err == nil && !isDir { - if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { - fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %q: %+v", name, errMetadata) - } - } return err } @@ -385,22 +377,8 @@ func (*AzureBlobFs) Chmod(_ string, _ os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (fs *AzureBlobFs) Chtimes(name string, _, mtime time.Time, isUploading bool) error { - if !plugin.Handler.HasMetadater() { - return ErrVfsUnsupported - } - if !isUploading { - info, err := fs.Stat(name) - if err != nil { - return err - } - if info.IsDir() { - return ErrVfsUnsupported - } - } - - return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), - util.GetTimeAsMsSinceEpoch(mtime)) +func (fs *AzureBlobFs) Chtimes(_ string, _, _ time.Time, _ bool) error { + return ErrVfsUnsupported } // Truncate changes the size of the named file. @@ -509,53 +487,6 @@ func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) { return fs.GetDirSize(fs.config.KeyPrefix) } -func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { - fileNames := make(map[string]bool) - prefix := "" - if fsPrefix != "/" { - prefix = strings.TrimPrefix(fsPrefix, "/") - } - - pager := fs.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ - Include: container.ListBlobsInclude{ - //Metadata: true, - }, - Prefix: &prefix, - MaxResults: &azureBlobDefaultPageSize, - }) - - for pager.More() { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer cancelFn() - - resp, err := pager.NextPage(ctx) - if err != nil { - metric.AZListObjectsCompleted(err) - return fileNames, err - } - for _, blobItem := range resp.ListBlobsHierarchySegmentResponse.Segment.BlobItems { - name := util.GetStringFromPointer(blobItem.Name) - name = strings.TrimPrefix(name, prefix) - if blobItem.Properties != nil { - contentType := util.GetStringFromPointer(blobItem.Properties.ContentType) - isDir := checkDirectoryMarkers(contentType, blobItem.Metadata) - if isDir { - continue - } - fileNames[name] = true - } - } - } - metric.AZListObjectsCompleted(nil) - - return fileNames, nil -} - -// CheckMetadata checks the metadata consistency -func (fs *AzureBlobFs) CheckMetadata() error { - return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) -} - // GetDirSize returns the number of files and the size for a folder // including any subfolders func (fs *AzureBlobFs) GetDirSize(dirname string) (int, int64, error) { @@ -848,16 +779,6 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo, rec } numFiles++ filesSize += fi.Size() - if plugin.Handler.HasMetadater() { - if !fi.IsDir() { - err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", - source, target, err) - } - } - } } err := fs.skipNotExistErr(fs.Remove(source, fi.IsDir())) return numFiles, filesSize, err @@ -1162,16 +1083,6 @@ func (fs *AzureBlobFs) downloadToWriter(name string, w PipeWriter) (int64, error return n, err } -func (fs *AzureBlobFs) getStorageID() string { - if fs.config.Endpoint != "" { - if !strings.HasSuffix(fs.config.Endpoint, "/") { - return fmt.Sprintf("azblob://%v/%v", fs.config.Endpoint, fs.config.Container) - } - return fmt.Sprintf("azblob://%v%v", fs.config.Endpoint, fs.config.Container) - } - return fmt.Sprintf("azblob://%v", fs.config.Container) -} - func checkDirectoryMarkers(contentType string, metadata map[string]*string) bool { if contentType == dirMimeType { return true diff --git a/internal/vfs/folder.go b/internal/vfs/folder.go index e203470d..e7168cc3 100644 --- a/internal/vfs/folder.go +++ b/internal/vfs/folder.go @@ -211,18 +211,6 @@ func (v *VirtualFolder) GetFilesystem(connectionID string, forbiddenSelfUsers [] } } -// CheckMetadataConsistency checks the consistency between the metadata stored -// in the configured metadata plugin and the filesystem -func (v *VirtualFolder) CheckMetadataConsistency() error { - fs, err := v.GetFilesystem(xid.New().String(), nil) - if err != nil { - return err - } - defer fs.Close() - - return fs.CheckMetadata() -} - // ScanQuota scans the folder and returns the number of files and their size func (v *VirtualFolder) ScanQuota() (int, int64, error) { if v.hasPathPlaceholder() { diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 0e6c3286..4c87042d 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -39,8 +39,6 @@ import ( "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/metric" - "github.com/drakkan/sftpgo/v2/internal/plugin" - "github.com/drakkan/sftpgo/v2/internal/util" "github.com/drakkan/sftpgo/v2/internal/version" ) @@ -112,10 +110,10 @@ func (fs *GCSFs) ConnectionID() string { // Stat returns a FileInfo describing the named file func (fs *GCSFs) Stat(name string) (os.FileInfo, error) { if name == "" || name == "/" || name == "." { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } if fs.config.KeyPrefix == name+"/" { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } return fs.getObjectStat(name) } @@ -304,11 +302,6 @@ func (fs *GCSFs) Remove(name string, isDir bool) error { err = fs.svc.Bucket(fs.config.Bucket).Object(strings.TrimSuffix(name, "/")).Delete(ctx) } metric.GCSDeleteObjectCompleted(err) - if plugin.Handler.HasMetadater() && err == nil && !isDir { - if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { - fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %q: %+v", name, errMetadata) - } - } return err } @@ -342,22 +335,8 @@ func (*GCSFs) Chmod(_ string, _ os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (fs *GCSFs) Chtimes(name string, _, mtime time.Time, isUploading bool) error { - if !plugin.Handler.HasMetadater() { - return ErrVfsUnsupported - } - if !isUploading { - info, err := fs.Stat(name) - if err != nil { - return err - } - if info.IsDir() { - return ErrVfsUnsupported - } - } - - return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), - util.GetTimeAsMsSinceEpoch(mtime)) +func (fs *GCSFs) Chtimes(_ string, _, _ time.Time, _ bool) error { + return ErrVfsUnsupported } // Truncate changes the size of the named file. @@ -459,67 +438,6 @@ func (fs *GCSFs) ScanRootDirContents() (int, int64, error) { return fs.GetDirSize(fs.config.KeyPrefix) } -func (fs *GCSFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { - fileNames := make(map[string]bool) - prefix := "" - if fsPrefix != "/" { - prefix = strings.TrimPrefix(fsPrefix, "/") - } - - query := &storage.Query{ - Prefix: prefix, - Delimiter: "/", - } - err := query.SetAttrSelection(gcsDefaultFieldsSelection) - if err != nil { - return fileNames, err - } - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) - defer cancelFn() - - bkt := fs.svc.Bucket(fs.config.Bucket) - it := bkt.Objects(ctx, query) - pager := iterator.NewPager(it, defaultGCSPageSize, "") - - for { - var objects []*storage.ObjectAttrs - pageToken, err := pager.NextPage(&objects) - if err != nil { - metric.GCSListObjectsCompleted(err) - return fileNames, err - } - - for _, attrs := range objects { - if !attrs.Deleted.IsZero() { - continue - } - if attrs.Prefix == "" { - name, isDir := fs.resolve(attrs.Name, prefix, attrs.ContentType) - if name == "" { - continue - } - if isDir { - continue - } - fileNames[name] = true - } - } - - objects = nil - if pageToken == "" { - break - } - } - - metric.GCSListObjectsCompleted(nil) - return fileNames, nil -} - -// CheckMetadata checks the metadata consistency -func (fs *GCSFs) CheckMetadata() error { - return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) -} - // GetDirSize returns the number of files and the size for a folder // including any subfolders func (fs *GCSFs) GetDirSize(dirname string) (int, int64, error) { @@ -698,7 +616,7 @@ func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) { objSize := attrs.Size objectModTime := attrs.Updated isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/") - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false)) + return NewFileInfo(name, isDir, objSize, objectModTime, false), nil } if !fs.IsNotExist(err) { return nil, err @@ -709,14 +627,14 @@ func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) { return nil, err } if hasContents { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } // finally check if this is an object with a trailing / attrs, err = fs.headObject(name + "/") if err != nil { return nil, err } - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false)) + return NewFileInfo(name, true, attrs.Size, attrs.Updated, false), nil } func (fs *GCSFs) setWriterAttrs(objectWriter *storage.Writer, flag int, name string) { @@ -827,14 +745,6 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion } numFiles++ filesSize += fi.Size() - if plugin.Handler.HasMetadater() { - err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", - source, target, err) - } - } } err := fs.Remove(source, fi.IsDir()) if fs.IsNotExist(err) { @@ -938,10 +848,6 @@ func (*GCSFs) getTempObject(name string) string { return filepath.Join(dir, ".sftpgo-partial."+guid+"."+filepath.Base(name)) } -func (fs *GCSFs) getStorageID() string { - return fmt.Sprintf("gs://%v", fs.config.Bucket) -} - type gcsDirLister struct { baseDirLister bucket *storage.BucketHandle diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index 6a6cd214..2cbdcce0 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -50,7 +50,6 @@ import ( "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/metric" - "github.com/drakkan/sftpgo/v2/internal/plugin" "github.com/drakkan/sftpgo/v2/internal/util" "github.com/drakkan/sftpgo/v2/internal/version" ) @@ -148,10 +147,10 @@ func (fs *S3Fs) ConnectionID() string { func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { var result *FileInfo if name == "" || name == "/" || name == "." { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } if fs.config.KeyPrefix == name+"/" { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } obj, err := fs.headObject(name) if err == nil { @@ -162,8 +161,7 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { _, err = fs.headObject(name + "/") isDir = err == nil } - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), - util.GetTimeFromPointer(obj.LastModified), false)) + return NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false), nil } if !fs.IsNotExist(err) { return result, err @@ -171,7 +169,7 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { // now check if this is a prefix (virtual directory) hasContents, err := fs.hasContents(name) if err == nil && hasContents { - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) + return NewFileInfo(name, true, 0, time.Unix(0, 0), false), nil } else if err != nil { return nil, err } @@ -188,8 +186,7 @@ func (fs *S3Fs) getStatForDir(name string) (os.FileInfo, error) { if err != nil { return result, err } - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, util.GetIntFromPointer(obj.ContentLength), - util.GetTimeFromPointer(obj.LastModified), false)) + return NewFileInfo(name, true, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false), nil } // Lstat returns a FileInfo describing the named file @@ -364,11 +361,6 @@ func (fs *S3Fs) Remove(name string, isDir bool) error { Key: aws.String(name), }) metric.S3DeleteObjectCompleted(err) - if plugin.Handler.HasMetadater() && err == nil && !isDir { - if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { - fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %q: %+v", name, errMetadata) - } - } return err } @@ -402,21 +394,8 @@ func (*S3Fs) Chmod(_ string, _ os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (fs *S3Fs) Chtimes(name string, _, mtime time.Time, isUploading bool) error { - if !plugin.Handler.HasMetadater() { - return ErrVfsUnsupported - } - if !isUploading { - info, err := fs.Stat(name) - if err != nil { - return err - } - if info.IsDir() { - return ErrVfsUnsupported - } - } - return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), - util.GetTimeAsMsSinceEpoch(mtime)) +func (fs *S3Fs) Chtimes(_ string, _, _ time.Time, _ bool) error { + return ErrVfsUnsupported } // Truncate changes the size of the named file. @@ -519,50 +498,6 @@ func (fs *S3Fs) ScanRootDirContents() (int, int64, error) { return fs.GetDirSize(fs.config.KeyPrefix) } -func (fs *S3Fs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { - fileNames := make(map[string]bool) - prefix := "" - if fsPrefix != "/" { - prefix = strings.TrimPrefix(fsPrefix, "/") - } - - paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(prefix), - Delimiter: aws.String("/"), - MaxKeys: &s3DefaultPageSize, - }) - - for paginator.HasMorePages() { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer cancelFn() - - page, err := paginator.NextPage(ctx) - if err != nil { - metric.S3ListObjectsCompleted(err) - if err != nil { - fsLog(fs, logger.LevelError, "unable to get content for prefix %q: %+v", prefix, err) - return nil, err - } - return fileNames, err - } - for _, fileObject := range page.Contents { - name, isDir := fs.resolve(fileObject.Key, prefix) - if name != "" && !isDir { - fileNames[name] = true - } - } - } - - metric.S3ListObjectsCompleted(nil) - return fileNames, nil -} - -// CheckMetadata checks the metadata consistency -func (fs *S3Fs) CheckMetadata() error { - return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) -} - // GetDirSize returns the number of files and the size for a folder // including any subfolders func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) { @@ -787,14 +722,6 @@ func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo, recursion } numFiles++ filesSize += fi.Size() - if plugin.Handler.HasMetadater() { - err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", - source, target, err) - } - } } err := fs.Remove(source, fi.IsDir()) if fs.IsNotExist(err) { @@ -1051,16 +978,6 @@ func (fs *S3Fs) downloadToWriter(name string, w PipeWriter) (int64, error) { return n, err } -func (fs *S3Fs) getStorageID() string { - if fs.config.Endpoint != "" { - if !strings.HasSuffix(fs.config.Endpoint, "/") { - return fmt.Sprintf("s3://%v/%v", fs.config.Endpoint, fs.config.Bucket) - } - return fmt.Sprintf("s3://%v%v", fs.config.Endpoint, fs.config.Bucket) - } - return fmt.Sprintf("s3://%v", fs.config.Bucket) -} - type s3DirLister struct { baseDirLister paginator *s3.ListObjectsV2Paginator diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 3af5b44b..dbb84501 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -31,11 +31,9 @@ import ( "github.com/eikenb/pipeat" "github.com/pkg/sftp" "github.com/sftpgo/sdk" - "github.com/sftpgo/sdk/plugin/metadata" "github.com/drakkan/sftpgo/v2/internal/kms" "github.com/drakkan/sftpgo/v2/internal/logger" - "github.com/drakkan/sftpgo/v2/internal/plugin" "github.com/drakkan/sftpgo/v2/internal/util" ) @@ -147,7 +145,6 @@ type Fs interface { HasVirtualFolders() bool GetMimeType(name string) (string, error) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) - CheckMetadata() error Close() error } @@ -157,13 +154,6 @@ type FsRealPather interface { RealPath(p string) (string, error) } -// fsMetadataChecker is a Fs that implements the getFileNamesInPrefix method. -// This interface is used to abstract metadata consistency checks -type fsMetadataChecker interface { - Fs - getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) -} - // FsFileCopier is a Fs that implements the CopyFile method. type FsFileCopier interface { Fs @@ -1090,90 +1080,6 @@ func IsUploadResumeSupported(fs Fs, size int64) bool { return fs.IsConditionalUploadResumeSupported(size) } -func updateFileInfoModTime(storageID, objectPath string, info *FileInfo) (*FileInfo, error) { - if !plugin.Handler.HasMetadater() { - return info, nil - } - if info.IsDir() { - return info, nil - } - mTime, err := plugin.Handler.GetModificationTime(storageID, ensureAbsPath(objectPath), info.IsDir()) - if errors.Is(err, metadata.ErrNoSuchObject) { - return info, nil - } - if err != nil { - return info, err - } - info.modTime = util.GetTimeFromMsecSinceEpoch(mTime) - return info, nil -} - -func ensureAbsPath(name string) string { - if path.IsAbs(name) { - return name - } - return path.Join("/", name) -} - -func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { - if !plugin.Handler.HasMetadater() { - return nil - } - limit := 100 - from := "" - for { - metadataFolders, err := plugin.Handler.GetMetadataFolders(storageID, from, limit) - if err != nil { - fsLog(fs, logger.LevelError, "unable to get folders: %v", err) - return err - } - for _, folder := range metadataFolders { - from = folder - fsPrefix := folder - if !strings.HasSuffix(folder, "/") { - fsPrefix += "/" - } - if keyPrefix != "" { - if !strings.HasPrefix(fsPrefix, "/"+keyPrefix) { - fsLog(fs, logger.LevelDebug, "skip metadata check for folder %q outside prefix %q", - folder, keyPrefix) - continue - } - } - fsLog(fs, logger.LevelDebug, "check metadata for folder %q", folder) - metadataValues, err := plugin.Handler.GetModificationTimes(storageID, folder) - if err != nil { - fsLog(fs, logger.LevelError, "unable to get modification times for folder %q: %v", folder, err) - return err - } - if len(metadataValues) == 0 { - fsLog(fs, logger.LevelDebug, "no metadata for folder %q", folder) - continue - } - fileNames, err := fs.getFileNamesInPrefix(fsPrefix) - if err != nil { - fsLog(fs, logger.LevelError, "unable to get content for prefix %q: %v", fsPrefix, err) - return err - } - // now check if we have metadata for a missing object - for k := range metadataValues { - if _, ok := fileNames[k]; !ok { - filePath := ensureAbsPath(path.Join(folder, k)) - if err = plugin.Handler.RemoveMetadata(storageID, filePath); err != nil { - fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %q: %v", filePath, err) - } else { - fsLog(fs, logger.LevelDebug, "metadata removed for missing file %q", filePath) - } - } - } - } - - if len(metadataFolders) < limit { - return nil - } - } -} - func validateOSFsConfig(config *sdk.OSFsConfig) error { if config.ReadBufferSize < 0 || config.ReadBufferSize > 10 { return fmt.Errorf("invalid read buffer size must be between 0 and 10 MB") diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 725a69e1..66fd3568 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1064,67 +1064,6 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' - /metadata/users/checks: - get: - tags: - - metadata - summary: Get metadata checks - description: Returns the active metadata checks - operationId: get_users_metadata_checks - responses: - '200': - description: successful operation - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/MetadataCheck' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '500': - $ref: '#/components/responses/InternalServerError' - default: - $ref: '#/components/responses/DefaultResponse' - /metadata/users/{username}/check: - parameters: - - name: username - in: path - description: the username - required: true - schema: - type: string - post: - tags: - - metadata - summary: Start a metadata check - description: 'Starts a new metadata check for the given user. A metadata check requires a metadata plugin and removes the metadata associated to missing items (for example objects deleted outside SFTPGo). If a metadata check for this user is already active a 409 status code is returned. Metadata are stored for cloud storage backends. This API does nothing for other backends or if no metadata plugin is configured' - operationId: start_user_metadata_check - responses: - '202': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/ApiResponse' - example: - message: Check started - '400': - $ref: '#/components/responses/BadRequest' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '404': - $ref: '#/components/responses/NotFound' - '409': - $ref: '#/components/responses/Conflict' - '500': - $ref: '#/components/responses/InternalServerError' - default: - $ref: '#/components/responses/DefaultResponse' /retention/users/checks: get: tags: @@ -5003,7 +4942,6 @@ components: - manage_defender - view_defender - retention_checks - - metadata_checks - view_events - manage_event_rules - manage_roles @@ -5027,7 +4965,6 @@ components: * `manage_defender` - remove ip from the dynamic blocklist is allowed * `view_defender` - list the dynamic blocklist is allowed * `retention_checks` - view and start retention checks is allowed - * `metadata_checks` - view and start metadata checks is allowed * `view_events` - view and search filesystem and provider events is allowed * `manage_event_rules` - manage event actions and rules is allowed * `manage_roles` - manage roles is allowed @@ -5078,7 +5015,6 @@ components: * `7` - Transfer quota reset * `8` - Data retention check * `9` - Filesystem - * `10` - Metadata check * `11` - Password expiration check * `12` - User expiration check * `13` - Identity Provider account check @@ -6322,16 +6258,6 @@ components: type: string format: email description: 'if the notification method is set to "Email", this is the e-mail address that receives the retention check report. This field is automatically set to the email address associated with the administrator starting the check' - MetadataCheck: - type: object - properties: - username: - type: string - description: username to which the check refers - start_time: - type: integer - format: int64 - description: check start time as unix timestamp in milliseconds QuotaScan: type: object properties: diff --git a/static/locales/en/translation.json b/static/locales/en/translation.json index 2a7e528f..cb89e2c3 100644 --- a/static/locales/en/translation.json +++ b/static/locales/en/translation.json @@ -260,7 +260,8 @@ "day_of_month": "Day of month", "month": "Month", "options": "Options", - "expired": "Expired" + "expired": "Expired", + "unsupported": "Feature no longer supported" }, "fs": { "view_file": "View file \"{{- path}}\"", @@ -968,7 +969,6 @@ "folder_quota_reset": "Folder quota reset", "transfer_quota_reset": "Transfer quota reset", "data_retention_check": "Data retention check", - "metadata_check": "Metadata check", "filesystem": "Filesystem", "password_expiration_check": "Password expiration check", "user_expiration_check": "User expiration check", diff --git a/static/locales/it/translation.json b/static/locales/it/translation.json index d13a379b..f73fd873 100644 --- a/static/locales/it/translation.json +++ b/static/locales/it/translation.json @@ -260,7 +260,8 @@ "day_of_month": "Giorno mese", "month": "Mese", "options": "Opzioni", - "expired": "Scaduto" + "expired": "Scaduto", + "unsupported": "Funzionalità non più supportata" }, "fs": { "view_file": "Visualizza file \"{{- path}}\"", @@ -968,7 +969,6 @@ "folder_quota_reset": "Ricalcolo quota cartella virtuale", "transfer_quota_reset": "Reimpostazione quota trasferimenti", "data_retention_check": "Controllo conservazione dati", - "metadata_check": "Controllo metadati", "filesystem": "Filesystem", "password_expiration_check": "Controllo password scadute", "user_expiration_check": "Controllo utenti scaduti", diff --git a/templates/webadmin/eventactions.html b/templates/webadmin/eventactions.html index e0d8b59c..226e164e 100644 --- a/templates/webadmin/eventactions.html +++ b/templates/webadmin/eventactions.html @@ -182,7 +182,8 @@ explicit grant from the SFTPGo Team (support@sftpgo.com). case 9: return $.t('actions.types.filesystem'); case 10: - return $.t('actions.types.metadata_check'); + // metadata check was removed + return ""; case 11: return $.t('actions.types.password_expiration_check'); case 12: