add data retention check hook

2024-06-11 21:02:13 +02:00 · 2021-10-03 15:17:49 +02:00 · 2021-10-03 15:17:49 +02:00 · 1b4a1fbbe5
parent ec81a7ac29
commit 1b4a1fbbe5
16 changed files with 312 additions and 94 deletions
--- a/common/common.go
+++ b/common/common.go
@ -394,6 +394,9 @@ type Configuration struct {
 	// and before he tries to login. It allows you to reject the connection based on the source
 	// ip address. Leave empty do disable.
 	PostConnectHook string `json:"post_connect_hook" mapstructure:"post_connect_hook"`
+	// Absolute path to an external program or an HTTP URL to invoke after a data retention check completes.
+	// Leave empty do disable.
+	DataRetentionHook string `json:"data_retention_hook" mapstructure:"data_retention_hook"`
 	// Maximum number of concurrent client connections. 0 means unlimited
 	MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"`
 	// Maximum number of concurrent client connections from the same host (IP). 0 means unlimited
--- a/common/dataretention.go
+++ b/common/dataretention.go
@ -2,13 +2,21 @@ package common

 import (
 	"bytes"
+	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
+	"os/exec"
 	"path"
+	"path/filepath"
+	"strings"
 	"sync"
 	"time"

 	"github.com/drakkan/sftpgo/v2/dataprovider"
+	"github.com/drakkan/sftpgo/v2/httpclient"
 	"github.com/drakkan/sftpgo/v2/logger"
 	"github.com/drakkan/sftpgo/v2/smtp"
 	"github.com/drakkan/sftpgo/v2/util"
@ -18,8 +26,8 @@ import (
 type RetentionCheckNotification = string

 const (
-	// no notification, the check results are recorded in the logs
-	RetentionCheckNotificationNone = "None"
+	// notify results using the defined "data_retention_hook"
+	RetentionCheckNotificationHook = "Hook"
 	// notify results by email
 	RetentionCheckNotificationEmail = "Email"
 )
@ -44,12 +52,14 @@ func (c *ActiveRetentionChecks) Get() []RetentionCheck {
 	for _, check := range c.Checks {
 		foldersCopy := make([]FolderRetention, len(check.Folders))
 		copy(foldersCopy, check.Folders)
+		notificationsCopy := make([]string, len(check.Notifications))
+		copy(notificationsCopy, check.Notifications)
 		checks = append(checks, RetentionCheck{
-			Username:     check.Username,
-			StartTime:    check.StartTime,
-			Notification: check.Notification,
-			Email:        check.Email,
-			Folders:      foldersCopy,
+			Username:      check.Username,
+			StartTime:     check.StartTime,
+			Notifications: notificationsCopy,
+			Email:         check.Email,
+			Folders:       foldersCopy,
 		})
 	}
 	return checks
@ -130,13 +140,13 @@ func (f *FolderRetention) isValid() error {
 }

 type folderRetentionCheckResult struct {
-	Path         string
-	Retention    int
-	DeletedFiles int
-	DeletedSize  int64
-	Elapsed      time.Duration
-	Info         string
-	Error        string
+	Path         string        `json:"path"`
+	Retention    int           `json:"retention"`
+	DeletedFiles int           `json:"deleted_files"`
+	DeletedSize  int64         `json:"deleted_size"`
+	Elapsed      time.Duration `json:"-"`
+	Info         string        `json:"info,omitempty"`
+	Error        string        `json:"error,omitempty"`
 }

 // RetentionCheck defines an active retention check
@ -148,7 +158,7 @@ type RetentionCheck struct {
 	// affected folders
 	Folders []FolderRetention `json:"folders"`
 	// how cleanup results will be notified
-	Notification RetentionCheckNotification `json:"notification"`
+	Notifications []RetentionCheckNotification `json:"notifications,omitempty"`
 	// email to use if the notification method is set to email
 	Email string `json:"email,omitempty"`
 	// Cleanup results
@ -176,16 +186,22 @@ func (c *RetentionCheck) Validate() error {
 	if nothingToDo {
 		return util.NewValidationError("nothing to delete!")
 	}
-	switch c.Notification {
-	case RetentionCheckNotificationEmail:
-		if !smtp.IsEnabled() {
-			return util.NewValidationError("in order to notify results via email you must configure an SMTP server")
+	for _, notification := range c.Notifications {
+		switch notification {
+		case RetentionCheckNotificationEmail:
+			if !smtp.IsEnabled() {
+				return util.NewValidationError("in order to notify results via email you must configure an SMTP server")
+			}
+			if c.Email == "" {
+				return util.NewValidationError("in order to notify results via email you must add a valid email address to your profile")
+			}
+		case RetentionCheckNotificationHook:
+			if Config.DataRetentionHook == "" {
+				return util.NewValidationError("in order to notify results via hook you must define a data_retention_hook")
+			}
+		default:
+			return util.NewValidationError(fmt.Sprintf("invalid notification %#v", notification))
 		}
-		if c.Email == "" {
-			return util.NewValidationError("in order to notify results via email you must add a valid email address to your profile")
-		}
-	default:
-		c.Notification = RetentionCheckNotificationNone
 	}
 	return nil
 }
@ -320,49 +336,124 @@ func (c *RetentionCheck) Start() {
 		if folder.Retention > 0 {
 			if err := c.cleanupFolder(folder.Path); err != nil {
 				c.conn.Log(logger.LevelWarn, "retention check failed, unable to cleanup folder %#v", folder.Path)
-				c.sendNotification(startTime, err) //nolint:errcheck
+				c.sendNotifications(time.Since(startTime), err)
 				return
 			}
 		}
 	}

 	c.conn.Log(logger.LevelInfo, "retention check completed")
-	c.sendNotification(startTime, nil) //nolint:errcheck
+	c.sendNotifications(time.Since(startTime), nil)
 }

-func (c *RetentionCheck) sendNotification(startTime time.Time, err error) error {
-	switch c.Notification {
-	case RetentionCheckNotificationEmail:
-		body := new(bytes.Buffer)
-		data := make(map[string]interface{})
-		data["Results"] = c.results
-		totalDeletedFiles := 0
-		totalDeletedSize := int64(0)
-		for _, result := range c.results {
-			totalDeletedFiles += result.DeletedFiles
-			totalDeletedSize += result.DeletedSize
+func (c *RetentionCheck) sendNotifications(elapsed time.Duration, err error) {
+	for _, notification := range c.Notifications {
+		switch notification {
+		case RetentionCheckNotificationEmail:
+			c.sendEmailNotification(elapsed, err) //nolint:errcheck
+		case RetentionCheckNotificationHook:
+			c.sendHookNotification(elapsed, err) //nolint:errcheck
 		}
-		data["HumanizeSize"] = util.ByteCountIEC
-		data["TotalFiles"] = totalDeletedFiles
-		data["TotalSize"] = totalDeletedSize
-		data["Elapsed"] = time.Since(startTime)
-		data["Username"] = c.conn.User.Username
-		data["StartTime"] = util.GetTimeFromMsecSinceEpoch(c.StartTime)
-		if err == nil {
-			data["Status"] = "Succeeded"
-		} else {
-			data["Status"] = "Failed"
-		}
-		if err := smtp.RenderRetentionReportTemplate(body, data); err != nil {
-			c.conn.Log(logger.LevelWarn, "unable to render retention check template: %v", err)
-			return err
-		}
-		subject := fmt.Sprintf("Retention check completed for user %#v", c.conn.User.Username)
-		if err := smtp.SendEmail(c.Email, subject, body.String(), smtp.EmailContentTypeTextHTML); err != nil {
-			c.conn.Log(logger.LevelWarn, "unable to notify retention check result via email: %v", err)
-			return err
-		}
-		c.conn.Log(logger.LevelInfo, "retention check result successfully notified via email")
 	}
+}
+
+func (c *RetentionCheck) sendEmailNotification(elapsed time.Duration, errCheck error) error {
+	body := new(bytes.Buffer)
+	data := make(map[string]interface{})
+	data["Results"] = c.results
+	totalDeletedFiles := 0
+	totalDeletedSize := int64(0)
+	for _, result := range c.results {
+		totalDeletedFiles += result.DeletedFiles
+		totalDeletedSize += result.DeletedSize
+	}
+	data["HumanizeSize"] = util.ByteCountIEC
+	data["TotalFiles"] = totalDeletedFiles
+	data["TotalSize"] = totalDeletedSize
+	data["Elapsed"] = elapsed
+	data["Username"] = c.conn.User.Username
+	data["StartTime"] = util.GetTimeFromMsecSinceEpoch(c.StartTime)
+	if errCheck == nil {
+		data["Status"] = "Succeeded"
+	} else {
+		data["Status"] = "Failed"
+	}
+	if err := smtp.RenderRetentionReportTemplate(body, data); err != nil {
+		c.conn.Log(logger.LevelWarn, "unable to render retention check template: %v", err)
+		return err
+	}
+	startTime := time.Now()
+	subject := fmt.Sprintf("Retention check completed for user %#v", c.conn.User.Username)
+	if err := smtp.SendEmail(c.Email, subject, body.String(), smtp.EmailContentTypeTextHTML); err != nil {
+		c.conn.Log(logger.LevelWarn, "unable to notify retention check result via email: %v, elapsed: %v", err,
+			time.Since(startTime))
+		return err
+	}
+	c.conn.Log(logger.LevelInfo, "retention check result successfully notified via email, elapsed: %v", time.Since(startTime))
 	return nil
 }
+
+func (c *RetentionCheck) sendHookNotification(elapsed time.Duration, errCheck error) error {
+	data := make(map[string]interface{})
+	totalDeletedFiles := 0
+	totalDeletedSize := int64(0)
+	for _, result := range c.results {
+		totalDeletedFiles += result.DeletedFiles
+		totalDeletedSize += result.DeletedSize
+	}
+	data["username"] = c.conn.User.Username
+	data["start_time"] = c.StartTime
+	data["elapsed"] = elapsed.Milliseconds()
+	if errCheck == nil {
+		data["status"] = 1
+	} else {
+		data["status"] = 0
+	}
+	data["total_deleted_files"] = totalDeletedFiles
+	data["total_deleted_size"] = totalDeletedSize
+	data["details"] = c.results
+	jsonData, _ := json.Marshal(data)
+
+	startTime := time.Now()
+
+	if strings.HasPrefix(Config.DataRetentionHook, "http") {
+		var url *url.URL
+		url, err := url.Parse(Config.DataRetentionHook)
+		if err != nil {
+			c.conn.Log(logger.LevelWarn, "invalid data retention hook %#v: %v", Config.DataRetentionHook, err)
+			return err
+		}
+		respCode := 0
+
+		resp, err := httpclient.RetryablePost(url.String(), "application/json", bytes.NewBuffer(jsonData))
+		if err == nil {
+			respCode = resp.StatusCode
+			resp.Body.Close()
+
+			if respCode != http.StatusOK {
+				err = errUnexpectedHTTResponse
+			}
+		}
+
+		c.conn.Log(logger.LevelDebug, "notified result to URL: %#v, status code: %v, elapsed: %v err: %v",
+			url.Redacted(), respCode, time.Since(startTime), err)
+
+		return err
+	}
+	if !filepath.IsAbs(Config.DataRetentionHook) {
+		err := fmt.Errorf("invalid data retention hook %#v", Config.DataRetentionHook)
+		c.conn.Log(logger.LevelWarn, "%v", err)
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, Config.DataRetentionHook)
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("SFTPGO_DATA_RETENTION_RESULT=%v", string(jsonData)))
+	err := cmd.Run()
+
+	c.conn.Log(logger.LevelDebug, "notified result using command: %v, elapsed: %v err: %v",
+		Config.DataRetentionHook, time.Since(startTime), err)
+	return err
+}
--- a/common/dataretention_test.go
+++ b/common/dataretention_test.go
@ -3,6 +3,8 @@ package common
 import (
 	"errors"
 	"fmt"
+	"os/exec"
+	"runtime"
 	"testing"
 	"time"

@ -65,10 +67,10 @@ func TestRetentionValidation(t *testing.T) {
 	}
 	err = check.Validate()
 	assert.NoError(t, err)
-	assert.Equal(t, RetentionCheckNotificationNone, check.Notification)
+	assert.Len(t, check.Notifications, 0)
 	assert.Empty(t, check.Email)

-	check.Notification = RetentionCheckNotificationEmail
+	check.Notifications = []RetentionCheckNotification{RetentionCheckNotificationEmail}
 	err = check.Validate()
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "you must configure an SMTP server")
@ -92,9 +94,19 @@ func TestRetentionValidation(t *testing.T) {
 	smtpCfg = smtp.Config{}
 	err = smtpCfg.Initialize("..")
 	require.NoError(t, err)
+
+	check.Notifications = []RetentionCheckNotification{RetentionCheckNotificationHook}
+	err = check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "data_retention_hook")
+
+	check.Notifications = []string{"not valid"}
+	err = check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid notification")
 }

-func TestEmailNotifications(t *testing.T) {
+func TestRetentionEmailNotifications(t *testing.T) {
 	smtpCfg := smtp.Config{
 		Host:          "127.0.0.1",
 		Port:          2525,
@ -111,8 +123,8 @@ func TestEmailNotifications(t *testing.T) {
 	user.Permissions = make(map[string][]string)
 	user.Permissions["/"] = []string{dataprovider.PermAny}
 	check := RetentionCheck{
-		Notification: RetentionCheckNotificationEmail,
-		Email:        "notification@example.com",
+		Notifications: []RetentionCheckNotification{RetentionCheckNotificationEmail},
+		Email:         "notification@example.com",
 		results: []*folderRetentionCheckResult{
 			{
 				Path:         "/",
@ -127,24 +139,80 @@ func TestEmailNotifications(t *testing.T) {
 	conn.SetProtocol(ProtocolDataRetention)
 	conn.ID = fmt.Sprintf("data_retention_%v", user.Username)
 	check.conn = conn
-	err = check.sendNotification(time.Now(), nil)
+	check.sendNotifications(1*time.Second, nil)
+	err = check.sendEmailNotification(1*time.Second, nil)
 	assert.NoError(t, err)
-	err = check.sendNotification(time.Now(), errors.New("test error"))
+	err = check.sendEmailNotification(1*time.Second, errors.New("test error"))
 	assert.NoError(t, err)

 	smtpCfg.Port = 2626
 	err = smtpCfg.Initialize("..")
 	require.NoError(t, err)
-	err = check.sendNotification(time.Now(), nil)
+	err = check.sendEmailNotification(1*time.Second, nil)
 	assert.Error(t, err)

 	smtpCfg = smtp.Config{}
 	err = smtpCfg.Initialize("..")
 	require.NoError(t, err)
-	err = check.sendNotification(time.Now(), nil)
+	err = check.sendEmailNotification(1*time.Second, nil)
 	assert.Error(t, err)
 }

+func TestRetentionHookNotifications(t *testing.T) {
+	dataRetentionHook := Config.DataRetentionHook
+
+	Config.DataRetentionHook = fmt.Sprintf("http://%v", httpAddr)
+	user := dataprovider.User{
+		BaseUser: sdk.BaseUser{
+			Username: "user2",
+		},
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	check := RetentionCheck{
+		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
+		results: []*folderRetentionCheckResult{
+			{
+				Path:         "/",
+				Retention:    24,
+				DeletedFiles: 10,
+				DeletedSize:  32657,
+				Elapsed:      10 * time.Second,
+			},
+		},
+	}
+	conn := NewBaseConnection("", "", "", "", user)
+	conn.SetProtocol(ProtocolDataRetention)
+	conn.ID = fmt.Sprintf("data_retention_%v", user.Username)
+	check.conn = conn
+	check.sendNotifications(1*time.Second, nil)
+	err := check.sendHookNotification(1*time.Second, nil)
+	assert.NoError(t, err)
+
+	Config.DataRetentionHook = fmt.Sprintf("http://%v/404", httpAddr)
+	err = check.sendHookNotification(1*time.Second, nil)
+	assert.ErrorIs(t, err, errUnexpectedHTTResponse)
+
+	Config.DataRetentionHook = "http://foo\x7f.com/retention"
+	err = check.sendHookNotification(1*time.Second, err)
+	assert.Error(t, err)
+
+	Config.DataRetentionHook = "relativepath"
+	err = check.sendHookNotification(1*time.Second, err)
+	assert.Error(t, err)
+
+	if runtime.GOOS != osWindows {
+		hookCmd, err := exec.LookPath("true")
+		assert.NoError(t, err)
+
+		Config.DataRetentionHook = hookCmd
+		err = check.sendHookNotification(1*time.Second, err)
+		assert.NoError(t, err)
+	}
+
+	Config.DataRetentionHook = dataRetentionHook
+}
+
 func TestRetentionPermissionsAndGetFolder(t *testing.T) {
 	user := dataprovider.User{
 		BaseUser: sdk.BaseUser{
@ -224,6 +292,7 @@ func TestRetentionCheckAddRemove(t *testing.T) {
 				Retention: 48,
 			},
 		},
+		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
 	}
 	assert.NotNil(t, RetentionChecks.Add(check, &user))
 	checks := RetentionChecks.Get()
@ -233,6 +302,8 @@ func TestRetentionCheckAddRemove(t *testing.T) {
 	require.Len(t, checks[0].Folders, 1)
 	assert.Equal(t, check.Folders[0].Path, checks[0].Folders[0].Path)
 	assert.Equal(t, check.Folders[0].Retention, checks[0].Folders[0].Retention)
+	require.Len(t, checks[0].Notifications, 1)
+	assert.Equal(t, RetentionCheckNotificationHook, checks[0].Notifications[0])

 	assert.Nil(t, RetentionChecks.Add(check, &user))
 	assert.True(t, RetentionChecks.remove(username))
--- a/config/config.go
+++ b/config/config.go
@ -134,6 +134,7 @@ func Init() {
 			ProxyProtocol:         0,
 			ProxyAllowed:          []string{},
 			PostConnectHook:       "",
+			DataRetentionHook:     "",
 			MaxTotalConnections:   0,
 			MaxPerHostConnections: 20,
 			DefenderConfig: common.DefenderConfig{
@ -453,6 +454,7 @@ func getRedactedGlobalConf() globalConfig {
 	conf.Common.Actions.Hook = util.GetRedactedURL(conf.Common.Actions.Hook)
 	conf.Common.StartupHook = util.GetRedactedURL(conf.Common.StartupHook)
 	conf.Common.PostConnectHook = util.GetRedactedURL(conf.Common.PostConnectHook)
+	conf.Common.DataRetentionHook = util.GetRedactedURL(conf.Common.DataRetentionHook)
 	conf.SFTPD.KeyboardInteractiveHook = util.GetRedactedURL(conf.SFTPD.KeyboardInteractiveHook)
 	conf.HTTPDConfig.SigningPassphrase = getRedactedPassword()
 	conf.ProviderConf.Password = getRedactedPassword()
@ -1052,6 +1054,7 @@ func setViperDefaults() {
 	viper.SetDefault("common.proxy_protocol", globalConf.Common.ProxyProtocol)
 	viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed)
 	viper.SetDefault("common.post_connect_hook", globalConf.Common.PostConnectHook)
+	viper.SetDefault("common.data_retention_hook", globalConf.Common.DataRetentionHook)
 	viper.SetDefault("common.max_total_connections", globalConf.Common.MaxTotalConnections)
 	viper.SetDefault("common.max_per_host_connections", globalConf.Common.MaxPerHostConnections)
 	viper.SetDefault("common.defender.enabled", globalConf.Common.DefenderConfig.Enabled)
--- a/docs/data-retention-hook.md
+++ b/docs/data-retention-hook.md
@ -0,0 +1,32 @@
+# Data retention hook
+
+This hook runs after a data retention check completes if you specify `Hook` between notifications methods when you start the check.
+
+The `data_retention_hook` can be defined as the absolute path of your program or an HTTP URL.
+
+If the hook defines an external program it can read the following environment variable:
+
+- `SFTPGO_DATA_RETENTION_RESULT`, it contains the data retention check result JSON serialized.
+
+Previous global environment variables aren't cleared when the script is called.
+The program must finish within 20 seconds.
+
+If the hook defines an HTTP URL then this URL will be invoked as HTTP POST and the POST body contains the data retention check result JSON serialized.
+
+The HTTP hook will use the global configuration for HTTP clients and will respect the retry configurations.
+
+Here is the schema for the data retention check result:
+
+- `username`, string
+- `status`, int. 1 means success, 0 error
+- `start_time`, int64. Start time as UNIX timestamp in milliseconds
+- `total_deleted_files`, int. Total number of files deleted
+- `total_deleted_size`, int64. Total size deleted in bytes
+- `elapsed`, int64. Elapsed time in milliseconds
+- `details`, list of struct with details for each checked path, each struct contains the following fields:
+  - `path`, string
+  - `retention`, int. Retention time in hours
+  - `deleted_files`, int. Number of files deleted
+  - `deleted_size`, int64. Size deleted in bytes
+  - `info`, string. Informative, non fatal, message if any. For example it can indicates that the check was skipped because the user doesn't have the required permissions on this path
+  - `error`, string. Error message if any
--- a/docs/full-configuration.md
+++ b/docs/full-configuration.md
@ -69,6 +69,7 @@ The configuration file contains the following sections:
    - If `proxy_protocol` is set to 2 and we receive a proxy header from an IP that is not in the list then the connection will be rejected
  - `startup_hook`, string. Absolute path to an external program or an HTTP URL to invoke as soon as SFTPGo starts. If you define an HTTP URL it will be invoked using a `GET` request. Please note that SFTPGo services may not yet be available when this hook is run. Leave empty do disable
  - `post_connect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post connect hook](./post-connect-hook.md) for more details. Leave empty to disable
+  - `data_retention_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Data retention hook](./data-retention-hook.md) for more details. Leave empty to disable
  - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited. Default: 0.
  - `max_per_host_connections`, integer.  Maximum number of concurrent client connections from the same host (IP). If the defender is enabled, exceeding this limit will generate `score_limit_exceeded` events and thus hosts that repeatedly exceed the max allowed connections can be automatically blocked. 0 means unlimited. Default: 20.
  - `defender`, struct containing the defender configuration. See [Defender](./defender.md) for more details.
--- a/docs/post-connect-hook.md
+++ b/docs/post-connect-hook.md
@ -4,7 +4,7 @@ This hook is executed as soon as a new connection is established. It notifies th

 Please keep in mind that you can easily configure specialized program such as [Fail2ban](http://www.fail2ban.org/) for brute force protection. Executing a hook for each connection can be heavy.

-The `post-connect-hook` can be defined as the absolute path of your program or an HTTP URL.
+The `post_connect_hook` can be defined as the absolute path of your program or an HTTP URL.

 If the hook defines an external program it can read the following environment variables:

--- a/go.mod
+++ b/go.mod
@ -63,7 +63,7 @@ require (
 	gocloud.dev v0.24.0
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
 	golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6
-	golang.org/x/sys v0.0.0-20211002104244-808efd93c36d
+	golang.org/x/sys v0.0.0-20211003122950-b1ebd4e1001c
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
 	google.golang.org/api v0.58.0
 	google.golang.org/genproto v0.0.0-20211001223012-bfb93cce50d9 // indirect
--- a/go.sum
+++ b/go.sum
@ -962,8 +962,8 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211002104244-808efd93c36d h1:SABT8Vei3iTiu+Gy8KOzpSNz+W1EQ5YBCRtiEETxF+0=
-golang.org/x/sys v0.0.0-20211002104244-808efd93c36d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211003122950-b1ebd4e1001c h1:EyJTLQbOxvk8V6oDdD8ILR1BOs3nEJXThD6aqsiPNkM=
+golang.org/x/sys v0.0.0-20211003122950-b1ebd4e1001c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
--- a/httpd/api_retention.go
+++ b/httpd/api_retention.go
@ -3,11 +3,13 @@ package httpd
 import (
 	"fmt"
 	"net/http"
+	"strings"

 	"github.com/go-chi/render"

 	"github.com/drakkan/sftpgo/v2/common"
 	"github.com/drakkan/sftpgo/v2/dataprovider"
+	"github.com/drakkan/sftpgo/v2/util"
 )

 func getRetentionChecks(w http.ResponseWriter, r *http.Request) {
@ -29,19 +31,27 @@ func startRetentionCheck(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
 		return
 	}
-	check.Notification = r.URL.Query().Get("notify")
-	if check.Notification == common.RetentionCheckNotificationEmail {
-		claims, err := getTokenClaims(r)
-		if err != nil {
-			sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest)
-			return
+	for _, val := range strings.Split(r.URL.Query().Get("notifications"), ",") {
+		val = strings.TrimSpace(val)
+		if val != "" {
+			check.Notifications = append(check.Notifications, val)
 		}
-		admin, err := dataprovider.AdminExists(claims.Username)
-		if err != nil {
-			sendAPIResponse(w, r, err, "", getRespStatus(err))
-			return
+	}
+	check.Notifications = util.RemoveDuplicates(check.Notifications)
+	for _, notification := range check.Notifications {
+		if notification == common.RetentionCheckNotificationEmail {
+			claims, err := getTokenClaims(r)
+			if err != nil {
+				sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest)
+				return
+			}
+			admin, err := dataprovider.AdminExists(claims.Username)
+			if err != nil {
+				sendAPIResponse(w, r, err, "", getRespStatus(err))
+				return
+			}
+			check.Email = admin.Email
 		}
-		check.Email = admin.Email
 	}
 	if err := check.Validate(); err != nil {
 		sendAPIResponse(w, r, err, "Invalid retention check", http.StatusBadRequest)
--- a/httpd/httpd_test.go
+++ b/httpd/httpd_test.go
@ -1637,7 +1637,7 @@ func TestRetentionAPI(t *testing.T) {

 	asJSON, err := json.Marshal(folderRetention)
 	assert.NoError(t, err)
-	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notify=Email",
+	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notifications=Email,",
 		bytes.NewBuffer(asJSON))
 	setBearerForReq(req, token)
 	rr = executeRequest(req)
@ -1646,7 +1646,7 @@ func TestRetentionAPI(t *testing.T) {

 	_, err = httpdtest.RemoveAdmin(admin, http.StatusOK)
 	assert.NoError(t, err)
-	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notify=Email",
+	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notifications=Email",
 		bytes.NewBuffer(asJSON))
 	setBearerForReq(req, token)
 	rr = executeRequest(req)
--- a/httpd/internal_test.go
+++ b/httpd/internal_test.go
@ -504,7 +504,7 @@ func TestRetentionInvalidTokenClaims(t *testing.T) {
 	}
 	asJSON, err := json.Marshal(folderRetention)
 	assert.NoError(t, err)
-	req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+username+"/check?notify=Email", bytes.NewBuffer(asJSON))
+	req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+username+"/check?notifications=Email", bytes.NewBuffer(asJSON))

 	rctx := chi.NewRouteContext()
 	rctx.URLParams.Add("username", username)
--- a/httpd/schema/openapi.yaml
+++ b/httpd/schema/openapi.yaml
@ -772,11 +772,14 @@ paths:
        required: true
        schema:
          type: string
-      - name: notify
+      - name: notifications
        in: query
-        description: 'specify how notify results'
+        description: 'specify how to notify results'
+        explode: false
        schema:
-          $ref: '#/components/schemas/RetentionCheckNotification'
+          type: array
+          items:
+            $ref: '#/components/schemas/RetentionCheckNotification'
    post:
      tags:
        - data retention
@ -3231,11 +3234,11 @@ components:
    RetentionCheckNotification:
      type: string
      enum:
-        - None
+        - Hook
        - Email
      description: |
        Options:
-          * `None` - no notification, the results are recorded in the logs
+          * `Hook` - notify result using the defined hook. A "data_retention_hook" must be defined in your configuration file for this to work
          * `Email` - notify results by email. The admin starting the retention check must have an associated email address and the SMTP server must be configured for this to work
    APIKeyScope:
      type: integer
@ -4005,8 +4008,10 @@ components:
          type: integer
          format: int64
          description: check start time as unix timestamp in milliseconds
-        notification:
-          $ref: '#/components/schemas/RetentionCheckNotification'
+        notifications:
+          type: array
+          items:
+            $ref: '#/components/schemas/RetentionCheckNotification'
        email:
          type: string
          format: email
--- a/sftpgo.json
+++ b/sftpgo.json
@ -13,6 +13,7 @@
    "proxy_allowed": [],
    "startup_hook": "",
    "post_connect_hook": "",
+    "data_retention_hook": "",
    "max_total_connections": 0,
    "max_per_host_connections": 20,
    "defender": {
--- a/templates/email/retention-check-report.html
+++ b/templates/email/retention-check-report.html
@ -27,7 +27,5 @@ Elapsed: {{.Elapsed}}
    Files deleted: {{.DeletedFiles}}
    <br>
    Size deleted: {{call $.HumanizeSize .DeletedSize}}
-    <br>
-    Elapsed: {{.Elapsed}}
 </p>
 {{end}}
--- a/util/util.go
+++ b/util/util.go
@ -602,6 +602,9 @@ func ParseAllowedIPAndRanges(allowed []string) ([]func(net.IP) bool, error) {

 // GetRedactedURL returns the url redacting the password if any
 func GetRedactedURL(rawurl string) string {
+	if !strings.HasPrefix(rawurl, "http") {
+		return rawurl
+	}
 	u, err := url.Parse(rawurl)
 	if err != nil {
 		return rawurl