feat: create archiver

This commit is contained in:
Fernandez Ludovic 2026-03-06 10:57:01 +01:00
commit 45c92d75be
9 changed files with 709 additions and 156 deletions

View file

@ -0,0 +1,77 @@
package storage
import (
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/go-acme/lego/v5/log"
"github.com/mattn/go-zglob"
)
const maxTimeBeforeCleaning = 30 * 24 * time.Hour
const (
baseArchivesFolderName = "archives"
)
// Archiver manages the archiving of accounts and certificates.
type Archiver struct {
basePath string
maxTimeBeforeCleaning time.Duration
accountsBasePath string
certificatesBasePath string
accountsArchivePath string
certificatesArchivePath string
}
// NewArchiver creates a new Archiver.
func NewArchiver(basePath string) *Archiver {
return &Archiver{
basePath: basePath,
maxTimeBeforeCleaning: maxTimeBeforeCleaning,
accountsBasePath: filepath.Join(basePath, baseAccountsRootFolderName),
certificatesBasePath: filepath.Join(basePath, baseCertificatesFolderName),
accountsArchivePath: filepath.Join(basePath, baseArchivesFolderName, baseAccountsRootFolderName),
certificatesArchivePath: filepath.Join(basePath, baseArchivesFolderName, baseCertificatesFolderName),
}
}
func (m *Archiver) cleanArchives(pattern string) error {
matches, err := zglob.Glob(pattern)
if err != nil {
return err
}
for _, filename := range matches {
li := strings.LastIndex(filename, "_")
v := strings.TrimSuffix(filename[li+1:], ".zip")
s, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return err
}
if time.Unix(s, 0).Add(m.maxTimeBeforeCleaning).After(time.Now()) {
log.Debug("The archive is not old enough to be cleaned.", slog.String("filename", filename))
continue
}
err = os.Remove(filename)
if err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,149 @@
package storage
import (
"fmt"
"log/slog"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/go-acme/lego/v5/cmd/internal/configuration"
"github.com/go-acme/lego/v5/log"
"github.com/mattn/go-zglob"
)
func (m *Archiver) Accounts(cfg *configuration.Configuration) error {
err := m.cleanArchivedAccounts()
if err != nil {
return fmt.Errorf("clean archived accounts: %w", err)
}
err = m.archiveAccounts(cfg)
if err != nil {
return fmt.Errorf("archive accounts: %w", err)
}
return nil
}
func (m *Archiver) archiveAccounts(cfg *configuration.Configuration) error {
_, err := os.Stat(m.accountsBasePath)
if os.IsNotExist(err) {
return nil
}
matches, err := zglob.Glob(filepath.Join(m.accountsBasePath, "**", accountFileName))
if err != nil {
return fmt.Errorf("search account files: %w", err)
}
accountTree, err := accountMapping(cfg)
if err != nil {
return fmt.Errorf("account mapping: %w", err)
}
date := strconv.FormatInt(time.Now().Unix(), 10)
for _, filename := range matches {
dirKt, _ := filepath.Split(filename)
dirAcc, kt := filepath.Split(filepath.Dir(dirKt))
dirSrv, accID := filepath.Split(filepath.Dir(dirAcc))
_, srv := filepath.Split(filepath.Dir(dirSrv))
if _, ok := accountTree[srv]; !ok {
err = m.archiveAccount("server", dirSrv, srv, date)
if err != nil {
return fmt.Errorf("archive account (server) %q: %w", srv, err)
}
continue
}
if _, ok := accountTree[srv][accID]; !ok {
err = m.archiveAccount("accountID", dirAcc, srv, accID, date)
if err != nil {
return fmt.Errorf("archive account (accountID) %q: %w", accID, err)
}
continue
}
if _, ok := accountTree[srv][accID][kt]; !ok {
err := m.archiveAccount("keyType", dirKt, srv, accID, kt, date)
if err != nil {
return fmt.Errorf("archive account (keyType) %q: %w", kt, err)
}
continue
}
}
return nil
}
func (m *Archiver) archiveAccount(scope, dir string, parts ...string) error {
dest := filepath.Join(m.accountsArchivePath, strings.Join(parts, "_")+".zip")
log.Info("Archive account",
slog.String("scope", scope),
slog.String("filepath", dir),
slog.String("archives", dest),
)
err := CreateNonExistingFolder(filepath.Dir(dest))
if err != nil {
return fmt.Errorf("could not check/create the accounts archive folder %q: %w", filepath.Dir(dest), err)
}
rel, err := filepath.Rel(m.basePath, dir)
if err != nil {
return err
}
err = compressDirectory(dest, dir, rel)
if err != nil {
return fmt.Errorf("compress account files: %w", err)
}
return os.RemoveAll(dir)
}
func (m *Archiver) cleanArchivedAccounts() error {
_, err := os.Stat(m.accountsArchivePath)
if os.IsNotExist(err) {
return nil
}
return m.cleanArchives(filepath.Join(m.accountsArchivePath, "**", "*.zip"))
}
func accountMapping(cfg *configuration.Configuration) (map[string]map[string]map[string]struct{}, error) {
// Server -> AccountID -> KeyType
accountTree := make(map[string]map[string]map[string]struct{})
for accID, account := range cfg.Accounts {
serverConfig := configuration.GetServerConfig(cfg, accID)
s, err := url.Parse(serverConfig.URL)
if err != nil {
return nil, err
}
server := sanitizeHost(s)
if _, ok := accountTree[server]; !ok {
accountTree[server] = make(map[string]map[string]struct{})
}
if _, ok := accountTree[server][accID]; !ok {
accountTree[server][accID] = make(map[string]struct{})
}
accountTree[server][accID][account.KeyType] = struct{}{}
}
return accountTree, nil
}

View file

@ -0,0 +1,70 @@
package storage
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/go-acme/lego/v5/cmd/internal/configuration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestArchiver_Accounts(t *testing.T) {
cfg := &configuration.Configuration{
Storage: t.TempDir(),
Accounts: map[string]*configuration.Account{
"foo": {
Server: "https://ca.example.com/dir",
KeyType: "EC256",
},
},
}
archiver := NewArchiver(cfg.Storage)
archiver.maxTimeBeforeCleaning = 0
err := os.MkdirAll(archiver.accountsBasePath, 0o700)
require.NoError(t, err)
generateFakeAccountFiles(t, archiver.accountsBasePath, "ca.example.com", "EC256", "foo")
generateFakeAccountFiles(t, archiver.accountsBasePath, "ca.example.com", "EC256", "bar")
// archive
err = archiver.Accounts(cfg)
require.NoError(t, err)
entries, err := os.ReadDir(archiver.accountsArchivePath)
require.NoError(t, err)
assert.Len(t, entries, 1)
// clean
err = archiver.Accounts(cfg)
require.NoError(t, err)
entries, err = os.ReadDir(archiver.accountsArchivePath)
require.NoError(t, err)
assert.Empty(t, entries)
}
func generateFakeAccountFiles(t *testing.T, accountsBasePath, server, keyType, accountID string) {
t.Helper()
filename := filepath.Join(accountsBasePath, server, accountID, keyType, "account.json")
err := os.MkdirAll(filepath.Dir(filename), 0o700)
require.NoError(t, err)
file, err := os.Create(filename)
require.NoError(t, err)
r := Account{ID: accountID}
err = json.NewEncoder(file).Encode(r)
require.NoError(t, err)
}

View file

@ -0,0 +1,151 @@
package storage
import (
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/go-acme/lego/v5/certificate"
"github.com/go-acme/lego/v5/cmd/internal/configuration"
"github.com/go-acme/lego/v5/log"
"github.com/mattn/go-zglob"
)
func (m *Archiver) Certificates(certificates map[string]*configuration.Certificate) error {
err := m.cleanArchivedCertificates()
if err != nil {
return fmt.Errorf("clean archived certificates: %w", err)
}
err = m.archiveRemovedCertificates(certificates)
if err != nil {
return fmt.Errorf("archive removed certificates: %w", err)
}
return nil
}
func (m *Archiver) archiveRemovedCertificates(certificates map[string]*configuration.Certificate) error {
// Only archive the certificates that are not in the configuration.
return m.archiveCertificates(func(resourceID string) bool {
_, ok := certificates[resourceID]
return ok
})
}
func (m *Archiver) archiveCertificate(certID string) error {
return m.archiveCertificates(func(resourceID string) bool {
return certID != resourceID
})
}
func (m *Archiver) archiveCertificates(skip func(resourceID string) bool) error {
_, err := os.Stat(m.certificatesBasePath)
if os.IsNotExist(err) {
return nil
}
matches, err := zglob.Glob(filepath.Join(m.certificatesBasePath, "*.json"))
if err != nil {
return fmt.Errorf("search certificate files: %w", err)
}
date := strconv.FormatInt(time.Now().Unix(), 10)
for _, filename := range matches {
file, err := os.ReadFile(filename)
if err != nil {
return fmt.Errorf("reading certificate file %q: %w", filename, err)
}
resource := new(certificate.Resource)
err = json.Unmarshal(file, resource)
if err != nil {
return fmt.Errorf("unmarshalling certificate file %q: %w", filename, err)
}
if skip(resource.ID) {
continue
}
err = m.archiveOneCertificate(filename, date, resource)
if err != nil {
return fmt.Errorf("archive certificate %q: %w", resource.ID, err)
}
}
return nil
}
func (m *Archiver) archiveOneCertificate(filename, date string, resource *certificate.Resource) error {
dest := filepath.Join(m.certificatesArchivePath, strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))+"_"+date+".zip")
log.Info("Archive certificate", log.CertNameAttr(resource.ID), slog.String("archives", dest))
err := CreateNonExistingFolder(filepath.Dir(dest))
if err != nil {
return fmt.Errorf("could not check/create the certificates archive folder %q: %w", filepath.Dir(dest), err)
}
files, err := getCertificateFiles(filename, resource.ID)
if err != nil {
return err
}
rel, err := filepath.Rel(m.basePath, filepath.Dir(filename))
if err != nil {
return err
}
err = compressFiles(dest, files, rel)
if err != nil {
return fmt.Errorf("compress certificate files: %w", err)
}
for _, file := range files {
err = os.Remove(file)
if err != nil {
return fmt.Errorf("remove certificate file %q: %w", file, err)
}
}
return nil
}
func (m *Archiver) cleanArchivedCertificates() error {
_, err := os.Stat(m.certificatesArchivePath)
if os.IsNotExist(err) {
return nil
}
return m.cleanArchives(filepath.Join(m.certificatesArchivePath, "*.zip"))
}
func getCertificateFiles(filename, resourceID string) ([]string, error) {
files, err := filepath.Glob(filepath.Join(filepath.Dir(filename), SanitizedName(resourceID)+".*"))
if err != nil {
return nil, err
}
var restrictedFiles []string
baseFilename := filepath.Join(filepath.Dir(filename), SanitizedName(resourceID))
// Filter files to avoid ambiguous names (ex: foo.com and foo.com.uk)
for _, file := range files {
if strings.TrimSuffix(file, filepath.Ext(file)) != baseFilename && file != baseFilename+ExtIssuer {
continue
}
restrictedFiles = append(restrictedFiles, file)
}
return restrictedFiles, nil
}

View file

@ -0,0 +1,177 @@
package storage
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
"testing"
"github.com/go-acme/lego/v5/certificate"
"github.com/go-acme/lego/v5/cmd/internal/configuration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestArchiver_Certificates(t *testing.T) {
domain := "example.com"
archiveDomain := "example.org"
cfg := &configuration.Configuration{
Storage: t.TempDir(),
Certificates: map[string]*configuration.Certificate{
domain: {},
},
}
archiver := NewArchiver(cfg.Storage)
archiver.maxTimeBeforeCleaning = 0
domainFiles := generateFakeCertificateFiles(t, archiver.certificatesBasePath, domain)
_ = generateFakeCertificateFiles(t, archiver.certificatesBasePath, archiveDomain)
// archive
err := archiver.Certificates(cfg.Certificates)
require.NoError(t, err)
root, err := os.ReadDir(archiver.certificatesBasePath)
require.NoError(t, err)
assert.Len(t, root, len(domainFiles))
archive, err := os.ReadDir(archiver.certificatesArchivePath)
require.NoError(t, err)
require.Len(t, archive, 1)
assert.Regexp(t, regexp.QuoteMeta(archiveDomain)+`_\d+\.zip`, archive[0].Name())
// clean
err = archiver.Certificates(cfg.Certificates)
require.NoError(t, err)
archive, err = os.ReadDir(archiver.certificatesArchivePath)
require.NoError(t, err)
require.Empty(t, archive)
}
func TestArchiver_archiveCertificate(t *testing.T) {
domain := "example.com"
archiver := NewArchiver(t.TempDir())
domainFiles := generateFakeCertificateFiles(t, archiver.certificatesBasePath, domain)
err := archiver.archiveCertificate(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.NoFileExists(t, file)
}
root, err := os.ReadDir(archiver.certificatesBasePath)
require.NoError(t, err)
require.Empty(t, root)
archive, err := os.ReadDir(archiver.certificatesArchivePath)
require.NoError(t, err)
require.Len(t, archive, 1)
assert.Regexp(t, regexp.QuoteMeta(domain)+`_\d+\.zip`, archive[0].Name())
}
func TestArchiver_archiveCertificate_noFileRelatedToDomain(t *testing.T) {
domain := "example.com"
archiver := NewArchiver(t.TempDir())
domainFiles := generateFakeCertificateFiles(t, archiver.certificatesBasePath, "example.org")
err := archiver.archiveCertificate(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.FileExists(t, file)
}
root, err := os.ReadDir(archiver.certificatesBasePath)
require.NoError(t, err)
assert.Len(t, root, len(domainFiles))
assert.NoFileExists(t, archiver.certificatesArchivePath)
}
func TestArchiver_archiveCertificate_ambiguousDomain(t *testing.T) {
domain := "example.com"
archiver := NewArchiver(t.TempDir())
domainFiles := generateFakeCertificateFiles(t, archiver.certificatesBasePath, domain)
otherDomainFiles := generateFakeCertificateFiles(t, archiver.certificatesBasePath, domain+".example.org")
err := archiver.archiveCertificate(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.NoFileExists(t, file)
}
for _, file := range otherDomainFiles {
assert.FileExists(t, file)
}
root, err := os.ReadDir(archiver.certificatesBasePath)
require.NoError(t, err)
require.Len(t, root, len(otherDomainFiles))
archive, err := os.ReadDir(archiver.certificatesArchivePath)
require.NoError(t, err)
require.Len(t, archive, 1)
assert.Regexp(t, regexp.QuoteMeta(domain)+`_\d+\.zip`, archive[0].Name())
}
func assertCertificateFileContent(t *testing.T, basePath, filename string) {
t.Helper()
actual, err := os.ReadFile(filepath.Join(basePath, baseCertificatesFolderName, filename))
require.NoError(t, err)
expected, err := os.ReadFile(filepath.Join("testdata", baseCertificatesFolderName, filename))
require.NoError(t, err)
assert.Equal(t, string(expected), string(actual))
}
func generateFakeCertificateFiles(t *testing.T, dir, domain string) []string {
t.Helper()
err := CreateNonExistingFolder(dir)
require.NoError(t, err)
var filenames []string
for _, ext := range []string{ExtIssuer, ExtCert, ExtKey, ExtPEM, ExtPFX} {
filename := filepath.Join(dir, domain+ext)
err = os.WriteFile(filename, []byte("test"), 0o666)
require.NoError(t, err)
filenames = append(filenames, filename)
}
filename := filepath.Join(dir, domain+ExtResource)
file, err := os.Create(filename)
require.NoError(t, err)
r := certificate.Resource{ID: domain}
err = json.NewEncoder(file).Encode(r)
require.NoError(t, err)
filenames = append(filenames, filename)
return filenames
}

View file

@ -0,0 +1,79 @@
package storage
import (
"archive/zip"
"io"
"os"
"path/filepath"
)
func compressDirectory(name, dir, comment string) error {
file, err := os.Create(name)
if err != nil {
return err
}
writer := zip.NewWriter(file)
defer func() { _ = writer.Close() }()
root, err := os.OpenRoot(dir)
if err != nil {
return err
}
defer func() { _ = root.Close() }()
err = writer.SetComment(comment)
if err != nil {
return err
}
return writer.AddFS(root.FS())
}
func compressFiles(dest string, files []string, comment string) error {
file, err := os.Create(dest)
if err != nil {
return err
}
writer := zip.NewWriter(file)
defer func() { _ = writer.Close() }()
err = writer.SetComment(comment)
if err != nil {
return err
}
for _, file := range files {
f, err := writer.Create(filepath.Base(file))
if err != nil {
return err
}
err = addFileToZip(f, file)
if err != nil {
return err
}
}
return nil
}
func addFileToZip(f io.Writer, file string) error {
in, err := os.Open(file)
if err != nil {
return err
}
defer func() { _ = in.Close() }()
_, err = io.Copy(f, in)
if err != nil {
return err
}
return nil
}

View file

@ -27,7 +27,6 @@ const (
const (
baseCertificatesFolderName = "certificates"
baseArchivesFolderName = "archives"
)
// CertificatesStorage a certificates' storage.
@ -44,15 +43,16 @@ const (
// │ └── archived certificates directory
// └── "path" option
type CertificatesStorage struct {
rootPath string
archivePath string
archiver *Archiver
rootPath string
}
// NewCertificatesStorage create a new certificates storage.
func NewCertificatesStorage(basePath string) *CertificatesStorage {
return &CertificatesStorage{
rootPath: getCertificatesRootPath(basePath),
archivePath: getCertificatesArchivePath(basePath),
archiver: NewArchiver(basePath),
rootPath: filepath.Join(basePath, baseCertificatesFolderName),
}
}
@ -66,14 +66,6 @@ func CreateNonExistingFolder(path string) error {
return nil
}
func getCertificatesRootPath(basePath string) string {
return filepath.Join(basePath, baseCertificatesFolderName)
}
func getCertificatesArchivePath(basePath string) string {
return filepath.Join(basePath, baseArchivesFolderName)
}
// SanitizedName Make sure no funny chars are in the cert names (like wildcards ;)).
func SanitizedName(name string) string {
safe, err := idna.ToASCII(strings.NewReplacer(":", "-", "*", "_").Replace(name))

View file

@ -9,10 +9,6 @@ import (
"fmt"
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/go-acme/lego/v5/certcrypto"
"github.com/go-acme/lego/v5/certificate"
@ -93,34 +89,7 @@ func (s *CertificatesStorage) Save(certRes *certificate.Resource, opts *SaveOpti
// Archive moves the certificate files to the archive folder.
func (s *CertificatesStorage) Archive(certID string) error {
err := CreateNonExistingFolder(s.archivePath)
if err != nil {
return fmt.Errorf("could not check/create the archive folder %q: %w", s.archivePath, err)
}
baseFilename := filepath.Join(s.rootPath, SanitizedName(certID))
matches, err := filepath.Glob(baseFilename + ".*")
if err != nil {
return err
}
for _, oldFile := range matches {
if strings.TrimSuffix(oldFile, filepath.Ext(oldFile)) != baseFilename && oldFile != baseFilename+ExtIssuer {
continue
}
date := strconv.FormatInt(time.Now().Unix(), 10)
filename := date + "." + filepath.Base(oldFile)
newFile := filepath.Join(s.archivePath, filename)
err = os.Rename(oldFile, newFile)
if err != nil {
return err
}
}
return nil
return s.archiver.archiveCertificate(certID)
}
func (s *CertificatesStorage) saveResource(certRes *certificate.Resource) error {

View file

@ -3,7 +3,6 @@ package storage
import (
"os"
"path/filepath"
"regexp"
"testing"
"github.com/go-acme/lego/v5/certificate"
@ -104,113 +103,3 @@ func TestCertificatesStorage_Save_pem(t *testing.T) {
assert.JSONEq(t, string(expected), string(actual))
}
func TestCertificatesStorage_Archive(t *testing.T) {
domain := "example.com"
certStorage := NewCertificatesStorage(t.TempDir())
domainFiles := generateTestFiles(t, certStorage.rootPath, domain)
err := certStorage.Archive(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.NoFileExists(t, file)
}
root, err := os.ReadDir(certStorage.rootPath)
require.NoError(t, err)
require.Empty(t, root)
archive, err := os.ReadDir(certStorage.archivePath)
require.NoError(t, err)
require.Len(t, archive, len(domainFiles))
assert.Regexp(t, `\d+\.`+regexp.QuoteMeta(domain), archive[0].Name())
}
func TestCertificatesStorage_Archive_noFileRelatedToDomain(t *testing.T) {
domain := "example.com"
certStorage := NewCertificatesStorage(t.TempDir())
domainFiles := generateTestFiles(t, certStorage.rootPath, "example.org")
err := certStorage.Archive(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.FileExists(t, file)
}
root, err := os.ReadDir(certStorage.rootPath)
require.NoError(t, err)
assert.Len(t, root, len(domainFiles))
archive, err := os.ReadDir(certStorage.archivePath)
require.NoError(t, err)
assert.Empty(t, archive)
}
func TestCertificatesStorage_Archive_ambiguousDomain(t *testing.T) {
domain := "example.com"
certStorage := NewCertificatesStorage(t.TempDir())
domainFiles := generateTestFiles(t, certStorage.rootPath, domain)
otherDomainFiles := generateTestFiles(t, certStorage.rootPath, domain+".example.org")
err := certStorage.Archive(domain)
require.NoError(t, err)
for _, file := range domainFiles {
assert.NoFileExists(t, file)
}
for _, file := range otherDomainFiles {
assert.FileExists(t, file)
}
root, err := os.ReadDir(certStorage.rootPath)
require.NoError(t, err)
require.Len(t, root, len(otherDomainFiles))
archive, err := os.ReadDir(certStorage.archivePath)
require.NoError(t, err)
require.Len(t, archive, len(domainFiles))
assert.Regexp(t, `\d+\.`+regexp.QuoteMeta(domain), archive[0].Name())
}
func assertCertificateFileContent(t *testing.T, basePath, filename string) {
t.Helper()
actual, err := os.ReadFile(filepath.Join(basePath, baseCertificatesFolderName, filename))
require.NoError(t, err)
expected, err := os.ReadFile(filepath.Join("testdata", baseCertificatesFolderName, filename))
require.NoError(t, err)
assert.Equal(t, string(expected), string(actual))
}
func generateTestFiles(t *testing.T, dir, domain string) []string {
t.Helper()
err := CreateNonExistingFolder(dir)
require.NoError(t, err)
var filenames []string
for _, ext := range []string{ExtIssuer, ExtCert, ExtKey, ExtPEM, ExtPFX, ExtResource} {
filename := filepath.Join(dir, domain+ext)
err := os.WriteFile(filename, []byte("test"), 0o666)
require.NoError(t, err)
filenames = append(filenames, filename)
}
return filenames
}