Drop: pogreb support (#175)

followup of #173

close #95

Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/175
This commit is contained in:
6543 2023-02-11 02:04:57 +00:00
parent 272c7ca76f
commit fd643d15f0
12 changed files with 6 additions and 262 deletions

View file

@ -65,19 +65,6 @@ pipeline:
- RAW_DOMAIN=raw.localhost.mock.directory - RAW_DOMAIN=raw.localhost.mock.directory
- PORT=4430 - PORT=4430
# TODO: remove in next version
integration-tests-legacy:
group: test
image: codeberg.org/6543/docker-images/golang_just
commands:
- just integration
environment:
- ACME_API=https://acme.mock.directory
- PAGES_DOMAIN=localhost.mock.directory
- RAW_DOMAIN=raw.localhost.mock.directory
- PORT=4430
- DB_TYPE=
release: release:
image: plugins/gitea-release image: plugins/gitea-release
settings: settings:

View file

@ -27,7 +27,7 @@ fmt: tool-gofumpt
clean: clean:
go clean ./... go clean ./...
rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json rm -rf build/ integration/certs.sqlite integration/acme-account.json
tool-golangci: tool-golangci:
@hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \

View file

@ -4,11 +4,7 @@ import (
"fmt" "fmt"
"time" "time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/database"
) )
var Certs = &cli.Command{ var Certs = &cli.Command{
@ -25,63 +21,8 @@ var Certs = &cli.Command{
Usage: "remove a certificate from the database", Usage: "remove a certificate from the database",
Action: removeCert, Action: removeCert,
}, },
{
Name: "migrate",
Usage: "migrate from \"pogreb\" driver to dbms driver",
Action: migrateCerts,
}, },
}, Flags: CertStorageFlags,
Flags: append(CertStorageFlags, []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
Usage: "print trace info",
EnvVars: []string{"VERBOSE"},
Value: false,
},
}...),
}
func migrateCerts(ctx *cli.Context) error {
dbType := ctx.String("db-type")
if dbType == "" {
dbType = "sqlite3"
}
dbConn := ctx.String("db-conn")
dbPogrebConn := ctx.String("db-pogreb")
verbose := ctx.Bool("verbose")
log.Level(zerolog.InfoLevel)
if verbose {
log.Level(zerolog.TraceLevel)
}
xormDB, err := database.NewXormDB(dbType, dbConn)
if err != nil {
return fmt.Errorf("could not connect to database: %w", err)
}
defer xormDB.Close()
pogrebDB, err := database.NewPogreb(dbPogrebConn)
if err != nil {
return fmt.Errorf("could not open database: %w", err)
}
defer pogrebDB.Close()
fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
certs, err := pogrebDB.Items(0, 0)
if err != nil {
return err
}
for _, cert := range certs {
if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
return err
}
}
fmt.Println("... done")
return nil
} }
func listCerts(ctx *cli.Context) error { func listCerts(ctx *cli.Context) error {

View file

@ -6,16 +6,9 @@ import (
var ( var (
CertStorageFlags = []cli.Flag{ CertStorageFlags = []cli.Flag{
&cli.StringFlag{
// TODO: remove in next version
// DEPRICATED
Name: "db-pogreb",
Value: "key-database.pogreb",
EnvVars: []string{"DB_POGREB"},
},
&cli.StringFlag{ &cli.StringFlag{
Name: "db-type", Name: "db-type",
Value: "", // TODO: "sqlite3" in next version Value: "sqlite3",
EnvVars: []string{"DB_TYPE"}, EnvVars: []string{"DB_TYPE"},
}, },
&cli.StringFlag{ &cli.StringFlag{

View file

@ -10,30 +10,10 @@ import (
) )
func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
if ctx.String("db-type") != "" {
log.Trace().Msg("use xorm mode")
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err) return nil, nil, fmt.Errorf("could not connect to database: %w", err)
} }
} else {
// TODO: remove in next version
fmt.Println(`
######################
## W A R N I N G !!! #
######################
You use "pogreb" witch is deprecated and will be removed in the next version.
Please switch to sqlite, mysql or postgres !!!
The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`)
log.Error().Msg("depricated \"pogreb\" used\n")
certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
if err != nil {
return nil, nil, fmt.Errorf("could not create database: %w", err)
}
}
closeFn = func() { closeFn = func() {
if err := certDB.Close(); err != nil { if err := certDB.Close(); err != nil {

1
go.mod
View file

@ -5,7 +5,6 @@ go 1.20
require ( require (
code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
github.com/akrylysov/pogreb v0.10.1
github.com/go-acme/lego/v4 v4.5.3 github.com/go-acme/lego/v4 v4.5.3
github.com/go-sql-driver/mysql v1.6.0 github.com/go-sql-driver/mysql v1.6.0
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0

2
go.sum
View file

@ -71,8 +71,6 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs=
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

View file

@ -510,14 +510,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
} }
} }
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
if err != nil {
log.Error().Err(err).Msg("Compacting key database failed")
} else {
log.Debug().Msgf("Compacted key database: %s", msg)
}
} }
// update main cert // update main cert

View file

@ -14,8 +14,6 @@ type CertDB interface {
Get(name string) (*certificate.Resource, error) Get(name string) (*certificate.Resource, error)
Delete(key string) error Delete(key string) error
Items(page, pageSize int) ([]*Cert, error) Items(page, pageSize int) ([]*Cert, error)
// Compact deprecated // TODO: remove in next version
Compact() (string, error)
} }
type Cert struct { type Cert struct {

View file

@ -37,11 +37,6 @@ func (p tmpDB) Delete(key string) error {
return nil return nil
} }
func (p tmpDB) Compact() (string, error) {
p.intern.Truncate()
return "Truncate done", nil
}
func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
return nil, fmt.Errorf("items not implemented for tmpDB") return nil, fmt.Errorf("items not implemented for tmpDB")
} }

View file

@ -1,134 +0,0 @@
package database
import (
"bytes"
"context"
"encoding/gob"
"errors"
"fmt"
"time"
"github.com/akrylysov/pogreb"
"github.com/akrylysov/pogreb/fs"
"github.com/go-acme/lego/v4/certificate"
"github.com/rs/zerolog/log"
)
var _ CertDB = aDB{}
type aDB struct {
ctx context.Context
cancel context.CancelFunc
intern *pogreb.DB
syncInterval time.Duration
}
func (p aDB) Close() error {
p.cancel()
return p.intern.Sync()
}
func (p aDB) Put(name string, cert *certificate.Resource) error {
var resGob bytes.Buffer
if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
return err
}
return p.intern.Put([]byte(name), resGob.Bytes())
}
func (p aDB) Get(name string) (*certificate.Resource, error) {
cert := &certificate.Resource{}
resBytes, err := p.intern.Get([]byte(name))
if err != nil {
return nil, err
}
if resBytes == nil {
return nil, nil
}
if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
return nil, err
}
return cert, nil
}
func (p aDB) Delete(key string) error {
return p.intern.Delete([]byte(key))
}
func (p aDB) Compact() (string, error) {
result, err := p.intern.Compact()
if err != nil {
return "", err
}
return fmt.Sprintf("%+v", result), nil
}
func (p aDB) Items(_, _ int) ([]*Cert, error) {
items := make([]*Cert, 0, p.intern.Count())
iterator := p.intern.Items()
for {
key, resBytes, err := iterator.Next()
if err != nil {
if errors.Is(err, pogreb.ErrIterationDone) {
break
}
return nil, err
}
res := &certificate.Resource{}
if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
return nil, err
}
cert, err := toCert(string(key), res)
if err != nil {
return nil, err
}
items = append(items, cert)
}
return items, nil
}
var _ CertDB = &aDB{}
func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Error().Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func NewPogreb(path string) (CertDB, error) {
if path == "" {
return nil, fmt.Errorf("path not set")
}
db, err := pogreb.Open(path, &pogreb.Options{
BackgroundSyncInterval: 30 * time.Second,
BackgroundCompactionInterval: 6 * time.Hour,
FileSystem: fs.OSMMap,
})
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
result := &aDB{
ctx: ctx,
cancel: cancel,
intern: db,
syncInterval: 5 * time.Minute,
}
go result.sync()
return result, nil
}

View file

@ -106,11 +106,6 @@ func (x xDB) Delete(domain string) error {
return err return err
} }
func (x xDB) Compact() (string, error) {
// not needed
return "", nil
}
// Items return al certs from db, if pageSize is 0 it does not use limit // Items return al certs from db, if pageSize is 0 it does not use limit
func (x xDB) Items(page, pageSize int) ([]*Cert, error) { func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
// paginated return // paginated return