Add pipeline (#65)

close #54

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/65
Reviewed-by: Andreas Shimokawa <ashimokawa@noreply.codeberg.org>
This commit is contained in:
6543 2022-03-27 21:54:06 +02:00
parent a5504acb0e
commit f5d0dc7447
12 changed files with 58 additions and 36 deletions

1
.gitignore vendored
View file

@ -4,3 +4,4 @@ key-database.pogreb/
acme-account.json acme-account.json
build/ build/
vendor/ vendor/
pages

25
.woodpecker.yml Normal file
View file

@ -0,0 +1,25 @@
pipeline:
# use vendor to cache dependencies
vendor:
image: golang
commands:
- go mod vendor
lint:
image: golangci/golangci-lint:v1.45.2
commands:
- go version
- go install mvdan.cc/gofumpt@latest
- "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
- golangci-lint run
# # TODO: make tests work
# test:
# image: golang
# commands:
# - go test ./...
build:
image: golang
commands:
- go build

View file

@ -2,6 +2,7 @@ package cmd
import ( import (
"fmt" "fmt"
"github.com/akrylysov/pogreb" "github.com/akrylysov/pogreb"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
@ -12,12 +13,12 @@ var Certs = &cli.Command{
Name: "certs", Name: "certs",
Usage: "manage certs manually", Usage: "manage certs manually",
Subcommands: []*cli.Command{ Subcommands: []*cli.Command{
&cli.Command{ {
Name: "list", Name: "list",
Usage: "list all certificates in the database", Usage: "list all certificates in the database",
Action: listCerts, Action: listCerts,
}, },
&cli.Command{ {
Name: "remove", Name: "remove",
Usage: "remove a certificate from the database", Usage: "remove a certificate from the database",
Action: removeCert, Action: removeCert,

View file

@ -72,14 +72,14 @@ func Serve(ctx *cli.Context) error {
keyCache := cache.NewKeyValueCache() keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache() challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains // canonicalDomainCache stores canonical domains
var canonicalDomainCache = cache.NewKeyValueCache() canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains // dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = cache.NewKeyValueCache() dnsLookupCache := cache.NewKeyValueCache()
// branchTimestampCache stores branch timestamps for faster cache checking // branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = cache.NewKeyValueCache() branchTimestampCache := cache.NewKeyValueCache()
// fileResponseCache stores responses from the Gitea server // fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit // TODO: make this an MRU cache with a size limit
var fileResponseCache = cache.NewKeyValueCache() fileResponseCache := cache.NewKeyValueCache()
// Create handler based on settings // Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain), handler := server.Handler(mainDomainSuffix, []byte(rawDomain),

View file

@ -9,10 +9,8 @@ import (
"codeberg.org/codeberg/pages/cmd" "codeberg.org/codeberg/pages/cmd"
) )
var (
// can be changed with -X on compile // can be changed with -X on compile
version = "dev" var version = "dev"
)
func main() { func main() {
app := cli.NewApp() app := cli.NewApp()

View file

@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{}
func (u *AcmeAccount) GetEmail() string { func (u *AcmeAccount) GetEmail() string {
return u.Email return u.Email
} }
func (u AcmeAccount) GetRegistration() *registration.Resource { func (u AcmeAccount) GetRegistration() *registration.Resource {
return u.Registration return u.Registration
} }
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
return u.Key return u.Key
} }

View file

@ -40,7 +40,8 @@ func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaAPIToken, dnsProvider string, giteaRoot, giteaAPIToken, dnsProvider string,
acmeUseRateLimits bool, acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB) *tls.Config { certDB database.CertDB,
) *tls.Config {
return &tls.Config{ return &tls.Config{
// check DNS name & get certificate from Let's Encrypt // check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@ -146,8 +147,10 @@ func checkUserLimit(user string) error {
return nil return nil
} }
var acmeClient, mainDomainAcmeClient *lego.Client var (
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} acmeClient, mainDomainAcmeClient *lego.Client
acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
)
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution? // TODO: when this is used a lot, we probably have to think of a somewhat better solution?
@ -166,6 +169,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
} }
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain) a.challengeCache.Remove(domain)
return nil return nil
@ -181,6 +185,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
} }
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token) a.challengeCache.Remove(domain + "/" + token)
return nil return nil
@ -388,7 +393,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {} select {}
} }
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600) err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil { if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {} select {}

View file

@ -13,6 +13,8 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
var _ CertDB = aDB{}
type aDB struct { type aDB struct {
ctx context.Context ctx context.Context
cancel context.CancelFunc cancel context.CancelFunc
@ -76,20 +78,6 @@ func (p aDB) sync() {
} }
} }
func (p aDB) compact() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func New(path string) (CertDB, error) { func New(path string) (CertDB, error) {
if path == "" { if path == "" {
return nil, fmt.Errorf("path not set") return nil, fmt.Errorf("path not set")

View file

@ -18,7 +18,8 @@ import (
func Handler(mainDomainSuffix, rawDomain []byte, func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string, giteaRoot, rawInfoPage, giteaAPIToken string,
blacklistedPaths, allowedCorsDomains [][]byte, blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@ -72,14 +73,14 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Prepare request information to Gitea // Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string var targetOwner, targetRepo, targetBranch, targetPath string
var targetOptions = &upstream.Options{ targetOptions := &upstream.Options{
ForbiddenMimeTypes: map[string]struct{}{}, ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true, TryIndexPages: true,
} }
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL. // also disallow search indexing and add a Link header to the canonical URL.
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool { tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
if repo == "" { if repo == "" {
return false return false
} }

View file

@ -2,10 +2,11 @@ package server
import ( import (
"fmt" "fmt"
"github.com/valyala/fasthttp"
"testing" "testing"
"time" "time"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
) )

View file

@ -19,8 +19,8 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
targetOwner, targetRepo, targetBranch, targetPath, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken string, giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) { canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain // check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)

View file

@ -27,7 +27,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = branch result.Branch = branch
if branch == "" { if branch == "" {
// Get default branch // Get default branch
var body = make([]byte, 0) body := make([]byte, 0)
// TODO: use header for API key? // TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 { if err != nil || status != 200 {
@ -37,7 +37,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = fastjson.GetString(body, "default_branch") result.Branch = fastjson.GetString(body, "default_branch")
} }
var body = make([]byte, 0) body := make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 { if err != nil || status != 200 {
return nil return nil