Add pipeline (#65)

close #54

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/65
Reviewed-by: Andreas Shimokawa <ashimokawa@noreply.codeberg.org>
This commit is contained in:
6543 2022-03-27 21:54:06 +02:00
parent a5504acb0e
commit f5d0dc7447
12 changed files with 58 additions and 36 deletions

1
.gitignore vendored
View file

@ -4,3 +4,4 @@ key-database.pogreb/
acme-account.json
build/
vendor/
pages

25
.woodpecker.yml Normal file
View file

@ -0,0 +1,25 @@
pipeline:
# use vendor to cache dependencies
vendor:
image: golang
commands:
- go mod vendor
lint:
image: golangci/golangci-lint:v1.45.2
commands:
- go version
- go install mvdan.cc/gofumpt@latest
- "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
- golangci-lint run
# # TODO: make tests work
# test:
# image: golang
# commands:
# - go test ./...
build:
image: golang
commands:
- go build

View file

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"github.com/akrylysov/pogreb"
"github.com/urfave/cli/v2"
@ -12,12 +13,12 @@ var Certs = &cli.Command{
Name: "certs",
Usage: "manage certs manually",
Subcommands: []*cli.Command{
&cli.Command{
{
Name: "list",
Usage: "list all certificates in the database",
Action: listCerts,
},
&cli.Command{
{
Name: "remove",
Usage: "remove a certificate from the database",
Action: removeCert,

View file

@ -72,14 +72,14 @@ func Serve(ctx *cli.Context) error {
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
var canonicalDomainCache = cache.NewKeyValueCache()
canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = cache.NewKeyValueCache()
dnsLookupCache := cache.NewKeyValueCache()
// branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = cache.NewKeyValueCache()
branchTimestampCache := cache.NewKeyValueCache()
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
var fileResponseCache = cache.NewKeyValueCache()
fileResponseCache := cache.NewKeyValueCache()
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),

View file

@ -9,10 +9,8 @@ import (
"codeberg.org/codeberg/pages/cmd"
)
var (
// can be changed with -X on compile
version = "dev"
)
// can be changed with -X on compile
var version = "dev"
func main() {
app := cli.NewApp()

View file

@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{}
func (u *AcmeAccount) GetEmail() string {
return u.Email
}
func (u AcmeAccount) GetRegistration() *registration.Resource {
return u.Registration
}
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
return u.Key
}

View file

@ -40,7 +40,8 @@ func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaAPIToken, dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB) *tls.Config {
certDB database.CertDB,
) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@ -146,8 +147,10 @@ func checkUserLimit(user string) error {
return nil
}
var acmeClient, mainDomainAcmeClient *lego.Client
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
var (
acmeClient, mainDomainAcmeClient *lego.Client
acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
)
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
@ -166,6 +169,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
@ -181,6 +185,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
@ -388,7 +393,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {}

View file

@ -13,6 +13,8 @@ import (
"github.com/rs/zerolog/log"
)
var _ CertDB = aDB{}
type aDB struct {
ctx context.Context
cancel context.CancelFunc
@ -76,20 +78,6 @@ func (p aDB) sync() {
}
}
func (p aDB) compact() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func New(path string) (CertDB, error) {
if path == "" {
return nil, fmt.Errorf("path not set")

View file

@ -18,7 +18,8 @@ import (
func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@ -72,14 +73,14 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string
var targetOptions = &upstream.Options{
targetOptions := &upstream.Options{
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
if repo == "" {
return false
}

View file

@ -2,10 +2,11 @@ package server
import (
"fmt"
"github.com/valyala/fasthttp"
"testing"
"time"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
)

View file

@ -19,8 +19,8 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)

View file

@ -27,7 +27,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = branch
if branch == "" {
// Get default branch
var body = make([]byte, 0)
body := make([]byte, 0)
// TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
@ -37,7 +37,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br
result.Branch = fastjson.GetString(body, "default_branch")
}
var body = make([]byte, 0)
body := make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
return nil