inject all cache

This commit is contained in:
6543 2021-12-05 15:02:44 +01:00
parent 2b49039252
commit b3830e979c
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
5 changed files with 44 additions and 25 deletions

View file

@ -64,8 +64,18 @@ func Serve(ctx *cli.Context) error {
mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
} }
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
var canonicalDomainCache = cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = cache.NewKeyValueCache()
// Create handler based on settings // Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain), giteaRoot, rawInfoPage, giteaAPIToken, BlacklistedPaths, allowedCorsDomains) handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaRoot, rawInfoPage, giteaAPIToken,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache)
// Enable compression by wrapping the handler with the compression function provided by FastHTTP // Enable compression by wrapping the handler with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
@ -93,11 +103,13 @@ func Serve(ctx *cli.Context) error {
if err != nil { if err != nil {
return fmt.Errorf("could not create database: %v", err) return fmt.Errorf("could not create database: %v", err)
} }
defer keyDatabase.Sync() // database has no close ... sync behave like it defer keyDatabase.Sync() //nolint:errcheck // database has no close ... sync behave like it
keyCache := cache.NewKeyValueCache() listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix,
challengeCache := cache.NewKeyValueCache() giteaRoot, giteaAPIToken, dnsProvider,
listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, challengeCache, keyDatabase)) acmeUseRateLimits,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
keyDatabase))
server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase)
if enableHTTPServer { if enableHTTPServer {

View file

@ -38,7 +38,11 @@ import (
) )
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) *tls.Config { func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaApiToken, dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
keyDatabase database.KeyDB) *tls.Config {
return &tls.Config{ return &tls.Config{
// check DNS name & get certificate from Let's Encrypt // check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@ -71,14 +75,14 @@ func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider st
sni = string(sniBytes) sni = string(sniBytes)
} else { } else {
var targetRepo, targetBranch string var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix)) targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" { if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs // DNS not set up, return main certificate to redirect to the docs
sniBytes = mainDomainSuffix sniBytes = mainDomainSuffix
sni = string(sniBytes) sni = string(sniBytes)
} else { } else {
_, _ = targetRepo, targetBranch _, _ = targetRepo, targetBranch
_, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken) _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache)
if !valid { if !valid {
sniBytes = mainDomainSuffix sniBytes = mainDomainSuffix
sni = string(sniBytes) sni = string(sniBytes)

View file

@ -5,21 +5,18 @@ import (
"strings" "strings"
"time" "time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/upstream"
) )
// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. // DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
var DnsLookupCacheTimeout = 15 * time.Minute var DnsLookupCacheTimeout = 15 * time.Minute
// dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = mcache.New()
// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data. // If everything is fine, it returns the target data.
func getTargetFromDNS(domain, mainDomainSuffix string) (targetOwner, targetRepo, targetBranch string) { func getTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT // Get CNAME or TXT
var cname string var cname string
var err error var err error
@ -68,11 +65,8 @@ func getTargetFromDNS(domain, mainDomainSuffix string) (targetOwner, targetRepo,
// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. // CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var CanonicalDomainCacheTimeout = 15 * time.Minute var CanonicalDomainCacheTimeout = 15 * time.Minute
// canonicalDomainCache stores canonical domains
var canonicalDomainCache = mcache.New()
// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). // checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`).
func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string) (canonicalDomain string, valid bool) { func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string, canonicalDomainCache cache.SetGetKey) (canonicalDomain string, valid bool) {
domains := []string{} domains := []string{}
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
domains = cachedValue.([]string) domains = cachedValue.([]string)

View file

@ -4,15 +4,20 @@ import (
"bytes" "bytes"
"strings" "strings"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils"
) )
// Handler handles a single HTTP request to the web server. // Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte) func(ctx *fasthttp.RequestCtx) { func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaApiToken string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@ -108,7 +113,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
var tryUpstream = func() { var tryUpstream = func() {
// check if a canonical domain exists on a request on MainDomain // check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken) canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI()) canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" { if targetRepo != "pages" {
@ -242,7 +247,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
trimmedHostStr := string(trimmedHost) trimmedHostStr := string(trimmedHost)
// Serve pages from external domains // Serve pages from external domains
targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix)) targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" { if targetOwner == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return return
@ -259,13 +264,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp
// Try to use the given repo on the given branch or the default branch // Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS") log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken) canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache)
if !valid { if !valid {
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return return
} else if canonicalDomain != trimmedHostStr { } else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page! // only redirect if the target is also a codeberg page!
targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
if targetOwner != "" { if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
return return

View file

@ -5,6 +5,8 @@ import (
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"testing" "testing"
"time" "time"
"codeberg.org/codeberg/pages/server/cache"
) )
func TestHandlerPerformance(t *testing.T) { func TestHandlerPerformance(t *testing.T) {
@ -16,6 +18,8 @@ func TestHandlerPerformance(t *testing.T) {
"", "",
[][]byte{[]byte("/.well-known/acme-challenge/")}, [][]byte{[]byte("/.well-known/acme-challenge/")},
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
) )
ctx := &fasthttp.RequestCtx{ ctx := &fasthttp.RequestCtx{