This commit is contained in:
6543 2021-12-05 15:53:46 +01:00
parent 76c867cfca
commit de4706bf58
No known key found for this signature in database
GPG key ID: C99B82E40B027BAE
6 changed files with 21 additions and 20 deletions

View file

@ -67,12 +67,17 @@ func Serve(ctx *cli.Context) error {
var canonicalDomainCache = cache.NewKeyValueCache() var canonicalDomainCache = cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains // dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = cache.NewKeyValueCache() var dnsLookupCache = cache.NewKeyValueCache()
// branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = cache.NewKeyValueCache()
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
var fileResponseCache = cache.NewKeyValueCache()
// Create handler based on settings // Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain), handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaRoot, rawInfoPage, giteaAPIToken, giteaRoot, rawInfoPage, giteaAPIToken,
BlacklistedPaths, allowedCorsDomains, BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache) dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
fastServer := server.SetupServer(handler) fastServer := server.SetupServer(handler)
httpServer := server.SetupHttpACMEChallengeServer(challengeCache) httpServer := server.SetupHttpACMEChallengeServer(challengeCache)

View file

@ -18,7 +18,7 @@ import (
func Handler(mainDomainSuffix, rawDomain []byte, func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaApiToken string, giteaRoot, rawInfoPage, giteaApiToken string,
blacklistedPaths, allowedCorsDomains [][]byte, blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
@ -85,7 +85,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} }
// Check if the branch exists, otherwise treat it as a file path // Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken) branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken, branchTimestampCache)
if branchTimestampResult == nil { if branchTimestampResult == nil {
// branch doesn't exist // branch doesn't exist
return false return false
@ -126,7 +126,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} }
// Try to request the file from the Gitea API // Try to request the file from the Gitea API
if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) { if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions, branchTimestampCache, fileResponseCache) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
} }
} }

View file

@ -20,6 +20,8 @@ func TestHandlerPerformance(t *testing.T) {
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(), cache.NewKeyValueCache(),
cache.NewKeyValueCache(), cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
) )
ctx := &fasthttp.RequestCtx{ ctx := &fasthttp.RequestCtx{

View file

@ -1,10 +0,0 @@
package upstream
import "github.com/OrlovEvgeny/go-mcache"
// branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = mcache.New()
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
var fileResponseCache = mcache.New()

View file

@ -5,6 +5,8 @@ import (
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"github.com/valyala/fastjson" "github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/cache"
) )
type branchTimestamp struct { type branchTimestamp struct {
@ -14,7 +16,7 @@ type branchTimestamp struct {
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist) // (or nil if the branch doesn't exist)
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp { func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil { if result == nil {
return nil return nil

View file

@ -10,10 +10,12 @@ import (
"strings" "strings"
"time" "time"
"codeberg.org/codeberg/pages/html"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
) )
// upstreamIndexPages lists pages that may be considered as index pages for directories. // upstreamIndexPages lists pages that may be considered as index pages for directories.
@ -39,7 +41,7 @@ var Client = fasthttp.Client{
} }
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options) (final bool) { func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger()
if options.ForbiddenMimeTypes == nil { if options.ForbiddenMimeTypes == nil {
@ -48,7 +50,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t
// Check if the branch exists and when it was modified // Check if the branch exists and when it was modified
if options.BranchTimestamp == (time.Time{}) { if options.BranchTimestamp == (time.Time{}) {
branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken) branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken, branchTimestampCache)
if branch == nil { if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -97,7 +99,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t
optionsForIndexPages.TryIndexPages = false optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.AppendTrailingSlash = true optionsForIndexPages.AppendTrailingSlash = true
for _, indexPage := range upstreamIndexPages { for _, indexPage := range upstreamIndexPages {
if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) { if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
exists: false, exists: false,
}, FileCacheTimeout) }, FileCacheTimeout)
@ -107,7 +109,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t
// compatibility fix for GitHub Pages (/example → /example.html) // compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.AppendTrailingSlash = false optionsForIndexPages.AppendTrailingSlash = false
optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html"
if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) { if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
exists: false, exists: false,
}, FileCacheTimeout) }, FileCacheTimeout)