Move gitea api calls in own "client" package (#78)

continue #75
close #16
- fix regression (from #34) _thanks to @crystal_
- create own gitea client package
- more logging
- add mock impl of CertDB

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: crystal <crystal@noreply.codeberg.org>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/78
Reviewed-by: crapStone <crapstone@noreply.codeberg.org>
This commit is contained in:
6543 2022-06-11 23:02:06 +02:00
parent 659932521c
commit 02bd942b04
19 changed files with 374 additions and 194 deletions

View file

@ -1,3 +1,5 @@
branches: main
pipeline: pipeline:
# use vendor to cache dependencies # use vendor to cache dependencies
vendor: vendor:

View file

@ -15,6 +15,9 @@ lint: tool-golangci tool-gofumpt
[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
golangci-lint run --timeout 5m golangci-lint run --timeout 5m
fmt: tool-gofumpt
gofumpt -w --extra .
tool-golangci: tool-golangci:
@hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \ go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \

View file

@ -61,7 +61,7 @@ func removeCert(ctx *cli.Context) error {
for _, domain := range domains { for _, domain := range domains {
fmt.Printf("Removing domain %s from the database...\n", domain) fmt.Printf("Removing domain %s from the database...\n", domain)
if err := keyDatabase.Delete([]byte(domain)); err != nil { if err := keyDatabase.Delete(domain); err != nil {
return err return err
} }
} }

View file

@ -18,6 +18,7 @@ import (
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/gitea"
) )
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error {
// TODO: make this an MRU cache with a size limit // TODO: make this an MRU cache with a size limit
fileResponseCache := cache.NewKeyValueCache() fileResponseCache := cache.NewKeyValueCache()
giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken)
// Create handler based on settings // Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain), handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaRoot, rawInfoPage, giteaAPIToken, giteaClient,
giteaRoot, rawInfoPage,
BlacklistedPaths, allowedCorsDomains, BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
@ -105,7 +109,8 @@ func Serve(ctx *cli.Context) error {
defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaRoot, giteaAPIToken, dnsProvider, giteaClient,
dnsProvider,
acmeUseRateLimits, acmeUseRateLimits,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB)) certDB))
@ -126,6 +131,7 @@ func Serve(ctx *cli.Context) error {
if enableHTTPServer { if enableHTTPServer {
go func() { go func() {
log.Info().Timestamp().Msg("Start listening on :80")
err := httpServer.ListenAndServe("[::]:80") err := httpServer.ListenAndServe("[::]:80")
if err != nil { if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@ -134,6 +140,7 @@ func Serve(ctx *cli.Context) error {
} }
// Start the web fastServer // Start the web fastServer
log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr())
err = fastServer.Serve(listener) err = fastServer.Serve(listener)
if err != nil { if err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer") log.Panic().Err(err).Msg("Couldn't start fastServer")

View file

@ -32,12 +32,14 @@ import (
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/database"
dnsutils "codeberg.org/codeberg/pages/server/dns" dnsutils "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/upstream"
) )
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix []byte, func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaAPIToken, dnsProvider string, giteaClient *gitea.Client,
dnsProvider string,
acmeUseRateLimits bool, acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB, certDB database.CertDB,
@ -81,7 +83,7 @@ func TLSConfig(mainDomainSuffix []byte,
sni = string(sniBytes) sni = string(sniBytes)
} else { } else {
_, _ = targetRepo, targetBranch _, _ = targetRepo, targetBranch
_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
if !valid { if !valid {
sniBytes = mainDomainSuffix sniBytes = mainDomainSuffix
sni = string(sniBytes) sni = string(sniBytes)
@ -193,7 +195,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
// parse certificate from database // parse certificate from database
res, err := certDB.Get(sni) res, err := certDB.Get(string(sni))
if err != nil { if err != nil {
panic(err) // TODO: no panic panic(err) // TODO: no panic
} }
@ -406,7 +408,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits // getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(mainDomainSuffix) mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
if err != nil { if err != nil {
return fmt.Errorf("cert database is not working") return fmt.Errorf("cert database is not working")
} }
@ -478,7 +480,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
if err != nil || !tlsCertificates[0].NotAfter.After(now) { if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := certDB.Delete(key) err := certDB.Delete(string(key))
if err != nil { if err != nil {
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
} else { } else {
@ -491,15 +493,15 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
// compact the database // compact the database
result, err := certDB.Compact() msg, err := certDB.Compact()
if err != nil { if err != nil {
log.Printf("[ERROR] Compacting key database failed: %s", err) log.Printf("[ERROR] Compacting key database failed: %s", err)
} else { } else {
log.Printf("[INFO] Compacted key database (%+v)", result) log.Printf("[INFO] Compacted key database (%s)", msg)
} }
// update main cert // update main cert
res, err := certDB.Get(mainDomainSuffix) res, err := certDB.Get(string(mainDomainSuffix))
if err != nil { if err != nil {
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix) log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
} else if res == nil { } else if res == nil {

View file

@ -0,0 +1,17 @@
package certificates
import (
"testing"
"codeberg.org/codeberg/pages/server/database"
"github.com/stretchr/testify/assert"
)
func TestMockCert(t *testing.T) {
db, err := database.NewTmpDB()
assert.NoError(t, err)
cert := mockCert("example.com", "some error msg", "codeberg.page", db)
if assert.NotEmpty(t, cert) {
assert.NotEmpty(t, cert.Certificate)
}
}

View file

@ -8,8 +8,8 @@ import (
type CertDB interface { type CertDB interface {
Close() error Close() error
Put(name string, cert *certificate.Resource) error Put(name string, cert *certificate.Resource) error
Get(name []byte) (*certificate.Resource, error) Get(name string) (*certificate.Resource, error)
Delete(key []byte) error Delete(key string) error
Compact() (pogreb.CompactionResult, error) Compact() (string, error)
Items() *pogreb.ItemIterator Items() *pogreb.ItemIterator
} }

55
server/database/mock.go Normal file
View file

@ -0,0 +1,55 @@
package database
import (
"fmt"
"time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/akrylysov/pogreb"
"github.com/go-acme/lego/v4/certificate"
)
var _ CertDB = tmpDB{}
type tmpDB struct {
intern *mcache.CacheDriver
ttl time.Duration
}
func (p tmpDB) Close() error {
_ = p.intern.Close()
return nil
}
func (p tmpDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Set(name, cert, p.ttl)
}
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
return nil, fmt.Errorf("cert for '%s' not found", name)
}
return cert.(*certificate.Resource), nil
}
func (p tmpDB) Delete(key string) error {
p.intern.Remove(key)
return nil
}
func (p tmpDB) Compact() (string, error) {
p.intern.Truncate()
return "Truncate done", nil
}
func (p tmpDB) Items() *pogreb.ItemIterator {
panic("ItemIterator not implemented for tmpDB")
}
func NewTmpDB() (CertDB, error) {
return &tmpDB{
intern: mcache.New(),
ttl: time.Minute,
}, nil
}

View file

@ -35,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Put([]byte(name), resGob.Bytes()) return p.intern.Put([]byte(name), resGob.Bytes())
} }
func (p aDB) Get(name []byte) (*certificate.Resource, error) { func (p aDB) Get(name string) (*certificate.Resource, error) {
cert := &certificate.Resource{} cert := &certificate.Resource{}
resBytes, err := p.intern.Get(name) resBytes, err := p.intern.Get([]byte(name))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -50,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) {
return cert, nil return cert, nil
} }
func (p aDB) Delete(key []byte) error { func (p aDB) Delete(key string) error {
return p.intern.Delete(key) return p.intern.Delete([]byte(key))
} }
func (p aDB) Compact() (pogreb.CompactionResult, error) { func (p aDB) Compact() (string, error) {
return p.intern.Compact() result, err := p.intern.Compact()
if err != nil {
return "", err
}
return fmt.Sprintf("%+v", result), nil
} }
func (p aDB) Items() *pogreb.ItemIterator { func (p aDB) Items() *pogreb.ItemIterator {

119
server/gitea/client.go Normal file
View file

@ -0,0 +1,119 @@
package gitea
import (
"errors"
"fmt"
"net/url"
"path"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
)
const giteaAPIRepos = "/api/v1/repos/"
var ErrorNotFound = errors.New("not found")
type Client struct {
giteaRoot string
giteaAPIToken string
fastClient *fasthttp.Client
infoTimeout time.Duration
contentTimeout time.Duration
}
type FileResponse struct {
Exists bool
MimeType string
Body []byte
}
func joinURL(giteaRoot string, paths ...string) string { return giteaRoot + path.Join(paths...) }
func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
func NewClient(giteaRoot, giteaAPIToken string) *Client {
return &Client{
giteaRoot: giteaRoot,
giteaAPIToken: giteaAPIToken,
infoTimeout: 5 * time.Second,
contentTimeout: 10 * time.Second,
fastClient: getFastHTTPClient(),
}
}
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
res, err := client.do(client.contentTimeout, url)
if err != nil {
return nil, err
}
switch res.StatusCode() {
case fasthttp.StatusOK:
return res.Body(), nil
case fasthttp.StatusNotFound:
return nil, ErrorNotFound
default:
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
}
func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
res, err := client.do(client.contentTimeout, url)
if err != nil {
return nil, err
}
// resp.SetBodyStream(&strings.Reader{}, -1)
if err != nil {
return nil, err
}
switch res.StatusCode() {
case fasthttp.StatusOK:
return res, nil
case fasthttp.StatusNotFound:
return nil, ErrorNotFound
default:
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
}
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
res, err := client.do(client.infoTimeout, url)
if err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
res, err := client.do(client.infoTimeout, url)
if err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}
func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(url)
req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
res := fasthttp.AcquireResponse()
err := client.fastClient.DoTimeout(req, res, timeout)
return res, err
}

View file

@ -0,0 +1,23 @@
package gitea
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestJoinURL(t *testing.T) {
url := joinURL("")
assert.EqualValues(t, "", url)
url = joinURL("", "", "")
assert.EqualValues(t, "", url)
url = joinURL("http://wwow.url.com", "a", "b/c/", "d")
// assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", url)
assert.EqualValues(t, "http://wwow.url.coma/b/c/d", url)
url = joinURL("h:://wrong", "acdc")
// assert.EqualValues(t, "h:://wrong/acdc", url)
assert.EqualValues(t, "h:://wrongacdc", url)
}

15
server/gitea/fasthttp.go Normal file
View file

@ -0,0 +1,15 @@
package gitea
import (
"time"
"github.com/valyala/fasthttp"
)
func getFastHTTPClient() *fasthttp.Client {
return &fasthttp.Client{
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}

View file

@ -4,19 +4,22 @@ import (
"bytes" "bytes"
"strings" "strings"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/dns" "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils" "codeberg.org/codeberg/pages/server/utils"
) )
// Handler handles a single HTTP request to the web server. // Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain []byte, func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string, giteaClient *gitea.Client,
giteaRoot, rawInfoPage string,
blacklistedPaths, allowedCorsDomains [][]byte, blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) { ) func(ctx *fasthttp.RequestCtx) {
@ -74,21 +77,21 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Prepare request information to Gitea // Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string var targetOwner, targetRepo, targetBranch, targetPath string
targetOptions := &upstream.Options{ targetOptions := &upstream.Options{
ForbiddenMimeTypes: map[string]struct{}{}, TryIndexPages: true,
TryIndexPages: true,
} }
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL. // also disallow search indexing and add a Link header to the canonical URL.
tryBranch := func(repo, branch string, path []string, canonicalLink string) bool { tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
if repo == "" { if repo == "" {
log.Debug().Msg("tryBranch: repo == ''")
return false return false
} }
// Check if the branch exists, otherwise treat it as a file path // Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache) branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
if branchTimestampResult == nil { if branchTimestampResult == nil {
// branch doesn't exist log.Debug().Msg("tryBranch: branch doesn't exist")
return false return false
} }
@ -108,6 +111,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
) )
} }
log.Debug().Msg("tryBranch: true")
return true return true
} }
@ -117,7 +121,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("raw domain") log.Debug().Msg("raw domain")
targetOptions.TryIndexPages = false targetOptions.TryIndexPages = false
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{} if targetOptions.ForbiddenMimeTypes == nil {
targetOptions.ForbiddenMimeTypes = make(map[string]bool)
}
targetOptions.ForbiddenMimeTypes["text/html"] = true
targetOptions.DefaultMimeType = "text/plain; charset=utf-8" targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@ -132,13 +139,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// raw.codeberg.org/example/myrepo/@main/index.html // raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
log.Debug().Msg("raw domain preparations, now trying with specified branch") log.Debug().Msg("raw domain preparations, now trying with specified branch")
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], if tryBranch(log,
targetRepo, pathElements[2][1:], pathElements[3:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) { ) {
log.Debug().Msg("tryBranch, now trying upstream 1") log.Debug().Msg("tryBranch, now trying upstream 1")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
return return
} }
@ -148,13 +155,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} }
log.Debug().Msg("raw domain preparations, now trying with default branch") log.Debug().Msg("raw domain preparations, now trying with default branch")
tryBranch(targetRepo, "", pathElements[2:], tryBranch(log,
targetRepo, "", pathElements[2:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) )
log.Debug().Msg("tryBranch, now trying upstream 2") log.Debug().Msg("tryBranch, now trying upstream 2")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
return return
@ -183,13 +190,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} }
log.Debug().Msg("main domain preparations, now trying with specified repo & branch") log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:], if tryBranch(log,
pathElements[0], pathElements[1][1:], pathElements[2:],
"/"+pathElements[0]+"/%p", "/"+pathElements[0]+"/%p",
) { ) {
log.Debug().Msg("tryBranch, now trying upstream 3") log.Debug().Msg("tryBranch, now trying upstream 3")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else { } else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -201,11 +208,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// example.codeberg.page/@main/index.html // example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") { if strings.HasPrefix(pathElements[0], "@") {
log.Debug().Msg("main domain preparations, now trying with specified branch") log.Debug().Msg("main domain preparations, now trying with specified branch")
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { if tryBranch(log,
"pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Debug().Msg("tryBranch, now trying upstream 4") log.Debug().Msg("tryBranch, now trying upstream 4")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else { } else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -217,11 +224,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// example.codeberg.page/myrepo/index.html // example.codeberg.page/myrepo/index.html
// example.codeberg.page/pages/... is not allowed here. // example.codeberg.page/pages/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo") log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { if pathElements[0] != "pages" && tryBranch(log,
pathElements[0], "pages", pathElements[1:], "") {
log.Debug().Msg("tryBranch, now trying upstream 5") log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
return return
} }
@ -229,11 +236,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Try to use the "pages" repo on its default branch // Try to use the "pages" repo on its default branch
// example.codeberg.page/index.html // example.codeberg.page/index.html
log.Debug().Msg("main domain preparations, now trying with default repo/branch") log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch("pages", "", pathElements, "") { if tryBranch(log,
"pages", "", pathElements, "") {
log.Debug().Msg("tryBranch, now trying upstream 6") log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
return return
} }
@ -261,8 +268,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Try to use the given repo on the given branch or the default branch // Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS") log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { if tryBranch(log,
canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
if !valid { if !valid {
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return return
@ -279,9 +287,8 @@ func Handler(mainDomainSuffix, rawDomain []byte,
} }
log.Debug().Msg("tryBranch, now trying upstream 7") log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, mainDomainSuffix, trimmedHost, tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath, targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache) canonicalDomainCache, branchTimestampCache, fileResponseCache)
return return
} }

View file

@ -8,15 +8,16 @@ import (
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
) )
func TestHandlerPerformance(t *testing.T) { func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
giteaClient := gitea.NewClient(giteaRoot, "")
testHandler := Handler( testHandler := Handler(
[]byte("codeberg.page"), []byte("codeberg.page"), []byte("raw.codeberg.org"),
[]byte("raw.codeberg.org"), giteaClient,
"https://codeberg.org", giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
"https://docs.codeberg.org/pages/raw-content/",
"",
[][]byte{[]byte("/.well-known/acme-challenge/")}, [][]byte{[]byte("/.well-known/acme-challenge/")},
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(), cache.NewKeyValueCache(),

View file

@ -8,22 +8,22 @@ import (
"codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/upstream"
) )
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *fasthttp.RequestCtx, func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost []byte, mainDomainSuffix, trimmedHost []byte,
targetOptions *upstream.Options, targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath, targetOwner, targetRepo, targetBranch, targetPath string,
giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) { ) {
// check if a canonical domain exists on a request on MainDomain // check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI()) canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" { if targetRepo != "pages" {
@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
targetOptions.TargetPath = targetPath targetOptions.TargetPath = targetPath
// Try to request the file from the Gitea API // Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
} }
} }

View file

@ -4,14 +4,11 @@ import (
"strings" "strings"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
) )
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache)
}
func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var ( var (
domains []string domains []string
valid bool valid bool
@ -25,7 +22,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
} }
} }
} else { } else {
body, err := giteaRawContent(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig) body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
if err == nil { if err == nil {
for _, domain := range strings.Split(string(body), "\n") { for _, domain := range strings.Split(string(body), "\n") {
domain = strings.ToLower(domain) domain = strings.ToLower(domain)

View file

@ -1,67 +0,0 @@
package upstream
import (
"fmt"
"net/url"
"path"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
)
const giteaAPIRepos = "/api/v1/repos/"
// TODOs:
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
return nil, err
}
if res.StatusCode() != fasthttp.StatusOK {
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return res.Body(), nil
}
func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}

View file

@ -1,9 +1,14 @@
package upstream package upstream
import ( import (
"mime"
"path"
"strconv"
"strings"
"time" "time"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
) )
type branchTimestamp struct { type branchTimestamp struct {
@ -13,7 +18,7 @@ type branchTimestamp struct {
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist) // (or nil if the branch doesn't exist)
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil { if result == nil {
return nil return nil
@ -25,7 +30,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
} }
if len(branch) == 0 { if len(branch) == 0 {
// Get default branch // Get default branch
defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken) defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
if err != nil { if err != nil {
_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
return nil return nil
@ -33,7 +38,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
result.Branch = defaultBranch result.Branch = defaultBranch
} }
timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken) timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
if err != nil { if err != nil {
return nil return nil
} }
@ -42,8 +47,26 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br
return result return result
} }
type fileResponse struct { func (o *Options) getMimeTypeByExtension() string {
exists bool if o.ForbiddenMimeTypes == nil {
mimeType string o.ForbiddenMimeTypes = make(map[string]bool)
body []byte }
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
return mimeType
}
func (o *Options) generateUri() string {
return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
}
func (o *Options) timestamp() string {
return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
} }

View file

@ -2,11 +2,9 @@ package upstream
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io" "io"
"mime"
"path"
"strconv"
"strings" "strings"
"time" "time"
@ -15,6 +13,7 @@ import (
"codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
) )
// upstreamIndexPages lists pages that may be considered as index pages for directories. // upstreamIndexPages lists pages that may be considered as index pages for directories.
@ -30,7 +29,7 @@ type Options struct {
TargetPath, TargetPath,
DefaultMimeType string DefaultMimeType string
ForbiddenMimeTypes map[string]struct{} ForbiddenMimeTypes map[string]bool
TryIndexPages bool TryIndexPages bool
BranchTimestamp time.Time BranchTimestamp time.Time
// internal // internal
@ -38,26 +37,13 @@ type Options struct {
redirectIfExists string redirectIfExists string
} }
func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.ForbiddenMimeTypes == nil {
o.ForbiddenMimeTypes = map[string]struct{}{}
}
// Check if the branch exists and when it was modified // Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() { if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
if branch == nil { if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@ -82,25 +68,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
log.Debug().Msg("preparations") log.Debug().Msg("preparations")
// Make a GET request to the upstream URL // Make a GET request to the upstream URL
uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) uri := o.generateUri()
var req *fasthttp.Request
var res *fasthttp.Response var res *fasthttp.Response
var cachedResponse fileResponse var cachedResponse gitea.FileResponse
var err error var err error
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
cachedResponse = cachedValue.(fileResponse) cachedResponse = cachedValue.(gitea.FileResponse)
} else { } else {
req = fasthttp.AcquireRequest() res, err = giteaClient.ServeRawContent(uri)
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = getFastHTTPClient(10*time.Second).Do(req, res)
} }
log.Debug().Msg("acquisition") log.Debug().Msg("acquisition")
// Handle errors // Handle errors
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
if o.TryIndexPages { if o.TryIndexPages {
// copy the o struct & try if an index page exists // copy the o struct & try if an index page exists
optionsForIndexPages := *o optionsForIndexPages := *o
@ -108,9 +88,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
optionsForIndexPages.appendTrailingSlash = true optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages { for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
exists: false, Exists: false,
}, fileCacheTimeout) }, fileCacheTimeout)
return true return true
} }
@ -119,9 +99,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
exists: false, Exists: false,
}, fileCacheTimeout) }, fileCacheTimeout)
return true return true
} }
@ -129,14 +109,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
ctx.Response.SetStatusCode(fasthttp.StatusNotFound) ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if res != nil { if res != nil {
// Update cache if the request is fresh // Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
exists: false, Exists: false,
}, fileCacheTimeout) }, fileCacheTimeout)
} }
return false return false
} }
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true return true
} }
@ -158,15 +138,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
log.Debug().Msg("error handling") log.Debug().Msg("error handling")
// Set the MIME type // Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) mimeType := o.getMimeTypeByExtension()
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType) ctx.Response.Header.SetContentType(mimeType)
// Everything's okay so far // Everything's okay so far
@ -185,20 +157,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
} }
} else { } else {
_, err = ctx.Write(cachedResponse.body) _, err = ctx.Write(cachedResponse.Body)
} }
if err != nil { if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true return true
} }
log.Debug().Msg("response") log.Debug().Msg("response")
if res != nil && ctx.Err() == nil { if res != nil && ctx.Err() == nil {
cachedResponse.exists = true cachedResponse.Exists = true
cachedResponse.mimeType = mimeType cachedResponse.MimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes() cachedResponse.Body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
} }
return true return true