From 02bd942b04033ab3d6c11b279b42440bac3c5739 Mon Sep 17 00:00:00 2001 From: 6543 <6543@noreply.codeberg.org> Date: Sat, 11 Jun 2022 23:02:06 +0200 Subject: [PATCH] Move gitea api calls in own "client" package (#78) continue #75 close #16 - fix regression (from #34) _thanks to @crystal_ - create own gitea client package - more logging - add mock impl of CertDB Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: crystal Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/78 Reviewed-by: crapStone --- .woodpecker.yml | 2 + Justfile | 3 + cmd/certs.go | 2 +- cmd/main.go | 11 ++- server/certificates/certificates.go | 18 +++-- server/certificates/mock_test.go | 17 ++++ server/database/interface.go | 6 +- server/database/mock.go | 55 +++++++++++++ server/database/setup.go | 16 ++-- server/gitea/client.go | 119 ++++++++++++++++++++++++++++ server/gitea/client_test.go | 23 ++++++ server/gitea/fasthttp.go | 15 ++++ server/handler.go | 65 ++++++++------- server/handler_test.go | 11 +-- server/try.go | 10 +-- server/upstream/domains.go | 9 +-- server/upstream/gitea.go | 67 ---------------- server/upstream/helper.go | 37 +++++++-- server/upstream/upstream.go | 82 +++++++------------ 19 files changed, 374 insertions(+), 194 deletions(-) create mode 100644 server/certificates/mock_test.go create mode 100644 server/database/mock.go create mode 100644 server/gitea/client.go create mode 100644 server/gitea/client_test.go create mode 100644 server/gitea/fasthttp.go delete mode 100644 server/upstream/gitea.go diff --git a/.woodpecker.yml b/.woodpecker.yml index 0440309..271aaca 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,3 +1,5 @@ +branches: main + pipeline: # use vendor to cache dependencies vendor: diff --git a/Justfile b/Justfile index 2c64574..bab0a1e 100644 --- a/Justfile +++ b/Justfile @@ -15,6 +15,9 @@ lint: tool-golangci tool-gofumpt [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ golangci-lint run --timeout 5m +fmt: tool-gofumpt + gofumpt -w --extra . + tool-golangci: @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \ diff --git a/cmd/certs.go b/cmd/certs.go index 83f2ac5..d93fe13 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -61,7 +61,7 @@ func removeCert(ctx *cli.Context) error { for _, domain := range domains { fmt.Printf("Removing domain %s from the database...\n", domain) - if err := keyDatabase.Delete([]byte(domain)); err != nil { + if err := keyDatabase.Delete(domain); err != nil { return err } } diff --git a/cmd/main.go b/cmd/main.go index 6836cee..257b724 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" + "codeberg.org/codeberg/pages/server/gitea" ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. @@ -81,9 +82,12 @@ func Serve(ctx *cli.Context) error { // TODO: make this an MRU cache with a size limit fileResponseCache := cache.NewKeyValueCache() + giteaClient := gitea.NewClient(giteaRoot, giteaAPIToken) + // Create handler based on settings handler := server.Handler(mainDomainSuffix, []byte(rawDomain), - giteaRoot, rawInfoPage, giteaAPIToken, + giteaClient, + giteaRoot, rawInfoPage, BlacklistedPaths, allowedCorsDomains, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) @@ -105,7 +109,8 @@ func Serve(ctx *cli.Context) error { defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, - giteaRoot, giteaAPIToken, dnsProvider, + giteaClient, + dnsProvider, acmeUseRateLimits, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB)) @@ -126,6 +131,7 @@ func Serve(ctx *cli.Context) error { if enableHTTPServer { go func() { + log.Info().Timestamp().Msg("Start listening on :80") err := httpServer.ListenAndServe("[::]:80") if err != nil { log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") @@ -134,6 +140,7 @@ func Serve(ctx *cli.Context) error { } // Start the web fastServer + log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr()) err = fastServer.Serve(listener) if err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index a13215c..2684dfa 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -32,12 +32,14 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/database" dnsutils "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. func TLSConfig(mainDomainSuffix []byte, - giteaRoot, giteaAPIToken, dnsProvider string, + giteaClient *gitea.Client, + dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, certDB database.CertDB, @@ -81,7 +83,7 @@ func TLSConfig(mainDomainSuffix []byte, sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache) if !valid { sniBytes = mainDomainSuffix sni = string(sniBytes) @@ -193,7 +195,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { // parse certificate from database - res, err := certDB.Get(sni) + res, err := certDB.Get(string(sni)) if err != nil { panic(err) // TODO: no panic } @@ -406,7 +408,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits - mainCertBytes, err := certDB.Get(mainDomainSuffix) + mainCertBytes, err := certDB.Get(string(mainDomainSuffix)) if err != nil { return fmt.Errorf("cert database is not working") } @@ -478,7 +480,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) if err != nil || !tlsCertificates[0].NotAfter.After(now) { - err := certDB.Delete(key) + err := certDB.Delete(string(key)) if err != nil { log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) } else { @@ -491,15 +493,15 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) // compact the database - result, err := certDB.Compact() + msg, err := certDB.Compact() if err != nil { log.Printf("[ERROR] Compacting key database failed: %s", err) } else { - log.Printf("[INFO] Compacted key database (%+v)", result) + log.Printf("[INFO] Compacted key database (%s)", msg) } // update main cert - res, err := certDB.Get(mainDomainSuffix) + res, err := certDB.Get(string(mainDomainSuffix)) if err != nil { log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix) } else if res == nil { diff --git a/server/certificates/mock_test.go b/server/certificates/mock_test.go new file mode 100644 index 0000000..1cbd1f6 --- /dev/null +++ b/server/certificates/mock_test.go @@ -0,0 +1,17 @@ +package certificates + +import ( + "testing" + + "codeberg.org/codeberg/pages/server/database" + "github.com/stretchr/testify/assert" +) + +func TestMockCert(t *testing.T) { + db, err := database.NewTmpDB() + assert.NoError(t, err) + cert := mockCert("example.com", "some error msg", "codeberg.page", db) + if assert.NotEmpty(t, cert) { + assert.NotEmpty(t, cert.Certificate) + } +} diff --git a/server/database/interface.go b/server/database/interface.go index 01b9872..3ba3efc 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -8,8 +8,8 @@ import ( type CertDB interface { Close() error Put(name string, cert *certificate.Resource) error - Get(name []byte) (*certificate.Resource, error) - Delete(key []byte) error - Compact() (pogreb.CompactionResult, error) + Get(name string) (*certificate.Resource, error) + Delete(key string) error + Compact() (string, error) Items() *pogreb.ItemIterator } diff --git a/server/database/mock.go b/server/database/mock.go new file mode 100644 index 0000000..e6c1b5a --- /dev/null +++ b/server/database/mock.go @@ -0,0 +1,55 @@ +package database + +import ( + "fmt" + "time" + + "github.com/OrlovEvgeny/go-mcache" + "github.com/akrylysov/pogreb" + "github.com/go-acme/lego/v4/certificate" +) + +var _ CertDB = tmpDB{} + +type tmpDB struct { + intern *mcache.CacheDriver + ttl time.Duration +} + +func (p tmpDB) Close() error { + _ = p.intern.Close() + return nil +} + +func (p tmpDB) Put(name string, cert *certificate.Resource) error { + return p.intern.Set(name, cert, p.ttl) +} + +func (p tmpDB) Get(name string) (*certificate.Resource, error) { + cert, has := p.intern.Get(name) + if !has { + return nil, fmt.Errorf("cert for '%s' not found", name) + } + return cert.(*certificate.Resource), nil +} + +func (p tmpDB) Delete(key string) error { + p.intern.Remove(key) + return nil +} + +func (p tmpDB) Compact() (string, error) { + p.intern.Truncate() + return "Truncate done", nil +} + +func (p tmpDB) Items() *pogreb.ItemIterator { + panic("ItemIterator not implemented for tmpDB") +} + +func NewTmpDB() (CertDB, error) { + return &tmpDB{ + intern: mcache.New(), + ttl: time.Minute, + }, nil +} diff --git a/server/database/setup.go b/server/database/setup.go index e48b661..bbcf431 100644 --- a/server/database/setup.go +++ b/server/database/setup.go @@ -35,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error { return p.intern.Put([]byte(name), resGob.Bytes()) } -func (p aDB) Get(name []byte) (*certificate.Resource, error) { +func (p aDB) Get(name string) (*certificate.Resource, error) { cert := &certificate.Resource{} - resBytes, err := p.intern.Get(name) + resBytes, err := p.intern.Get([]byte(name)) if err != nil { return nil, err } @@ -50,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) { return cert, nil } -func (p aDB) Delete(key []byte) error { - return p.intern.Delete(key) +func (p aDB) Delete(key string) error { + return p.intern.Delete([]byte(key)) } -func (p aDB) Compact() (pogreb.CompactionResult, error) { - return p.intern.Compact() +func (p aDB) Compact() (string, error) { + result, err := p.intern.Compact() + if err != nil { + return "", err + } + return fmt.Sprintf("%+v", result), nil } func (p aDB) Items() *pogreb.ItemIterator { diff --git a/server/gitea/client.go b/server/gitea/client.go new file mode 100644 index 0000000..d4eb980 --- /dev/null +++ b/server/gitea/client.go @@ -0,0 +1,119 @@ +package gitea + +import ( + "errors" + "fmt" + "net/url" + "path" + "time" + + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" +) + +const giteaAPIRepos = "/api/v1/repos/" + +var ErrorNotFound = errors.New("not found") + +type Client struct { + giteaRoot string + giteaAPIToken string + fastClient *fasthttp.Client + infoTimeout time.Duration + contentTimeout time.Duration +} + +type FileResponse struct { + Exists bool + MimeType string + Body []byte +} + +func joinURL(giteaRoot string, paths ...string) string { return giteaRoot + path.Join(paths...) } + +func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } + +func NewClient(giteaRoot, giteaAPIToken string) *Client { + return &Client{ + giteaRoot: giteaRoot, + giteaAPIToken: giteaAPIToken, + infoTimeout: 5 * time.Second, + contentTimeout: 10 * time.Second, + fastClient: getFastHTTPClient(), + } +} + +func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { + url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) + res, err := client.do(client.contentTimeout, url) + if err != nil { + return nil, err + } + + switch res.StatusCode() { + case fasthttp.StatusOK: + return res.Body(), nil + case fasthttp.StatusNotFound: + return nil, ErrorNotFound + default: + return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } +} + +func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) { + url := joinURL(client.giteaRoot, giteaAPIRepos, uri) + res, err := client.do(client.contentTimeout, url) + if err != nil { + return nil, err + } + // resp.SetBodyStream(&strings.Reader{}, -1) + + if err != nil { + return nil, err + } + + switch res.StatusCode() { + case fasthttp.StatusOK: + return res, nil + case fasthttp.StatusNotFound: + return nil, ErrorNotFound + default: + return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } +} + +func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) { + url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName) + res, err := client.do(client.infoTimeout, url) + if err != nil { + return time.Time{}, err + } + if res.StatusCode() != fasthttp.StatusOK { + return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) +} + +func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { + url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName) + res, err := client.do(client.infoTimeout, url) + if err != nil { + return "", err + } + if res.StatusCode() != fasthttp.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return fastjson.GetString(res.Body(), "default_branch"), nil +} + +func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) { + req := fasthttp.AcquireRequest() + + req.SetRequestURI(url) + req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken) + res := fasthttp.AcquireResponse() + + err := client.fastClient.DoTimeout(req, res, timeout) + + return res, err +} diff --git a/server/gitea/client_test.go b/server/gitea/client_test.go new file mode 100644 index 0000000..bae9d4e --- /dev/null +++ b/server/gitea/client_test.go @@ -0,0 +1,23 @@ +package gitea + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJoinURL(t *testing.T) { + url := joinURL("") + assert.EqualValues(t, "", url) + + url = joinURL("", "", "") + assert.EqualValues(t, "", url) + + url = joinURL("http://wwow.url.com", "a", "b/c/", "d") + // assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", url) + assert.EqualValues(t, "http://wwow.url.coma/b/c/d", url) + + url = joinURL("h:://wrong", "acdc") + // assert.EqualValues(t, "h:://wrong/acdc", url) + assert.EqualValues(t, "h:://wrongacdc", url) +} diff --git a/server/gitea/fasthttp.go b/server/gitea/fasthttp.go new file mode 100644 index 0000000..4ff0f4a --- /dev/null +++ b/server/gitea/fasthttp.go @@ -0,0 +1,15 @@ +package gitea + +import ( + "time" + + "github.com/valyala/fasthttp" +) + +func getFastHTTPClient() *fasthttp.Client { + return &fasthttp.Client{ + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! + } +} diff --git a/server/handler.go b/server/handler.go index 0dc90ce..bda7bd0 100644 --- a/server/handler.go +++ b/server/handler.go @@ -4,19 +4,22 @@ import ( "bytes" "strings" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/utils" ) // Handler handles a single HTTP request to the web server. func Handler(mainDomainSuffix, rawDomain []byte, - giteaRoot, rawInfoPage, giteaAPIToken string, + giteaClient *gitea.Client, + giteaRoot, rawInfoPage string, blacklistedPaths, allowedCorsDomains [][]byte, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, ) func(ctx *fasthttp.RequestCtx) { @@ -74,21 +77,21 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Prepare request information to Gitea var targetOwner, targetRepo, targetBranch, targetPath string targetOptions := &upstream.Options{ - ForbiddenMimeTypes: map[string]struct{}{}, - TryIndexPages: true, + TryIndexPages: true, } // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // also disallow search indexing and add a Link header to the canonical URL. - tryBranch := func(repo, branch string, path []string, canonicalLink string) bool { + tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool { if repo == "" { + log.Debug().Msg("tryBranch: repo == ''") return false } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache) + branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache) if branchTimestampResult == nil { - // branch doesn't exist + log.Debug().Msg("tryBranch: branch doesn't exist") return false } @@ -108,6 +111,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, ) } + log.Debug().Msg("tryBranch: true") return true } @@ -117,7 +121,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("raw domain") targetOptions.TryIndexPages = false - targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{} + if targetOptions.ForbiddenMimeTypes == nil { + targetOptions.ForbiddenMimeTypes = make(map[string]bool) + } + targetOptions.ForbiddenMimeTypes["text/html"] = true targetOptions.DefaultMimeType = "text/plain; charset=utf-8" pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") @@ -132,13 +139,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, // raw.codeberg.org/example/myrepo/@main/index.html if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { log.Debug().Msg("raw domain preparations, now trying with specified branch") - if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], + if tryBranch(log, + targetRepo, pathElements[2][1:], pathElements[3:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { log.Debug().Msg("tryBranch, now trying upstream 1") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -148,13 +155,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, } log.Debug().Msg("raw domain preparations, now trying with default branch") - tryBranch(targetRepo, "", pathElements[2:], + tryBranch(log, + targetRepo, "", pathElements[2:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) log.Debug().Msg("tryBranch, now trying upstream 2") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return @@ -183,13 +190,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, } log.Debug().Msg("main domain preparations, now trying with specified repo & branch") - if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:], + if tryBranch(log, + pathElements[0], pathElements[1][1:], pathElements[2:], "/"+pathElements[0]+"/%p", ) { log.Debug().Msg("tryBranch, now trying upstream 3") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -201,11 +208,11 @@ func Handler(mainDomainSuffix, rawDomain []byte, // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { log.Debug().Msg("main domain preparations, now trying with specified branch") - if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { + if tryBranch(log, + "pages", pathElements[0][1:], pathElements[1:], "/%p") { log.Debug().Msg("tryBranch, now trying upstream 4") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -217,11 +224,11 @@ func Handler(mainDomainSuffix, rawDomain []byte, // example.codeberg.page/myrepo/index.html // example.codeberg.page/pages/... is not allowed here. log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { + if pathElements[0] != "pages" && tryBranch(log, + pathElements[0], "pages", pathElements[1:], "") { log.Debug().Msg("tryBranch, now trying upstream 5") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -229,11 +236,11 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Try to use the "pages" repo on its default branch // example.codeberg.page/index.html log.Debug().Msg("main domain preparations, now trying with default repo/branch") - if tryBranch("pages", "", pathElements, "") { + if tryBranch(log, + "pages", "", pathElements, "") { log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -261,8 +268,9 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") - if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + if tryBranch(log, + targetRepo, targetBranch, pathElements, canonicalLink) { + canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return @@ -279,9 +287,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, } log.Debug().Msg("tryBranch, now trying upstream 7") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } diff --git a/server/handler_test.go b/server/handler_test.go index 3b4d21a..73002a2 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -8,15 +8,16 @@ import ( "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) func TestHandlerPerformance(t *testing.T) { + giteaRoot := "https://codeberg.org" + giteaClient := gitea.NewClient(giteaRoot, "") testHandler := Handler( - []byte("codeberg.page"), - []byte("raw.codeberg.org"), - "https://codeberg.org", - "https://docs.codeberg.org/pages/raw-content/", - "", + []byte("codeberg.page"), []byte("raw.codeberg.org"), + giteaClient, + giteaRoot, "https://docs.codeberg.org/pages/raw-content/", [][]byte{[]byte("/.well-known/acme-challenge/")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, cache.NewKeyValueCache(), diff --git a/server/try.go b/server/try.go index 4eda5b2..254d3ec 100644 --- a/server/try.go +++ b/server/try.go @@ -8,22 +8,22 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *fasthttp.RequestCtx, +func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost []byte, targetOptions *upstream.Options, - targetOwner, targetRepo, targetBranch, targetPath, + targetOwner, targetRepo, targetBranch, targetPath string, - giteaRoot, giteaAPIToken string, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { @@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, targetOptions.TargetPath = targetPath // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 8669d08..553c148 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -4,14 +4,11 @@ import ( "strings" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { - return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache) -} - -func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { +func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( domains []string valid bool @@ -25,7 +22,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m } } } else { - body, err := giteaRawContent(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig) + body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig) if err == nil { for _, domain := range strings.Split(string(body), "\n") { domain = strings.ToLower(domain) diff --git a/server/upstream/gitea.go b/server/upstream/gitea.go deleted file mode 100644 index eeeb0a6..0000000 --- a/server/upstream/gitea.go +++ /dev/null @@ -1,67 +0,0 @@ -package upstream - -import ( - "fmt" - "net/url" - "path" - "time" - - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" -) - -const giteaAPIRepos = "/api/v1/repos/" - -// TODOs: -// * own client to store token & giteaRoot -// * handle 404 -> page will show 500 atm - -func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil { - return nil, err - } - if res.StatusCode() != fasthttp.StatusOK { - return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return res.Body(), nil -} - -func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) { - client := getFastHTTPClient(5 * time.Second) - - req := fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := client.Do(req, res); err != nil { - return time.Time{}, err - } - if res.StatusCode() != fasthttp.StatusOK { - return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) -} - -func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) { - client := getFastHTTPClient(5 * time.Second) - - req := fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res := fasthttp.AcquireResponse() - - if err := client.Do(req, res); err != nil { - return "", err - } - if res.StatusCode() != fasthttp.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - return fastjson.GetString(res.Body(), "default_branch"), nil -} diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 3b51479..5bbe833 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -1,9 +1,14 @@ package upstream import ( + "mime" + "path" + "strconv" + "strings" "time" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) type branchTimestamp struct { @@ -13,7 +18,7 @@ type branchTimestamp struct { // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { +func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { return nil @@ -25,7 +30,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br } if len(branch) == 0 { // Get default branch - defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken) + defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) if err != nil { _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil @@ -33,7 +38,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br result.Branch = defaultBranch } - timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken) + timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch) if err != nil { return nil } @@ -42,8 +47,26 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, br return result } -type fileResponse struct { - exists bool - mimeType string - body []byte +func (o *Options) getMimeTypeByExtension() string { + if o.ForbiddenMimeTypes == nil { + o.ForbiddenMimeTypes = make(map[string]bool) + } + mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { + if o.DefaultMimeType != "" { + mimeType = o.DefaultMimeType + } else { + mimeType = "application/octet-stream" + } + } + return mimeType +} + +func (o *Options) generateUri() string { + return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) +} + +func (o *Options) timestamp() string { + return strconv.FormatInt(o.BranchTimestamp.Unix(), 10) } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 33c80e9..edf4f3f 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -2,11 +2,9 @@ package upstream import ( "bytes" + "errors" "fmt" "io" - "mime" - "path" - "strconv" "strings" "time" @@ -15,6 +13,7 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/gitea" ) // upstreamIndexPages lists pages that may be considered as index pages for directories. @@ -30,7 +29,7 @@ type Options struct { TargetPath, DefaultMimeType string - ForbiddenMimeTypes map[string]struct{} + ForbiddenMimeTypes map[string]bool TryIndexPages bool BranchTimestamp time.Time // internal @@ -38,26 +37,13 @@ type Options struct { redirectIfExists string } -func getFastHTTPClient(timeout time.Duration) *fasthttp.Client { - return &fasthttp.Client{ - ReadTimeout: timeout, - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! - } -} - // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { +func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() - if o.ForbiddenMimeTypes == nil { - o.ForbiddenMimeTypes = map[string]struct{}{} - } - // Check if the branch exists and when it was modified if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) + branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache) if branch == nil { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -82,25 +68,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st log.Debug().Msg("preparations") // Make a GET request to the upstream URL - uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) - var req *fasthttp.Request + uri := o.generateUri() var res *fasthttp.Response - var cachedResponse fileResponse + var cachedResponse gitea.FileResponse var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { - cachedResponse = cachedValue.(fileResponse) + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { + cachedResponse = cachedValue.(gitea.FileResponse) } else { - req = fasthttp.AcquireRequest() - req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri)) - req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) - res = fasthttp.AcquireResponse() - res.SetBodyStream(&strings.Reader{}, -1) - err = getFastHTTPClient(10*time.Second).Do(req, res) + res, err = giteaClient.ServeRawContent(uri) } log.Debug().Msg("acquisition") // Handle errors - if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { + if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) { if o.TryIndexPages { // copy the o struct & try if an index page exists optionsForIndexPages := *o @@ -108,9 +88,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) return true } @@ -119,9 +99,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) return true } @@ -129,14 +109,14 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st ctx.Response.SetStatusCode(fasthttp.StatusNotFound) if res != nil { // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + Exists: false, }, fileCacheTimeout) } return false } if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) + fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode()) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } @@ -158,15 +138,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st log.Debug().Msg("error handling") // Set the MIME type - mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { - if o.DefaultMimeType != "" { - mimeType = o.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } + mimeType := o.getMimeTypeByExtension() ctx.Response.Header.SetContentType(mimeType) // Everything's okay so far @@ -185,20 +157,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) } } else { - _, err = ctx.Write(cachedResponse.body) + _, err = ctx.Write(cachedResponse.Body) } if err != nil { - fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) + fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err) html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } log.Debug().Msg("response") if res != nil && ctx.Err() == nil { - cachedResponse.exists = true - cachedResponse.mimeType = mimeType - cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) + cachedResponse.Exists = true + cachedResponse.MimeType = mimeType + cachedResponse.Body = cacheBodyWriter.Bytes() + _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) } return true