From 8f2699407d5c452e04bd0de8bbb367a60b4a88b8 Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 13 Sep 2022 23:06:31 +0200 Subject: [PATCH 01/62] Make verbose checks in tryBranch (#127) - It's likely that the tryBranch is returning false when it should be returning true, make these logs more verbose so they show up on production logs. Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/127 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/handler.go b/server/handler.go index cd67aa7..ed06659 100644 --- a/server/handler.go +++ b/server/handler.go @@ -85,7 +85,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, // also disallow search indexing and add a Link header to the canonical URL. tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool { if repo == "" { - log.Debug().Msg("tryBranch: repo is empty") + log.Warn().Msg("tryBranch: repo is empty") return false } @@ -96,7 +96,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Check if the branch exists, otherwise treat it as a file path branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache) if branchTimestampResult == nil { - log.Debug().Msg("tryBranch: branch doesn't exist") + log.Warn().Msg("tryBranch: branch doesn't exist") return false } From 2a730b2439cae5ebd17d90e38b4ff27e4154a651 Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 13 Sep 2022 23:26:45 +0200 Subject: [PATCH 02/62] Update README (#128) - Update readme accordingly to the https://codeberg.org/Codeberg/pages-server/commit/876a53d9a245c11c58b61617052f7f2281d16358 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/128 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 700f279..c4758ac 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. -- `DEBUG` (default: false): Set this to true to enable debug logging. +- `LOG_LEVEL` (default: warn): Set this to specify the level of logging. ## Contributing to the development @@ -106,4 +106,4 @@ run `just dev` now this pages should work: - https://magiclike.localhost.mock.directory:4430/ - https://momar.localhost.mock.directory:4430/ci-testing/ - - https://momar.localhost.mock.directory:4430/pag/@master/ \ No newline at end of file + - https://momar.localhost.mock.directory:4430/pag/@master/ From 091e6c8ed9f737b3b84a3610c5836953677d122c Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 18 Sep 2022 16:13:27 +0200 Subject: [PATCH 03/62] Add explicit logging in `GetBranchTimestamp` (#130) - Logs are currently indicating that it's returning `nil` in valid scenarios, therefor this patch adds extra logging in this code to better understand what it is doing in this function. Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/130 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/handler.go | 2 +- server/upstream/helper.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/handler.go b/server/handler.go index ed06659..fb8b419 100644 --- a/server/handler.go +++ b/server/handler.go @@ -296,7 +296,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, return } - log.Info().Msg("tryBranch, now trying upstream 7 %s") + log.Info().Msg("tryBranch, now trying upstream 7") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, canonicalDomainCache, branchTimestampCache, fileResponseCache) diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 0714dcd..28f4474 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -9,6 +9,7 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" + "github.com/rs/zerolog/log" ) type branchTimestamp struct { @@ -19,10 +20,13 @@ type branchTimestamp struct { // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp { + log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger() if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { + log.Debug().Msg("branchTimestampCache found item, but result is empty") return nil } + log.Debug().Msg("branchTimestampCache found item, returning result") return result.(*branchTimestamp) } result := &branchTimestamp{ @@ -32,16 +36,20 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, b // Get default branch defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) if err != nil { + log.Err(err).Msg("Could't fetch default branch from repository") _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil } + log.Debug().Msg("Succesfully fetched default branch from Gitea") result.Branch = defaultBranch } timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch) if err != nil { + log.Err(err).Msg("Could not get latest commit's timestamp from branch") return nil } + log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache") result.Timestamp = timestamp _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout) return result From df2228b6d59b175df87b8849391b4a8d317e7eb3 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 10 Oct 2022 23:25:21 +0200 Subject: [PATCH 04/62] ci: let tag run pipeline --- .woodpecker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 2674e9b..198648f 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,3 @@ -branches: main - pipeline: # use vendor to cache dependencies vendor: From b9e9f1420954d1a352f7e9e7e799b160bd224eb1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 10 Oct 2022 23:27:33 +0200 Subject: [PATCH 05/62] use codeberg.org/6543/docker-images/golang_just Signed-off-by: 6543 <6543@obermui.de> --- .woodpecker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 198648f..db51eba 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -17,7 +17,7 @@ pipeline: build: group: compliant - image: a6543/golang_just + image: codeberg.org/6543/docker-images/golang_just commands: - go version - just build @@ -37,7 +37,7 @@ pipeline: build-tag: group: compliant - image: a6543/golang_just + image: codeberg.org/6543/docker-images/golang_just commands: - go version - just build-tag ${CI_COMMIT_TAG##v} @@ -46,13 +46,13 @@ pipeline: test: group: test - image: a6543/golang_just + image: codeberg.org/6543/docker-images/golang_just commands: - just test integration-tests: group: test - image: a6543/golang_just + image: codeberg.org/6543/docker-images/golang_just commands: - just integration environment: From bf9a08e1fd4107667dc6816e1df0d0ec2c0acbe9 Mon Sep 17 00:00:00 2001 From: Gusted Date: Mon, 7 Nov 2022 16:27:37 +0100 Subject: [PATCH 06/62] Fatal on ACME Client creation failure (#133) - For production(*cough* Codeberg *cough*), it's important to not use mock certs. So fail right from the start if this is the case and not try to "handle it gracefully", as it would break production. - Resolves #131 CC @6543 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/133 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/certificates/certificates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 8944468..2f59fb4 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -414,7 +414,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * acmeClient, err = lego.NewClient(acmeConfig) if err != nil { - log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") + log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { From 91b54bef29ca9e7d3394fad1e44fa4b6f795035c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 7 Nov 2022 23:09:41 +0100 Subject: [PATCH 07/62] add newline --- server/upstream/upstream.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 0e27727..61c90de 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -85,6 +85,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, } else { res, err = giteaClient.ServeRawContent(o.generateUriClientArgs()) } + log.Debug().Msg("Aquisting") // Handle errors @@ -158,6 +159,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect) return true } + log.Debug().Msg("Handling error") // Set the MIME type @@ -200,6 +202,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } + log.Debug().Msg("Sending response") if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil { From 8e67d28c4fddfc0b4f4f0dd5db1e60e4d80cf56a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 11 Nov 2022 23:51:45 +0100 Subject: [PATCH 08/62] Add editorconfig, fix files and lint via ci --- .ecrc | 9 +++++++++ .editorconfig | 17 +++++++++++++++++ .woodpecker.yml | 4 ++++ Dockerfile | 4 ++-- Justfile | 1 + README.md | 8 +++----- haproxy-sni/dhparam.pem | 2 +- haproxy-sni/docker-compose.yml | 2 +- html/404.html | 6 +++--- 9 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 .ecrc create mode 100644 .editorconfig diff --git a/.ecrc b/.ecrc new file mode 100644 index 0000000..d9ee788 --- /dev/null +++ b/.ecrc @@ -0,0 +1,9 @@ +{ + "Exclude": [ + ".git", + "go.mod", "go.sum", + "vendor", + "LICENSE", + "_test.go" + ] +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..354a828 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +tab_width = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false +indent_size = 1 diff --git a/.woodpecker.yml b/.woodpecker.yml index db51eba..5c9a7fd 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -15,6 +15,10 @@ pipeline: - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }" - golangci-lint run --timeout 5m --build-tags integration + editor-config: + group: compliant + image: mstruebing/editorconfig-checker + build: group: compliant image: codeberg.org/6543/docker-images/golang_just diff --git a/Dockerfile b/Dockerfile index 71dd236..904d6f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,5 @@ COPY --from=build /workspace/pages /pages COPY --from=build \ /etc/ssl/certs/ca-certificates.crt \ /etc/ssl/certs/ca-certificates.crt - -ENTRYPOINT ["/pages"] \ No newline at end of file + +ENTRYPOINT ["/pages"] diff --git a/Justfile b/Justfile index 03a7436..f7ea414 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,7 @@ build-tag VERSION: lint: tool-golangci tool-gofumpt [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ golangci-lint run --timeout 5m --build-tags integration + # TODO: run editorconfig-checker fmt: tool-gofumpt gofumpt -w --extra . diff --git a/README.md b/README.md index c4758ac..50cdc3c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview) and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/). - ## Quickstart This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. @@ -20,17 +19,16 @@ first line will be the canonical domain/URL; all other occurrences will be redir 2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else. If the branch name contains slash characters, you need to replace "/" in the branch name to "~"): - `www.example.org. IN CNAME main.pages.example.codeberg.page.` + `www.example.org. IN CNAME main.pages.example.codeberg.page.` 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT record that points to your repo (just like the CNAME record): - `example.org IN ALIAS codeberg.page.` - `example.org IN TXT main.pages.example.codeberg.page.` + `example.org IN ALIAS codeberg.page.` + `example.org IN TXT main.pages.example.codeberg.page.` Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. - ## Deployment **Warning: Some Caveats Apply** diff --git a/haproxy-sni/dhparam.pem b/haproxy-sni/dhparam.pem index 088f967..9b182b7 100644 --- a/haproxy-sni/dhparam.pem +++ b/haproxy-sni/dhparam.pem @@ -5,4 +5,4 @@ MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== ------END DH PARAMETERS----- \ No newline at end of file +-----END DH PARAMETERS----- diff --git a/haproxy-sni/docker-compose.yml b/haproxy-sni/docker-compose.yml index 4dd8677..66ff52b 100644 --- a/haproxy-sni/docker-compose.yml +++ b/haproxy-sni/docker-compose.yml @@ -19,4 +19,4 @@ services: volumes: - ./pages-www:/srv:ro - ./pages.Caddyfile:/etc/caddy/Caddyfile:ro - + diff --git a/html/404.html b/html/404.html index b7ec96e..21d968e 100644 --- a/html/404.html +++ b/html/404.html @@ -23,11 +23,11 @@

- Page not found! + Page not found!

- Sorry, but this page couldn't be found or is inaccessible (%status).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! + Sorry, but this page couldn't be found or is inaccessible (%status).
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
From 69eabb248a26e33b65732604267c486fd0daee84 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 00:23:45 +0100 Subject: [PATCH 09/62] CI publish next only on default branch --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5c9a7fd..20254fe 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -94,6 +94,7 @@ pipeline: from_secret: bot_token when: event: [ "push" ] + branch: ${CI_REPO_DEFAULT_BRANCH} docker-tag: image: plugins/kaniko From b9966487f6f7a9fc88ae285c7350917e455f27a0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 20:37:20 +0100 Subject: [PATCH 10/62] switch to std http implementation instead of fasthttp (#106) close #100 close #109 close #113 close #28 close #63 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/106 --- cmd/main.go | 41 ++-- go.mod | 17 +- go.sum | 33 +-- html/404.html | 8 +- html/error.go | 47 +++-- html/error.html | 38 ++++ html/html.go | 5 +- integration/get_test.go | 33 ++- server/certificates/certificates.go | 36 ++-- server/context/context.go | 62 ++++++ server/database/mock.go | 2 +- server/dns/const.go | 6 - server/dns/dns.go | 4 + server/gitea/cache.go | 112 +++++++++- server/gitea/client.go | 308 ++++++++++++++++++++-------- server/gitea/client_test.go | 23 --- server/gitea/fasthttp.go | 15 -- server/handler.go | 186 +++++++++-------- server/handler_test.go | 36 ++-- server/helpers.go | 8 +- server/setup.go | 54 ++--- server/try.go | 21 +- server/upstream/const.go | 24 --- server/upstream/domains.go | 10 + server/upstream/helper.go | 74 ++----- server/upstream/upstream.go | 194 +++++++++--------- server/utils/utils.go | 8 +- server/utils/utils_test.go | 6 +- 28 files changed, 827 insertions(+), 584 deletions(-) create mode 100644 html/error.html create mode 100644 server/context/context.go delete mode 100644 server/dns/const.go delete mode 100644 server/gitea/client_test.go delete mode 100644 server/gitea/fasthttp.go delete mode 100644 server/upstream/const.go diff --git a/cmd/main.go b/cmd/main.go index 41809cb..a3a61e1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,12 +1,12 @@ package cmd import ( - "bytes" "context" "crypto/tls" "errors" "fmt" "net" + "net/http" "os" "strings" "time" @@ -24,15 +24,15 @@ import ( // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. // TODO: make it a flag -var AllowedCorsDomains = [][]byte{ - []byte("fonts.codeberg.org"), - []byte("design.codeberg.org"), +var AllowedCorsDomains = []string{ + "fonts.codeberg.org", + "design.codeberg.org", } // BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. // TODO: Make it a flag too -var BlacklistedPaths = [][]byte{ - []byte("/.well-known/acme-challenge/"), +var BlacklistedPaths = []string{ + "/.well-known/acme-challenge/", } // Serve sets up and starts the web server. @@ -47,7 +47,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") - mainDomainSuffix := []byte(ctx.String("pages-domain")) + mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") @@ -65,12 +65,12 @@ func Serve(ctx *cli.Context) error { allowedCorsDomains := AllowedCorsDomains if len(rawDomain) != 0 { - allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain)) + allowedCorsDomains = append(allowedCorsDomains, rawDomain) } // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash - if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) { - mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) + if !strings.HasPrefix(mainDomainSuffix, ".") { + mainDomainSuffix = "." + mainDomainSuffix } keyCache := cache.NewKeyValueCache() @@ -79,26 +79,22 @@ func Serve(ctx *cli.Context) error { canonicalDomainCache := cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains dnsLookupCache := cache.NewKeyValueCache() - // branchTimestampCache stores branch timestamps for faster cache checking - branchTimestampCache := cache.NewKeyValueCache() - // fileResponseCache stores responses from the Gitea server - // TODO: make this an MRU cache with a size limit - fileResponseCache := cache.NewKeyValueCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewKeyValueCache() - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) + giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) if err != nil { return fmt.Errorf("could not create new gitea client: %v", err) } // Create handler based on settings - handler := server.Handler(mainDomainSuffix, []byte(rawDomain), + httpsHandler := server.Handler(mainDomainSuffix, rawDomain, giteaClient, giteaRoot, rawInfoPage, BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) + dnsLookupCache, canonicalDomainCache) - fastServer := server.SetupServer(handler) - httpServer := server.SetupHTTPACMEChallengeServer(challengeCache) + httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) @@ -138,7 +134,7 @@ func Serve(ctx *cli.Context) error { if enableHTTPServer { go func() { log.Info().Msg("Start HTTP server listening on :80") - err := httpServer.ListenAndServe("[::]:80") + err := http.ListenAndServe("[::]:80", httpHandler) if err != nil { log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") } @@ -147,8 +143,7 @@ func Serve(ctx *cli.Context) error { // Start the web fastServer log.Info().Msgf("Start listening on %s", listener.Addr()) - err = fastServer.Serve(listener) - if err != nil { + if err := http.Serve(listener, httpsHandler); err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") } diff --git a/go.mod b/go.mod index 479c328..77ed762 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module codeberg.org/codeberg/pages -go 1.18 +go 1.19 require ( + code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 @@ -11,8 +12,6 @@ require ( github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 - github.com/valyala/fasthttp v1.31.0 - github.com/valyala/fastjson v1.6.3 ) require ( @@ -31,7 +30,6 @@ require ( github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect - github.com/andybalholm/brotli v1.0.2 // indirect github.com/aws/aws-sdk-go v1.39.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect @@ -39,6 +37,7 @@ require ( github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dnsimple/dnsimple-go v0.70.1 // indirect @@ -46,6 +45,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/go-errors/errors v1.0.1 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect @@ -57,13 +57,13 @@ require ( github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect github.com/jarcoal/httpmock v1.0.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.7 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect - github.com/klauspost/compress v1.13.4 // indirect github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect @@ -104,15 +104,14 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect github.com/vultr/govultr/v2 v2.7.1 // indirect go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect diff --git a/go.sum b/go.sum index 23a58bc..a44001c 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -66,8 +68,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbxuNdz3FeyvHFnRjmo= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= -github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -106,6 +106,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/deepmap/oapi-codegen v1.6.1 h1:2BvsmRb6pogGNtr8Ann+esAbSKFXx2CZN18VpAMecnw= github.com/deepmap/oapi-codegen v1.6.1/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -136,6 +138,8 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -182,7 +186,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -243,6 +246,9 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -282,8 +288,6 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcM github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -494,15 +498,9 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE= -github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= -github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 h1:TFXGGMHmml4rs29PdPisC/aaCzOxUu1Vsh9on/IpUfE= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14/go.mod h1:RWc47jtnVuQv6+lY3c768WtXCas/Xi+U5UFc5xULmYg= github.com/vultr/govultr/v2 v2.7.1 h1:uF9ERet++Gb+7Cqs3p1P6b6yebeaZqVd7t5P2uZCaJU= @@ -539,8 +537,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -604,8 +604,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -667,12 +667,13 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/html/404.html b/html/404.html index 21d968e..7c721b5 100644 --- a/html/404.html +++ b/html/404.html @@ -3,7 +3,7 @@ - %status + %status% @@ -23,11 +23,11 @@

- Page not found! + Page not found!

- Sorry, but this page couldn't be found or is inaccessible (%status).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! + Sorry, but this page couldn't be found or is inaccessible (%status%).
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
diff --git a/html/error.go b/html/error.go index 325dada..826c42b 100644 --- a/html/error.go +++ b/html/error.go @@ -1,24 +1,45 @@ package html import ( - "bytes" + "net/http" "strconv" + "strings" - "github.com/valyala/fasthttp" + "codeberg.org/codeberg/pages/server/context" ) -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) { - ctx.Response.SetStatusCode(code) - ctx.Response.Header.SetContentType("text/html; charset=utf-8") - message := fasthttp.StatusMessage(code) - if code == fasthttp.StatusMisdirectedRequest { - message += " - domain not specified in .domains file" +// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, +// with "%status%" and %message% replaced with the provided statusCode and msg +func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { + ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") + ctx.RespWriter.WriteHeader(statusCode) + + if msg == "" { + msg = errorBody(statusCode) + } else { + // TODO: use template engine + msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) } - if code == fasthttp.StatusFailedDependency { + + _, _ = ctx.RespWriter.Write([]byte(msg)) +} + +func errorMessage(statusCode int) string { + message := http.StatusText(statusCode) + + switch statusCode { + case http.StatusMisdirectedRequest: + message += " - domain not specified in .domains file" + case http.StatusFailedDependency: message += " - target repo/branch doesn't exist or is private" } - // TODO: use template engine? - ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) + + return message +} + +// TODO: use template engine +func errorBody(statusCode int) string { + return strings.ReplaceAll(NotFoundPage, + "%status%", + strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) } diff --git a/html/error.html b/html/error.html new file mode 100644 index 0000000..f1975f7 --- /dev/null +++ b/html/error.html @@ -0,0 +1,38 @@ + + + + + + %status% + + + + + + + + + +

+ %status%! +

+
+ Sorry, but this page couldn't be served.
+ We got an "%message%"
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! +
+ + + Static pages made easy - Codeberg Pages + + + diff --git a/html/html.go b/html/html.go index d223e15..a76ce59 100644 --- a/html/html.go +++ b/html/html.go @@ -3,4 +3,7 @@ package html import _ "embed" //go:embed 404.html -var NotFoundPage []byte +var NotFoundPage string + +//go:embed error.html +var ErrorPage string diff --git a/integration/get_test.go b/integration/get_test.go index 6054e17..8794651 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -25,7 +25,7 @@ func TestGetRedirect(t *testing.T) { t.FailNow() } assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location")) - assert.EqualValues(t, 0, getSize(resp.Body)) + assert.EqualValues(t, `Temporary Redirect.`, strings.TrimSpace(string(getBytes(resp.Body)))) } func TestGetContent(t *testing.T) { @@ -44,12 +44,13 @@ func TestGetContent(t *testing.T) { // specify branch resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.True(t, getSize(resp.Body) > 1000) - assert.Len(t, resp.Header.Get("ETag"), 42) + assert.Len(t, resp.Header.Get("ETag"), 44) // access branch name contains '/' resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/") @@ -59,7 +60,7 @@ func TestGetContent(t *testing.T) { } assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.True(t, getSize(resp.Body) > 100) - assert.Len(t, resp.Header.Get("ETag"), 42) + assert.Len(t, resp.Header.Get("ETag"), 44) // TODO: test get of non cachable content (content size > fileCacheSizeLimit) } @@ -68,9 +69,10 @@ func TestCustomDomain(t *testing.T) { log.Println("=== TestCustomDomain ===") resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type")) assert.EqualValues(t, "106", resp.Header.Get("Content-Length")) assert.EqualValues(t, 106, getSize(resp.Body)) @@ -81,9 +83,10 @@ func TestGetNotFound(t *testing.T) { // test custom not found pages resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.EqualValues(t, "37", resp.Header.Get("Content-Length")) assert.EqualValues(t, 37, getSize(resp.Body)) @@ -94,9 +97,10 @@ func TestFollowSymlink(t *testing.T) { resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type")) assert.EqualValues(t, "4", resp.Header.Get("Content-Length")) body := getBytes(resp.Body) @@ -109,14 +113,27 @@ func TestLFSSupport(t *testing.T) { resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) body := strings.TrimSpace(string(getBytes(resp.Body))) assert.EqualValues(t, 12, len(body)) assert.EqualValues(t, "actual value", body) } +func TestGetOptions(t *testing.T) { + log.Println("=== TestGetOptions ===") + req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", nil) + resp, err := getTestHTTPSClient().Do(req) + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusNoContent, resp.StatusCode) + assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow")) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 2f59fb4..429ab23 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -36,7 +36,7 @@ import ( ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TLSConfig(mainDomainSuffix []byte, +func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, dnsProvider string, acmeUseRateLimits bool, @@ -47,7 +47,6 @@ func TLSConfig(mainDomainSuffix []byte, // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { sni := strings.ToLower(strings.TrimSpace(info.ServerName)) - sniBytes := []byte(sni) if len(sni) < 1 { return nil, errors.New("missing sni") } @@ -69,23 +68,20 @@ func TLSConfig(mainDomainSuffix []byte, } targetOwner := "" - if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) { + if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) { // deliver default certificate for the main domain (*.codeberg.page) - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } else { _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, mainDomainSuffix, canonicalDomainCache) if !valid { - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } } } @@ -98,9 +94,9 @@ func TLSConfig(mainDomainSuffix []byte, var tlsCertificate tls.Certificate var err error var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { + if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { // request a new certificate - if bytes.Equal(sniBytes, mainDomainSuffix) { + if strings.EqualFold(sni, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } @@ -192,7 +188,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { // parse certificate from database res, err := certDB.Get(string(sni)) if err != nil { @@ -208,7 +204,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs } // TODO: document & put into own function - if !bytes.Equal(sni, mainDomainSuffix) { + if !strings.EqualFold(sni, mainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { panic(err) @@ -239,7 +235,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -252,7 +248,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } @@ -405,7 +401,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce return myAcmeConfig, nil } -func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { +func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits mainCertBytes, err := certDB.Get(string(mainDomainSuffix)) if err != nil { @@ -460,7 +456,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * return nil } -func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { +func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { for { // clean up expired certs now := time.Now() @@ -468,7 +464,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi keyDatabaseIterator := certDB.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { - if !bytes.Equal(key, mainDomainSuffix) { + if !strings.EqualFold(string(key), mainDomainSuffix) { resGob := bytes.NewBuffer(resBytes) resDec := gob.NewDecoder(resGob) res := &certificate.Resource{} diff --git a/server/context/context.go b/server/context/context.go new file mode 100644 index 0000000..be01df0 --- /dev/null +++ b/server/context/context.go @@ -0,0 +1,62 @@ +package context + +import ( + stdContext "context" + "net/http" +) + +type Context struct { + RespWriter http.ResponseWriter + Req *http.Request + StatusCode int +} + +func New(w http.ResponseWriter, r *http.Request) *Context { + return &Context{ + RespWriter: w, + Req: r, + StatusCode: http.StatusOK, + } +} + +func (c *Context) Context() stdContext.Context { + if c.Req != nil { + return c.Req.Context() + } + return stdContext.Background() +} + +func (c *Context) Response() *http.Response { + if c.Req != nil && c.Req.Response != nil { + return c.Req.Response + } + return nil +} + +func (c *Context) String(raw string, status ...int) { + code := http.StatusOK + if len(status) != 0 { + code = status[0] + } + c.RespWriter.WriteHeader(code) + _, _ = c.RespWriter.Write([]byte(raw)) +} + +func (c *Context) IsMethod(m string) bool { + return c.Req.Method == m +} + +func (c *Context) Redirect(uri string, statusCode int) { + http.Redirect(c.RespWriter, c.Req, uri, statusCode) +} + +// Path returns requested path. +// +// The returned bytes are valid until your request handler returns. +func (c *Context) Path() string { + return c.Req.URL.Path +} + +func (c *Context) Host() string { + return c.Req.URL.Host +} diff --git a/server/database/mock.go b/server/database/mock.go index e6c1b5a..dfe2316 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error { func (p tmpDB) Get(name string) (*certificate.Resource, error) { cert, has := p.intern.Get(name) if !has { - return nil, fmt.Errorf("cert for '%s' not found", name) + return nil, fmt.Errorf("cert for %q not found", name) } return cert.(*certificate.Resource), nil } diff --git a/server/dns/const.go b/server/dns/const.go deleted file mode 100644 index bb2413b..0000000 --- a/server/dns/const.go +++ /dev/null @@ -1,6 +0,0 @@ -package dns - -import "time" - -// lookupCacheTimeout specifies the timeout for the DNS lookup cache. -var lookupCacheTimeout = 15 * time.Minute diff --git a/server/dns/dns.go b/server/dns/dns.go index dc759b0..818e29a 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -3,10 +3,14 @@ package dns import ( "net" "strings" + "time" "codeberg.org/codeberg/pages/server/cache" ) +// lookupCacheTimeout specifies the timeout for the DNS lookup cache. +var lookupCacheTimeout = 15 * time.Minute + // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 932ff3c..b11a370 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -1,12 +1,116 @@ package gitea +import ( + "bytes" + "fmt" + "io" + "net/http" + "time" + + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/server/cache" +) + +const ( + // defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. + defaultBranchCacheTimeout = 15 * time.Minute + + // branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter + // than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be + // picked up faster, while still allowing the content to be cached longer if nothing changes. + branchExistenceCacheTimeout = 5 * time.Minute + + // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending + // on your available memory. + // TODO: move as option into cache interface + fileCacheTimeout = 5 * time.Minute + + // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. + fileCacheSizeLimit = int64(1000 * 1000) +) + type FileResponse struct { - Exists bool - ETag []byte - MimeType string - Body []byte + Exists bool + IsSymlink bool + ETag string + MimeType string + Body []byte } func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } + +func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) { + header := make(http.Header) + var statusCode int + + if f.Exists { + statusCode = http.StatusOK + } else { + statusCode = http.StatusNotFound + } + + if f.IsSymlink { + header.Set(giteaObjectTypeHeader, objTypeSymlink) + } + header.Set(ETagHeader, f.ETag) + header.Set(ContentTypeHeader, f.MimeType) + header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body))) + header.Set(PagesCacheIndicatorHeader, "true") + + log.Trace().Msgf("fileCache for %q used", cacheKey) + return header, statusCode +} + +type BranchTimestamp struct { + Branch string + Timestamp time.Time + notFound bool +} + +type writeCacheReader struct { + originalReader io.ReadCloser + buffer *bytes.Buffer + rileResponse *FileResponse + cacheKey string + cache cache.SetGetKey + hasError bool +} + +func (t *writeCacheReader) Read(p []byte) (n int, err error) { + n, err = t.originalReader.Read(p) + if err != nil { + log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) + t.hasError = true + } else if n > 0 { + _, _ = t.buffer.Write(p[:n]) + } + return +} + +func (t *writeCacheReader) Close() error { + if !t.hasError { + fc := *t.rileResponse + fc.Body = t.buffer.Bytes() + _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + } + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError) + return t.originalReader.Close() +} + +func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { + if r == nil || cache == nil || cacheKey == "" { + log.Error().Msg("could not create CacheReader") + return nil + } + + return &writeCacheReader{ + originalReader: r, + buffer: bytes.NewBuffer(make([]byte, 0)), + rileResponse: &f, + cache: cache, + cacheKey: cacheKey, + } +} diff --git a/server/gitea/client.go b/server/gitea/client.go index 16cba84..c63ee21 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -1,142 +1,276 @@ package gitea import ( + "bytes" "errors" "fmt" + "io" + "mime" + "net/http" "net/url" + "path" + "strconv" "strings" "time" + "code.gitea.io/sdk/gitea" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" -) -const ( - giteaAPIRepos = "/api/v1/repos/" - giteaObjectTypeHeader = "X-Gitea-Object-Type" + "codeberg.org/codeberg/pages/server/cache" ) var ErrorNotFound = errors.New("not found") +const ( + // cache key prefixe + branchTimestampCacheKeyPrefix = "branchTime" + defaultBranchCacheKeyPrefix = "defaultBranch" + rawContentCacheKeyPrefix = "rawContent" + + // pages server + PagesCacheIndicatorHeader = "X-Pages-Cache" + symlinkReadLimit = 10000 + + // gitea + giteaObjectTypeHeader = "X-Gitea-Object-Type" + objTypeSymlink = "symlink" + + // std + ETagHeader = "ETag" + ContentTypeHeader = "Content-Type" + ContentLengthHeader = "Content-Length" +) + type Client struct { - giteaRoot string - giteaAPIToken string - fastClient *fasthttp.Client - infoTimeout time.Duration - contentTimeout time.Duration + sdkClient *gitea.Client + responseCache cache.SetGetKey followSymlinks bool supportLFS bool + + forbiddenMimeTypes map[string]bool + defaultMimeType string } -// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package -func joinURL(baseURL string, paths ...string) string { - p := make([]string, 0, len(paths)) - for i := range paths { - path := strings.TrimSpace(paths[i]) - path = strings.Trim(path, "/") - if len(path) != 0 { - p = append(p, path) - } - } - - return baseURL + "/" + strings.Join(p, "/") -} - -func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) { +func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { rootURL, err := url.Parse(giteaRoot) + if err != nil { + return nil, err + } giteaRoot = strings.Trim(rootURL.String(), "/") + stdClient := http.Client{Timeout: 10 * time.Second} + + // TODO: pass down + var ( + forbiddenMimeTypes map[string]bool + defaultMimeType string + ) + + if forbiddenMimeTypes == nil { + forbiddenMimeTypes = make(map[string]bool) + } + if defaultMimeType == "" { + defaultMimeType = "application/octet-stream" + } + + sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) return &Client{ - giteaRoot: giteaRoot, - giteaAPIToken: giteaAPIToken, - infoTimeout: 5 * time.Second, - contentTimeout: 10 * time.Second, - fastClient: getFastHTTPClient(), + sdkClient: sdk, + responseCache: respCache, followSymlinks: followSymlinks, supportLFS: supportLFS, + + forbiddenMimeTypes: forbiddenMimeTypes, + defaultMimeType: defaultMimeType, }, err } func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { - resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) + reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) if err != nil { return nil, err } - return resp.Body(), nil + defer reader.Close() + return io.ReadAll(reader) } -func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { - var apiURL string - if client.supportLFS { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref)) - } else { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - } - resp, err := client.do(client.contentTimeout, apiURL) - if err != nil { - return nil, err - } +func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) { + cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource) + log := log.With().Str("cache_key", cacheKey).Logger() - if err != nil { - return nil, err - } - - switch resp.StatusCode() { - case fasthttp.StatusOK: - objType := string(resp.Header.Peek(giteaObjectTypeHeader)) - log.Trace().Msgf("server raw content object: %s", objType) - if client.followSymlinks && objType == "symlink" { - // TODO: limit to 1000 chars if we switched to std - linkDest := strings.TrimSpace(string(resp.Body())) - log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) - return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + // handle if cache entry exist + if cache, ok := client.responseCache.Get(cacheKey); ok { + cache := cache.(FileResponse) + cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) + // TODO: check against some timestamp missmatch?!? + if cache.Exists { + if cache.IsSymlink { + linkDest := string(cache.Body) + log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } else { + log.Debug().Msg("[cache] return bytes") + return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil + } + } else { + return nil, cachedHeader, cachedStatusCode, ErrorNotFound } - - return resp, nil - - case fasthttp.StatusNotFound: - return nil, ErrorNotFound - - default: - return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode()) } + + // not in cache, open reader via gitea api + reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) + if resp != nil { + switch resp.StatusCode { + case http.StatusOK: + // first handle symlinks + { + objType := resp.Header.Get(giteaObjectTypeHeader) + log.Trace().Msgf("server raw content object %q", objType) + if client.followSymlinks && objType == objTypeSymlink { + defer reader.Close() + // read limited chars for symlink + linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit)) + if err != nil { + return nil, nil, http.StatusInternalServerError, err + } + linkDest := strings.TrimSpace(string(linkDestBytes)) + + // we store symlink not content to reduce duplicates in cache + if err := client.responseCache.Set(cacheKey, FileResponse{ + Exists: true, + IsSymlink: true, + Body: []byte(linkDest), + ETag: resp.Header.Get(ETagHeader), + }, fileCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + + log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } + } + + // now we are sure it's content so set the MIME type + mimeType := client.getMimeTypeByExtension(resource) + resp.Response.Header.Set(ContentTypeHeader, mimeType) + + if !shouldRespBeSavedToCache(resp.Response) { + return reader, resp.Response.Header, resp.StatusCode, err + } + + // now we write to cache and respond at the sime time + fileResp := FileResponse{ + Exists: true, + ETag: resp.Header.Get(ETagHeader), + MimeType: mimeType, + } + return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil + + case http.StatusNotFound: + if err := client.responseCache.Set(cacheKey, FileResponse{ + Exists: false, + ETag: resp.Header.Get(ETagHeader), + }, fileCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + + return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound + default: + return nil, resp.Response.Header, resp.StatusCode, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) + } + } + return nil, nil, http.StatusInternalServerError, err } -func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName) - res, err := client.do(client.infoTimeout, url) +func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) { + cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) + + if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { + branchTimeStamp := stamp.(*BranchTimestamp) + if branchTimeStamp.notFound { + log.Trace().Msgf("[cache] use branch %q not found", branchName) + return &BranchTimestamp{}, ErrorNotFound + } + log.Trace().Msgf("[cache] use branch %q exist", branchName) + return branchTimeStamp, nil + } + + branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) if err != nil { - return time.Time{}, err + if resp != nil && resp.StatusCode == http.StatusNotFound { + log.Trace().Msgf("[cache] set cache branch %q not found", branchName) + if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return &BranchTimestamp{}, ErrorNotFound + } + return &BranchTimestamp{}, err } - if res.StatusCode() != fasthttp.StatusOK { - return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + if resp.StatusCode != http.StatusOK { + return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) } - return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) + + stamp := &BranchTimestamp{ + Branch: branch.Name, + Timestamp: branch.Commit.Timestamp, + } + + log.Trace().Msgf("set cache branch [%s] exist", branchName) + if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return stamp, nil } func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName) - res, err := client.do(client.infoTimeout, url) + cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) + + if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil { + return branch.(string), nil + } + + repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName) if err != nil { return "", err } - if res.StatusCode() != fasthttp.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode) } - return fastjson.GetString(res.Body(), "default_branch"), nil + + branch := repo.DefaultBranch + if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return branch, nil } -func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(url) - req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken) - res := fasthttp.AcquireResponse() - - err := client.fastClient.DoTimeout(req, res, timeout) - - return res, err +func (client *Client) getMimeTypeByExtension(resource string) string { + mimeType := mime.TypeByExtension(path.Ext(resource)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { + mimeType = client.defaultMimeType + } + log.Trace().Msgf("probe mime of %q is %q", resource, mimeType) + return mimeType +} + +func shouldRespBeSavedToCache(resp *http.Response) bool { + if resp == nil { + return false + } + + contentLengthRaw := resp.Header.Get(ContentLengthHeader) + if contentLengthRaw == "" { + return false + } + + contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64) + if err != nil { + log.Error().Err(err).Msg("could not parse content length") + } + + // if content to big or could not be determined we not cache it + return contentLeng > 0 && contentLeng < fileCacheSizeLimit } diff --git a/server/gitea/client_test.go b/server/gitea/client_test.go deleted file mode 100644 index 7dbad68..0000000 --- a/server/gitea/client_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package gitea - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestJoinURL(t *testing.T) { - baseURL := "" - assert.EqualValues(t, "/", joinURL(baseURL)) - assert.EqualValues(t, "/", joinURL(baseURL, "", "")) - - baseURL = "http://wwow.url.com" - assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d")) - - baseURL = "http://wow.url.com/subpath/2" - assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main"))) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main")) -} diff --git a/server/gitea/fasthttp.go b/server/gitea/fasthttp.go deleted file mode 100644 index 4ff0f4a..0000000 --- a/server/gitea/fasthttp.go +++ /dev/null @@ -1,15 +0,0 @@ -package gitea - -import ( - "time" - - "github.com/valyala/fasthttp" -) - -func getFastHTTPClient() *fasthttp.Client { - return &fasthttp.Client{ - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! - } -} diff --git a/server/handler.go b/server/handler.go index fb8b419..894cd25 100644 --- a/server/handler.go +++ b/server/handler.go @@ -1,15 +1,17 @@ package server import ( - "bytes" + "fmt" + "net/http" + "path" "strings" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/dns" "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" @@ -17,42 +19,48 @@ import ( "codeberg.org/codeberg/pages/server/version" ) +const ( + headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" + headerAccessControlAllowMethods = "Access-Control-Allow-Methods" +) + // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain []byte, +func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, giteaRoot, rawInfoPage string, - blacklistedPaths, allowedCorsDomains [][]byte, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, -) func(ctx *fasthttp.RequestCtx) { - return func(ctx *fasthttp.RequestCtx) { - log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger() + blacklistedPaths, allowedCorsDomains []string, + dnsLookupCache, canonicalDomainCache cache.SetGetKey, +) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger() + ctx := context.New(w, req) - ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version) + ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin - ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin") + ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") // Enable browser caching for up to 10 minutes - ctx.Response.Header.Set("Cache-Control", "public, max-age=600") + ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600") - trimmedHost := utils.TrimHostPort(ctx.Request.Host()) + trimmedHost := utils.TrimHostPort(req.Host) // Add HSTS for RawDomain and MainDomainSuffix - if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { - ctx.Response.Header.Set("Strict-Transport-Security", hsts) + if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) } // Block all methods not required for static pages - if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() { - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed) + if !ctx.IsMethod(http.MethodGet) && !ctx.IsMethod(http.MethodHead) && !ctx.IsMethod(http.MethodOptions) { + ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 + ctx.String("Method not allowed", http.StatusMethodNotAllowed) return } // Block blacklisted paths (like ACME challenges) for _, blacklistedPath := range blacklistedPaths { - if bytes.HasPrefix(ctx.Path(), blacklistedPath) { - html.ReturnErrorPage(ctx, fasthttp.StatusForbidden) + if strings.HasPrefix(ctx.Path(), blacklistedPath) { + html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) return } } @@ -60,18 +68,19 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Allow CORS for specified domains allowCors := false for _, allowedCorsDomain := range allowedCorsDomains { - if bytes.Equal(trimmedHost, allowedCorsDomain) { + if strings.EqualFold(trimmedHost, allowedCorsDomain) { allowCors = true break } } if allowCors { - ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") - ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD") + ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*") + ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead) } - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - if ctx.IsOptions() { - ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent) + + ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 + if ctx.IsMethod(http.MethodOptions) { + ctx.RespWriter.WriteHeader(http.StatusNoContent) return } @@ -83,9 +92,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // also disallow search indexing and add a Link header to the canonical URL. - tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool { + // TODO: move into external func to not alert vars indirectly + tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool { if repo == "" { - log.Warn().Msg("tryBranch: repo is empty") + log.Debug().Msg("tryBranch: repo is empty") return false } @@ -94,23 +104,23 @@ func Handler(mainDomainSuffix, rawDomain []byte, branch = strings.ReplaceAll(branch, "~", "/") // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache) + branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch) if branchTimestampResult == nil { - log.Warn().Msg("tryBranch: branch doesn't exist") + log.Debug().Msg("tryBranch: branch doesn't exist") return false } // Branch exists, use it targetRepo = repo - targetPath = strings.Trim(strings.Join(path, "/"), "/") + targetPath = path.Join(_path...) targetBranch = branchTimestampResult.Branch targetOptions.BranchTimestamp = branchTimestampResult.Timestamp if canonicalLink != "" { // Hide from search machines & add canonical link - ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex") - ctx.Response.Header.Set("Link", + ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex") + ctx.RespWriter.Header().Set("Link", strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ "; rel=\"canonical\"", ) @@ -120,22 +130,18 @@ func Handler(mainDomainSuffix, rawDomain []byte, return true } - log.Debug().Msg("Preparing") - if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) { + log.Debug().Msg("preparations") + if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { // Serve raw content from RawDomain - log.Debug().Msg("Serving raw domain") + log.Debug().Msg("raw domain") targetOptions.TryIndexPages = false - if targetOptions.ForbiddenMimeTypes == nil { - targetOptions.ForbiddenMimeTypes = make(map[string]bool) - } - targetOptions.ForbiddenMimeTypes["text/html"] = true - targetOptions.DefaultMimeType = "text/plain; charset=utf-8" + targetOptions.ServeRaw = true - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") if len(pathElements) < 2 { // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect) + ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) return } targetOwner = pathElements[0] @@ -143,45 +149,45 @@ func Handler(mainDomainSuffix, rawDomain []byte, // raw.codeberg.org/example/myrepo/@main/index.html if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - log.Debug().Msg("Preparing raw domain, now trying with specified branch") + log.Debug().Msg("raw domain preparations, now trying with specified branch") if tryBranch(log, targetRepo, pathElements[2][1:], pathElements[3:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { - log.Info().Msg("tryBranch, now trying upstream 1") + log.Debug().Msg("tryBranch, now trying upstream 1") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } - log.Warn().Msg("Path missed a branch") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + log.Debug().Msg("missing branch info") + html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency) return } - log.Debug().Msg("Preparing raw domain, now trying with default branch") + log.Debug().Msg("raw domain preparations, now trying with default branch") tryBranch(log, targetRepo, "", pathElements[2:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) - log.Info().Msg("tryBranch, now trying upstream 2") + log.Debug().Msg("tryBranch, now trying upstream 2") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return - } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { // Serve pages from subdomains of MainDomainSuffix - log.Info().Msg("Serve pages from main domain suffix") + log.Debug().Msg("main domain suffix") - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix)) + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") + targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix) targetRepo = pathElements[0] targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") if targetOwner == "www" { // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? - ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) + ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect) return } @@ -190,22 +196,24 @@ func Handler(mainDomainSuffix, rawDomain []byte, if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { if targetRepo == "pages" { // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... - ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect) + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) return } - log.Debug().Msg("Preparing main domain, now trying with specified repo & branch") + log.Debug().Msg("main domain preparations, now trying with specified repo & branch") + branch := pathElements[1][1:] if tryBranch(log, - pathElements[0], pathElements[1][1:], pathElements[2:], + pathElements[0], branch, pathElements[2:], "/"+pathElements[0]+"/%p", ) { - log.Info().Msg("tryBranch, now trying upstream 3") + log.Debug().Msg("tryBranch, now trying upstream 3") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) } else { - log.Warn().Msg("tryBranch: upstream 3 failed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo), + http.StatusFailedDependency) } return } @@ -213,16 +221,18 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Check if the first directory is a branch for the "pages" repo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { - log.Debug().Msg("Preparing main domain, now trying with specified branch") + log.Debug().Msg("main domain preparations, now trying with specified branch") + branch := pathElements[0][1:] if tryBranch(log, - "pages", pathElements[0][1:], pathElements[1:], "/%p") { - log.Info().Msg("tryBranch, now trying upstream 4") + "pages", branch, pathElements[1:], "/%p") { + log.Debug().Msg("tryBranch, now trying upstream 4") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + targetOptions, targetOwner, "pages", targetBranch, targetPath, + canonicalDomainCache) } else { - log.Warn().Msg("tryBranch: upstream 4 failed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"), + http.StatusFailedDependency) } return } @@ -233,10 +243,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified repo") if pathElements[0] != "pages" && tryBranch(log, pathElements[0], "pages", pathElements[1:], "") { - log.Info().Msg("tryBranch, now trying upstream 5") + log.Debug().Msg("tryBranch, now trying upstream 5") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } @@ -245,28 +255,31 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with default repo/branch") if tryBranch(log, "pages", "", pathElements, "") { - log.Info().Msg("tryBranch, now trying upstream 6") + log.Debug().Msg("tryBranch, now trying upstream 6") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } // Couldn't find a valid repo/branch - - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("couldn't find a valid repo[%s]/branch[%s]", targetRepo, targetBranch), + http.StatusFailedDependency) return } else { trimmedHostStr := string(trimmedHost) - // Serve pages from external domains + // Serve pages from custom domains targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + "could not obtain repo owner from custom domain", + http.StatusFailedDependency) return } - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") canonicalLink := "" if strings.HasPrefix(pathElements[0], "@") { targetBranch = pathElements[0][1:] @@ -275,36 +288,33 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Try to use the given repo on the given branch or the default branch - log.Debug().Msg("Preparing custom domain, now trying with details from DNS") + log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(log, targetRepo, targetBranch, pathElements, canonicalLink) { canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) if !valid { - log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical") - html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHostStr { // only redirect if the target is also a codeberg page! targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect) return } - log.Warn().Msg("Custom domains, targetOwner from DNS is empty") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency) return } - log.Info().Msg("tryBranch, now trying upstream 7") + log.Debug().Msg("tryBranch, now trying upstream 7") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } - log.Warn().Msg("Couldn't handle request, none of the options succeed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency) return } } diff --git a/server/handler_test.go b/server/handler_test.go index f9a721a..c0aca14 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1,44 +1,42 @@ package server import ( - "fmt" + "net/http/httptest" "testing" "time" - "github.com/valyala/fasthttp" - "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" + "github.com/rs/zerolog/log" ) func TestHandlerPerformance(t *testing.T) { giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false) + giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false) testHandler := Handler( - []byte("codeberg.page"), []byte("raw.codeberg.org"), + "codeberg.page", "raw.codeberg.org", giteaClient, giteaRoot, "https://docs.codeberg.org/pages/raw-content/", - [][]byte{[]byte("/.well-known/acme-challenge/")}, - [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), + []string{"/.well-known/acme-challenge/"}, + []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), ) testCase := func(uri string, status int) { - ctx := &fasthttp.RequestCtx{ - Request: *fasthttp.AcquireRequest(), - Response: *fasthttp.AcquireResponse(), - } - ctx.Request.SetRequestURI(uri) - fmt.Printf("Start: %v\n", time.Now()) + req := httptest.NewRequest("GET", uri, nil) + w := httptest.NewRecorder() + + log.Printf("Start: %v\n", time.Now()) start := time.Now() - testHandler(ctx) + testHandler(w, req) end := time.Now() - fmt.Printf("Done: %v\n", time.Now()) - if ctx.Response.StatusCode() != status { - t.Errorf("request failed with status code %d", ctx.Response.StatusCode()) + log.Printf("Done: %v\n", time.Now()) + + resp := w.Result() + + if resp.StatusCode != status { + t.Errorf("request failed with status code %d", resp.StatusCode) } else { t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds()) } diff --git a/server/helpers.go b/server/helpers.go index 6d55ddf..7c898cd 100644 --- a/server/helpers.go +++ b/server/helpers.go @@ -1,13 +1,13 @@ package server import ( - "bytes" + "strings" ) -// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty +// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty // string for custom domains. -func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string { - if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) { +func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string { + if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) { return "max-age=63072000; includeSubdomains; preload" } else { return "" diff --git a/server/setup.go b/server/setup.go index 176bb42..e7194ed 100644 --- a/server/setup.go +++ b/server/setup.go @@ -1,53 +1,27 @@ package server import ( - "bytes" - "fmt" "net/http" - "time" - - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" + "strings" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/utils" ) -type fasthttpLogger struct{} +func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc { + challengePath := "/.well-known/acme-challenge/" -func (fasthttpLogger) Printf(format string, args ...interface{}) { - log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...)) -} - -func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server { - // Enable compression by wrapping the handler with the compression function provided by FastHTTP - compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - - return &fasthttp.Server{ - Handler: compressedHandler, - DisablePreParseMultipartForm: true, - NoDefaultServerHeader: true, - NoDefaultDate: true, - ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge - Logger: fasthttpLogger{}, - } -} - -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { - challengePath := []byte("/.well-known/acme-challenge/") - - return &fasthttp.Server{ - Handler: func(ctx *fasthttp.RequestCtx) { - if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) - if !ok || challenge == nil { - ctx.SetStatusCode(http.StatusNotFound) - ctx.SetBodyString("no challenge for this token") - } - ctx.SetBodyString(challenge.(string)) - } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) + return func(w http.ResponseWriter, req *http.Request) { + ctx := context.New(w, req) + if strings.HasPrefix(ctx.Path(), challengePath) { + challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + string(strings.TrimPrefix(ctx.Path(), challengePath))) + if !ok || challenge == nil { + ctx.String("no challenge for this token", http.StatusNotFound) } - }, + ctx.String(challenge.(string)) + } else { + ctx.Redirect("https://"+string(ctx.Host())+string(ctx.Path()), http.StatusMovedPermanently) + } } } diff --git a/server/try.go b/server/try.go index 24831c4..135c1e0 100644 --- a/server/try.go +++ b/server/try.go @@ -1,38 +1,37 @@ package server import ( - "bytes" + "net/http" "strings" - "github.com/valyala/fasthttp" - "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, - mainDomainSuffix, trimmedHost []byte, +func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix, trimmedHost string, targetOptions *upstream.Options, targetOwner, targetRepo, targetBranch, targetPath string, - canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, + canonicalDomainCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain - if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + if strings.HasSuffix(trimmedHost, mainDomainSuffix) { canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { - canonicalPath := string(ctx.RequestURI()) + canonicalPath := ctx.Req.RequestURI if targetRepo != "pages" { path := strings.SplitN(canonicalPath, "/", 3) if len(path) >= 3 { canonicalPath = "/" + path[2] } } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect) return } } @@ -44,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, targetOptions.Host = string(trimmedHost) // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) + if !targetOptions.Upstream(ctx, giteaClient) { + html.ReturnErrorPage(ctx, "", ctx.StatusCode) } } diff --git a/server/upstream/const.go b/server/upstream/const.go deleted file mode 100644 index 247e1d1..0000000 --- a/server/upstream/const.go +++ /dev/null @@ -1,24 +0,0 @@ -package upstream - -import "time" - -// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. -var defaultBranchCacheTimeout = 15 * time.Minute - -// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter -// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be -// picked up faster, while still allowing the content to be cached longer if nothing changes. -var branchExistenceCacheTimeout = 5 * time.Minute - -// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending -// on your available memory. -// TODO: move as option into cache interface -var fileCacheTimeout = 5 * time.Minute - -// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. -var fileCacheSizeLimit = 1024 * 1024 - -// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. -var canonicalDomainCacheTimeout = 15 * time.Minute - -const canonicalDomainConfig = ".domains" diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 553c148..6ad6506 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -2,11 +2,19 @@ package upstream import ( "strings" + "time" + + "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" ) +// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. +var canonicalDomainCacheTimeout = 15 * time.Minute + +const canonicalDomainConfig = ".domains" + // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( @@ -36,6 +44,8 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta valid = true } } + } else { + log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo) } domains = append(domains, targetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 28f4474..6bc23c8 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -1,84 +1,36 @@ package upstream import ( - "mime" - "path" - "strconv" - "strings" - "time" + "errors" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/gitea" "github.com/rs/zerolog/log" -) -type branchTimestamp struct { - Branch string - Timestamp time.Time -} + "codeberg.org/codeberg/pages/server/gitea" +) // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp { +func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp { log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger() - if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { - if result == nil { - log.Debug().Msg("branchTimestampCache found item, but result is empty") - return nil - } - log.Debug().Msg("branchTimestampCache found item, returning result") - return result.(*branchTimestamp) - } - result := &branchTimestamp{ - Branch: branch, - } + if len(branch) == 0 { // Get default branch defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) if err != nil { log.Err(err).Msg("Could't fetch default branch from repository") - _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil } - log.Debug().Msg("Succesfully fetched default branch from Gitea") - result.Branch = defaultBranch + log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch) + branch = defaultBranch } - timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch) + timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch) if err != nil { - log.Err(err).Msg("Could not get latest commit's timestamp from branch") + if !errors.Is(err, gitea.ErrorNotFound) { + log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch") + } return nil } - log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache") - result.Timestamp = timestamp - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout) - return result -} - -func (o *Options) getMimeTypeByExtension() string { - if o.ForbiddenMimeTypes == nil { - o.ForbiddenMimeTypes = make(map[string]bool) - } - mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { - if o.DefaultMimeType != "" { - mimeType = o.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } - return mimeType -} - -func (o *Options) generateUri() string { - return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) -} - -func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) { - return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath -} - -func (o *Options) timestamp() string { - return strconv.FormatInt(o.BranchTimestamp.Unix(), 10) + log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp) + return timestamp } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 61c90de..d37c35e 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -1,20 +1,27 @@ package upstream import ( - "bytes" "errors" + "fmt" "io" + "net/http" "strings" "time" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" ) +const ( + headerLastModified = "Last-Modified" + headerIfModifiedSince = "If-Modified-Since" + + rawMime = "text/plain; charset=utf-8" +) + // upstreamIndexPages lists pages that may be considered as index pages for directories. var upstreamIndexPages = []string{ "index.html", @@ -35,61 +42,61 @@ type Options struct { // Used for debugging purposes. Host string - DefaultMimeType string - ForbiddenMimeTypes map[string]bool - TryIndexPages bool - BranchTimestamp time.Time + TryIndexPages bool + BranchTimestamp time.Time // internal appendTrailingSlash bool redirectIfExists string + + ServeRaw bool } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { - log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger() +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) { + log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() + + if o.TargetOwner == "" || o.TargetRepo == "" { + html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest) + return true + } // Check if the branch exists and when it was modified if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache) + branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) - if branch == nil { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + if branch == nil || branch.Branch == "" { + html.ReturnErrorPage(ctx, + fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch), + http.StatusFailedDependency) return true } o.TargetBranch = branch.Branch o.BranchTimestamp = branch.Timestamp } - if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest) - return true - } - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(o.BranchTimestamp) { - ctx.Response.SetStatusCode(fasthttp.StatusNotModified) - return true + if ctx.Response() != nil { + if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil { + if !ifModifiedSince.Before(o.BranchTimestamp) { + ctx.RespWriter.WriteHeader(http.StatusNotModified) + log.Trace().Msg("check response against last modified: valid") + return true + } } + log.Trace().Msg("check response against last modified: outdated") } log.Debug().Msg("Preparing") - // Make a GET request to the upstream URL - uri := o.generateUri() - var res *fasthttp.Response - var cachedResponse gitea.FileResponse - var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { - cachedResponse = cachedValue.(gitea.FileResponse) - } else { - res, err = giteaClient.ServeRawContent(o.generateUriClientArgs()) + reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath) + if reader != nil { + defer reader.Close() } log.Debug().Msg("Aquisting") - // Handle errors - if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) { + // Handle not found error + if err != nil && errors.Is(err, gitea.ErrorNotFound) { if o.TryIndexPages { // copy the o struct & try if an index page exists optionsForIndexPages := *o @@ -97,25 +104,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForIndexPages.Upstream(ctx, giteaClient) { return true } } // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.appendTrailingSlash = false - optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" + optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForIndexPages.Upstream(ctx, giteaClient) { return true } } - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) + + ctx.StatusCode = http.StatusNotFound if o.TryIndexPages { // copy the o struct & try if a not found page exists optionsForNotFoundPages := *o @@ -123,94 +125,84 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, optionsForNotFoundPages.appendTrailingSlash = false for _, notFoundPage := range upstreamNotFoundPages { optionsForNotFoundPages.TargetPath = "/" + notFoundPage - if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForNotFoundPages.Upstream(ctx, giteaClient) { return true } } } - if res != nil { - // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) - } return false } - if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode()) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) + + // handle unexpected client errors + if err != nil || reader == nil || statusCode != http.StatusOK { + log.Debug().Msg("Handling error") + var msg string + + if err != nil { + msg = "gitea client returned unexpected error" + log.Error().Err(err).Msg(msg) + msg = fmt.Sprintf("%s: %v", msg, err) + } + if reader == nil { + msg = "gitea client returned no reader" + log.Error().Msg(msg) + } + if statusCode != http.StatusOK { + msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode) + log.Error().Msg(msg) + } + + html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError) return true } // Append trailing slash if missing (for index files), and redirect to fix filenames in general // o.appendTrailingSlash is only true when looking for index pages - if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { - ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) + if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { + ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) return true } - if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { - ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) + if strings.HasSuffix(ctx.Path(), "/index.html") { + ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) return true } if o.redirectIfExists != "" { - ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect) + ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect) return true } - log.Debug().Msg("Handling error") - - // Set the MIME type - mimeType := o.getMimeTypeByExtension() - ctx.Response.Header.SetContentType(mimeType) - - // Set ETag - if cachedResponse.Exists { - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) - } else if res != nil { - cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag) - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) + // Set ETag & MIME + if eTag := header.Get(gitea.ETagHeader); eTag != "" { + ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag) } - - if ctx.Response.StatusCode() != fasthttp.StatusNotFound { - // Everything's okay so far - ctx.Response.SetStatusCode(fasthttp.StatusOK) + if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" { + ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator) } - ctx.Response.Header.SetLastModified(o.BranchTimestamp) + if length := header.Get(gitea.ContentLengthHeader); length != "" { + ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length) + } + if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime) + } else { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime) + } + ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) log.Debug().Msg("Prepare response") - // Write the response body to the original request - var cacheBodyWriter bytes.Buffer - if res != nil { - if res.Header.ContentLength() > fileCacheSizeLimit { - // fasthttp else will set "Content-Length: 0" - ctx.Response.SetBodyStream(&strings.Reader{}, -1) + ctx.RespWriter.WriteHeader(ctx.StatusCode) - err = res.BodyWriteTo(ctx.Response.BodyWriter()) - } else { - // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? - err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) + // Write the response body to the original request + if reader != nil { + _, err := io.Copy(ctx.RespWriter, reader) + if err != nil { + log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath) + html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) + return true } - } else { - _, err = ctx.Write(cachedResponse.Body) - } - if err != nil { - log.Error().Err(err).Msgf("Couldn't write body for %q", uri) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true } log.Debug().Msg("Sending response") - if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil { - cachedResponse.Exists = true - cachedResponse.MimeType = mimeType - cachedResponse.Body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) - } - return true } diff --git a/server/utils/utils.go b/server/utils/utils.go index 7be330f..30f948d 100644 --- a/server/utils/utils.go +++ b/server/utils/utils.go @@ -1,9 +1,11 @@ package utils -import "bytes" +import ( + "strings" +) -func TrimHostPort(host []byte) []byte { - i := bytes.IndexByte(host, ':') +func TrimHostPort(host string) string { + i := strings.IndexByte(host, ':') if i >= 0 { return host[:i] } diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go index 3dc0632..2532392 100644 --- a/server/utils/utils_test.go +++ b/server/utils/utils_test.go @@ -7,7 +7,7 @@ import ( ) func TestTrimHostPort(t *testing.T) { - assert.EqualValues(t, "aa", TrimHostPort([]byte("aa"))) - assert.EqualValues(t, "", TrimHostPort([]byte(":"))) - assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80"))) + assert.EqualValues(t, "aa", TrimHostPort("aa")) + assert.EqualValues(t, "", TrimHostPort(":")) + assert.EqualValues(t, "example.com", TrimHostPort("example.com:80")) } From 6c63b66ce44535cb2fdc3543780e01fa9994e3d5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 20:43:44 +0100 Subject: [PATCH 11/62] Refactor split long functions (#135) we have big functions that handle all stuff ... we should split this into smaler chuncks so we could test them seperate and make clear cuts in what happens where Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/135 --- cmd/main.go | 5 +- server/certificates/certificates.go | 8 +- server/context/context.go | 10 +- server/gitea/client.go | 8 + server/handler.go | 321 ------------------------ server/handler/handler.go | 111 ++++++++ server/handler/handler_custom_domain.go | 71 ++++++ server/handler/handler_raw_domain.go | 67 +++++ server/handler/handler_sub_domain.go | 120 +++++++++ server/{ => handler}/handler_test.go | 7 +- server/{helpers.go => handler/hsts.go} | 2 +- server/handler/try.go | 76 ++++++ server/try.go | 49 ---- server/upstream/domains.go | 16 +- server/upstream/header.go | 28 +++ server/upstream/helper.go | 33 ++- server/upstream/upstream.go | 42 ++-- 17 files changed, 547 insertions(+), 427 deletions(-) delete mode 100644 server/handler.go create mode 100644 server/handler/handler.go create mode 100644 server/handler/handler_custom_domain.go create mode 100644 server/handler/handler_raw_domain.go create mode 100644 server/handler/handler_sub_domain.go rename server/{ => handler}/handler_test.go (85%) rename server/{helpers.go => handler/hsts.go} (96%) create mode 100644 server/handler/try.go delete mode 100644 server/try.go create mode 100644 server/upstream/header.go diff --git a/cmd/main.go b/cmd/main.go index a3a61e1..6ad1aa8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ import ( "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/handler" ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. @@ -88,9 +89,9 @@ func Serve(ctx *cli.Context) error { } // Create handler based on settings - httpsHandler := server.Handler(mainDomainSuffix, rawDomain, + httpsHandler := handler.Handler(mainDomainSuffix, rawDomain, giteaClient, - giteaRoot, rawInfoPage, + rawInfoPage, BlacklistedPaths, allowedCorsDomains, dnsLookupCache, canonicalDomainCache) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 429ab23..42620b1 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -78,8 +78,12 @@ func TLSConfig(mainDomainSuffix string, // DNS not set up, return main certificate to redirect to the docs sni = mainDomainSuffix } else { - _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, mainDomainSuffix, canonicalDomainCache) + targetOpt := &upstream.Options{ + TargetOwner: targetOwner, + TargetRepo: targetRepo, + TargetBranch: targetBranch, + } + _, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache) if !valid { sni = mainDomainSuffix } diff --git a/server/context/context.go b/server/context/context.go index be01df0..481fee2 100644 --- a/server/context/context.go +++ b/server/context/context.go @@ -3,6 +3,8 @@ package context import ( stdContext "context" "net/http" + + "codeberg.org/codeberg/pages/server/utils" ) type Context struct { @@ -42,10 +44,6 @@ func (c *Context) String(raw string, status ...int) { _, _ = c.RespWriter.Write([]byte(raw)) } -func (c *Context) IsMethod(m string) bool { - return c.Req.Method == m -} - func (c *Context) Redirect(uri string, statusCode int) { http.Redirect(c.RespWriter, c.Req, uri, statusCode) } @@ -60,3 +58,7 @@ func (c *Context) Path() string { func (c *Context) Host() string { return c.Req.URL.Host } + +func (c *Context) TrimHostPort() string { + return utils.TrimHostPort(c.Req.Host) +} diff --git a/server/gitea/client.go b/server/gitea/client.go index c63ee21..51647ba 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -45,6 +45,8 @@ type Client struct { sdkClient *gitea.Client responseCache cache.SetGetKey + giteaRoot string + followSymlinks bool supportLFS bool @@ -79,6 +81,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo sdkClient: sdk, responseCache: respCache, + giteaRoot: giteaRoot, + followSymlinks: followSymlinks, supportLFS: supportLFS, @@ -87,6 +91,10 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo }, err } +func (client *Client) ContentWebLink(targetOwner, targetRepo, branch, resource string) string { + return path.Join(client.giteaRoot, targetOwner, targetRepo, "src/branch", branch, resource) +} + func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) if err != nil { diff --git a/server/handler.go b/server/handler.go deleted file mode 100644 index 894cd25..0000000 --- a/server/handler.go +++ /dev/null @@ -1,321 +0,0 @@ -package server - -import ( - "fmt" - "net/http" - "path" - "strings" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/context" - "codeberg.org/codeberg/pages/server/dns" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/upstream" - "codeberg.org/codeberg/pages/server/utils" - "codeberg.org/codeberg/pages/server/version" -) - -const ( - headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" - headerAccessControlAllowMethods = "Access-Control-Allow-Methods" -) - -// Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain string, - giteaClient *gitea.Client, - giteaRoot, rawInfoPage string, - blacklistedPaths, allowedCorsDomains []string, - dnsLookupCache, canonicalDomainCache cache.SetGetKey, -) http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger() - ctx := context.New(w, req) - - ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) - - // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin - ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") - - // Enable browser caching for up to 10 minutes - ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600") - - trimmedHost := utils.TrimHostPort(req.Host) - - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { - ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) - } - - // Block all methods not required for static pages - if !ctx.IsMethod(http.MethodGet) && !ctx.IsMethod(http.MethodHead) && !ctx.IsMethod(http.MethodOptions) { - ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 - ctx.String("Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range blacklistedPaths { - if strings.HasPrefix(ctx.Path(), blacklistedPath) { - html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) - return - } - } - - // Allow CORS for specified domains - allowCors := false - for _, allowedCorsDomain := range allowedCorsDomains { - if strings.EqualFold(trimmedHost, allowedCorsDomain) { - allowCors = true - break - } - } - if allowCors { - ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*") - ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead) - } - - ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 - if ctx.IsMethod(http.MethodOptions) { - ctx.RespWriter.WriteHeader(http.StatusNoContent) - return - } - - // Prepare request information to Gitea - var targetOwner, targetRepo, targetBranch, targetPath string - targetOptions := &upstream.Options{ - TryIndexPages: true, - } - - // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will - // also disallow search indexing and add a Link header to the canonical URL. - // TODO: move into external func to not alert vars indirectly - tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool { - if repo == "" { - log.Debug().Msg("tryBranch: repo is empty") - return false - } - - // Replace "~" to "/" so we can access branch that contains slash character - // Branch name cannot contain "~" so doing this is okay - branch = strings.ReplaceAll(branch, "~", "/") - - // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch) - if branchTimestampResult == nil { - log.Debug().Msg("tryBranch: branch doesn't exist") - return false - } - - // Branch exists, use it - targetRepo = repo - targetPath = path.Join(_path...) - targetBranch = branchTimestampResult.Branch - - targetOptions.BranchTimestamp = branchTimestampResult.Timestamp - - if canonicalLink != "" { - // Hide from search machines & add canonical link - ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex") - ctx.RespWriter.Header().Set("Link", - strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ - "; rel=\"canonical\"", - ) - } - - log.Debug().Msg("tryBranch: true") - return true - } - - log.Debug().Msg("preparations") - if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { - // Serve raw content from RawDomain - log.Debug().Msg("raw domain") - - targetOptions.TryIndexPages = false - targetOptions.ServeRaw = true - - pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - if len(pathElements) < 2 { - // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) - return - } - targetOwner = pathElements[0] - targetRepo = pathElements[1] - - // raw.codeberg.org/example/myrepo/@main/index.html - if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - log.Debug().Msg("raw domain preparations, now trying with specified branch") - if tryBranch(log, - targetRepo, pathElements[2][1:], pathElements[3:], - giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream 1") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - log.Debug().Msg("missing branch info") - html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency) - return - } - - log.Debug().Msg("raw domain preparations, now trying with default branch") - tryBranch(log, - targetRepo, "", pathElements[2:], - giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) - log.Debug().Msg("tryBranch, now trying upstream 2") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - - } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - // Serve pages from subdomains of MainDomainSuffix - log.Debug().Msg("main domain suffix") - - pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix) - targetRepo = pathElements[0] - targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") - - if targetOwner == "www" { - // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? - ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect) - return - } - - // Check if the first directory is a repo with the second directory as a branch - // example.codeberg.page/myrepo/@main/index.html - if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { - if targetRepo == "pages" { - // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... - ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) - return - } - - log.Debug().Msg("main domain preparations, now trying with specified repo & branch") - branch := pathElements[1][1:] - if tryBranch(log, - pathElements[0], branch, pathElements[2:], - "/"+pathElements[0]+"/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream 3") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo), - http.StatusFailedDependency) - } - return - } - - // Check if the first directory is a branch for the "pages" repo - // example.codeberg.page/@main/index.html - if strings.HasPrefix(pathElements[0], "@") { - log.Debug().Msg("main domain preparations, now trying with specified branch") - branch := pathElements[0][1:] - if tryBranch(log, - "pages", branch, pathElements[1:], "/%p") { - log.Debug().Msg("tryBranch, now trying upstream 4") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, "pages", targetBranch, targetPath, - canonicalDomainCache) - } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"), - http.StatusFailedDependency) - } - return - } - - // Check if the first directory is a repo with a "pages" branch - // example.codeberg.page/myrepo/index.html - // example.codeberg.page/pages/... is not allowed here. - log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != "pages" && tryBranch(log, - pathElements[0], "pages", pathElements[1:], "") { - log.Debug().Msg("tryBranch, now trying upstream 5") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - // Try to use the "pages" repo on its default branch - // example.codeberg.page/index.html - log.Debug().Msg("main domain preparations, now trying with default repo/branch") - if tryBranch(log, - "pages", "", pathElements, "") { - log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - // Couldn't find a valid repo/branch - html.ReturnErrorPage(ctx, - fmt.Sprintf("couldn't find a valid repo[%s]/branch[%s]", targetRepo, targetBranch), - http.StatusFailedDependency) - return - } else { - trimmedHostStr := string(trimmedHost) - - // Serve pages from custom domains - targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) - if targetOwner == "" { - html.ReturnErrorPage(ctx, - "could not obtain repo owner from custom domain", - http.StatusFailedDependency) - return - } - - pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - canonicalLink := "" - if strings.HasPrefix(pathElements[0], "@") { - targetBranch = pathElements[0][1:] - pathElements = pathElements[1:] - canonicalLink = "/%p" - } - - // Try to use the given repo on the given branch or the default branch - log.Debug().Msg("custom domain preparations, now trying with details from DNS") - if tryBranch(log, - targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) - if !valid { - html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) - return - } else if canonicalDomain != trimmedHostStr { - // only redirect if the target is also a codeberg page! - targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) - if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect) - return - } - - html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency) - return - } - - log.Debug().Msg("tryBranch, now trying upstream 7") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency) - return - } - } -} diff --git a/server/handler/handler.go b/server/handler/handler.go new file mode 100644 index 0000000..b42751e --- /dev/null +++ b/server/handler/handler.go @@ -0,0 +1,111 @@ +package handler + +import ( + "net/http" + "strings" + + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/version" +) + +const ( + headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" + headerAccessControlAllowMethods = "Access-Control-Allow-Methods" +) + +// Handler handles a single HTTP request to the web server. +func Handler(mainDomainSuffix, rawDomain string, + giteaClient *gitea.Client, + rawInfoPage string, + blacklistedPaths, allowedCorsDomains []string, + dnsLookupCache, canonicalDomainCache cache.SetGetKey, +) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger() + ctx := context.New(w, req) + + ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) + + // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin + ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") + + // Enable browser caching for up to 10 minutes + ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600") + + trimmedHost := ctx.TrimHostPort() + + // Add HSTS for RawDomain and MainDomainSuffix + if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) + } + + // Handle all http methods + ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) + switch ctx.Req.Method { + case http.MethodOptions: + // return Allow header + ctx.RespWriter.WriteHeader(http.StatusNoContent) + return + case http.MethodGet, + http.MethodHead: + // end switch case and handle allowed requests + break + default: + // Block all methods not required for static pages + ctx.String("Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Block blacklisted paths (like ACME challenges) + for _, blacklistedPath := range blacklistedPaths { + if strings.HasPrefix(ctx.Path(), blacklistedPath) { + html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) + return + } + } + + // Allow CORS for specified domains + allowCors := false + for _, allowedCorsDomain := range allowedCorsDomains { + if strings.EqualFold(trimmedHost, allowedCorsDomain) { + allowCors = true + break + } + } + if allowCors { + ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*") + ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead) + } + + // Prepare request information to Gitea + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") + + if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { + log.Debug().Msg("raw domain request detecded") + handleRaw(log, ctx, giteaClient, + mainDomainSuffix, rawInfoPage, + trimmedHost, + pathElements, + canonicalDomainCache) + } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + log.Debug().Msg("subdomain request detecded") + handleSubDomain(log, ctx, giteaClient, + mainDomainSuffix, + trimmedHost, + pathElements, + canonicalDomainCache) + } else { + log.Debug().Msg("custom domain request detecded") + handleCustomDomain(log, ctx, giteaClient, + mainDomainSuffix, + trimmedHost, + pathElements, + dnsLookupCache, canonicalDomainCache) + } + } +} diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go new file mode 100644 index 0000000..bec3b46 --- /dev/null +++ b/server/handler/handler_custom_domain.go @@ -0,0 +1,71 @@ +package handler + +import ( + "net/http" + "path" + "strings" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/upstream" + "github.com/rs/zerolog" +) + +func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix string, + trimmedHost string, + pathElements []string, + dnsLookupCache, canonicalDomainCache cache.SetGetKey, +) { + // Serve pages from custom domains + targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, string(mainDomainSuffix), dnsLookupCache) + if targetOwner == "" { + html.ReturnErrorPage(ctx, + "could not obtain repo owner from custom domain", + http.StatusFailedDependency) + return + } + + pathParts := pathElements + canonicalLink := false + if strings.HasPrefix(pathElements[0], "@") { + targetBranch = pathElements[0][1:] + pathParts = pathElements[1:] + canonicalLink = true + } + + // Try to use the given repo on the given branch or the default branch + log.Debug().Msg("custom domain preparations, now trying with details from DNS") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: targetRepo, + TargetBranch: targetBranch, + TargetPath: path.Join(pathParts...), + }, canonicalLink); works { + canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, string(mainDomainSuffix), canonicalDomainCache) + if !valid { + html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) + return + } else if canonicalDomain != trimmedHost { + // only redirect if the target is also a codeberg page! + targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) + if targetOwner != "" { + ctx.Redirect("https://"+canonicalDomain+string(targetOpt.TargetPath), http.StatusTemporaryRedirect) + return + } + + html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency) + return + } + + log.Debug().Msg("tryBranch, now trying upstream 7") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + + html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency) +} diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go new file mode 100644 index 0000000..5e974da --- /dev/null +++ b/server/handler/handler_raw_domain.go @@ -0,0 +1,67 @@ +package handler + +import ( + "fmt" + "net/http" + "path" + "strings" + + "github.com/rs/zerolog" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/upstream" +) + +func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix, rawInfoPage string, + trimmedHost string, + pathElements []string, + canonicalDomainCache cache.SetGetKey, +) { + // Serve raw content from RawDomain + log.Debug().Msg("raw domain") + + if len(pathElements) < 2 { + // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required + ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) + return + } + + // raw.codeberg.org/example/myrepo/@main/index.html + if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { + log.Debug().Msg("raw domain preparations, now trying with specified branch") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + ServeRaw: true, + TargetOwner: pathElements[0], + TargetRepo: pathElements[1], + TargetBranch: pathElements[2][1:], + TargetPath: path.Join(pathElements[3:]...), + }, true); works { + log.Trace().Msg("tryUpstream: serve raw domain with specified branch") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + log.Debug().Msg("missing branch info") + html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency) + return + } + + log.Debug().Msg("raw domain preparations, now trying with default branch") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: false, + ServeRaw: true, + TargetOwner: pathElements[0], + TargetRepo: pathElements[1], + TargetPath: path.Join(pathElements[2:]...), + }, true); works { + log.Trace().Msg("tryUpstream: serve raw domain with default branch") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + } else { + html.ReturnErrorPage(ctx, + fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusNotFound) + } +} diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go new file mode 100644 index 0000000..df42d61 --- /dev/null +++ b/server/handler/handler_sub_domain.go @@ -0,0 +1,120 @@ +package handler + +import ( + "fmt" + "net/http" + "path" + "strings" + + "github.com/rs/zerolog" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/upstream" +) + +func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix string, + trimmedHost string, + pathElements []string, + canonicalDomainCache cache.SetGetKey, +) { + // Serve pages from subdomains of MainDomainSuffix + log.Debug().Msg("main domain suffix") + + targetOwner := strings.TrimSuffix(trimmedHost, mainDomainSuffix) + targetRepo := pathElements[0] + + if targetOwner == "www" { + // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? + ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect) + return + } + + // Check if the first directory is a repo with the second directory as a branch + // example.codeberg.page/myrepo/@main/index.html + if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { + if targetRepo == "pages" { + // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) + return + } + + log.Debug().Msg("main domain preparations, now trying with specified repo & branch") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: pathElements[0], + TargetBranch: pathElements[1][1:], + TargetPath: path.Join(pathElements[2:]...), + }, true); works { + log.Trace().Msg("tryUpstream: serve with specified repo and branch") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + } else { + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency) + } + return + } + + // Check if the first directory is a branch for the "pages" repo + // example.codeberg.page/@main/index.html + if strings.HasPrefix(pathElements[0], "@") { + log.Debug().Msg("main domain preparations, now trying with specified branch") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: "pages", + TargetBranch: pathElements[0][1:], + TargetPath: path.Join(pathElements[1:]...), + }, true); works { + log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + } else { + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency) + } + return + } + + // Check if the first directory is a repo with a "pages" branch + // example.codeberg.page/myrepo/index.html + // example.codeberg.page/pages/... is not allowed here. + log.Debug().Msg("main domain preparations, now trying with specified repo") + if pathElements[0] != "pages" { + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: pathElements[0], + TargetBranch: "pages", + TargetPath: path.Join(pathElements[1:]...), + }, false); works { + log.Debug().Msg("tryBranch, now trying upstream 5") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + } + + // Try to use the "pages" repo on its default branch + // example.codeberg.page/index.html + log.Debug().Msg("main domain preparations, now trying with default repo/branch") + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: "pages", + TargetPath: path.Join(pathElements...), + }, false); works { + log.Debug().Msg("tryBranch, now trying upstream 6") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + + // Couldn't find a valid repo/branch + html.ReturnErrorPage(ctx, + fmt.Sprintf("couldn't find a valid repo[%s]", targetRepo), + http.StatusFailedDependency) +} diff --git a/server/handler_test.go b/server/handler/handler_test.go similarity index 85% rename from server/handler_test.go rename to server/handler/handler_test.go index c0aca14..f5538c9 100644 --- a/server/handler_test.go +++ b/server/handler/handler_test.go @@ -1,4 +1,4 @@ -package server +package handler import ( "net/http/httptest" @@ -11,12 +11,11 @@ import ( ) func TestHandlerPerformance(t *testing.T) { - giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false) + giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false) testHandler := Handler( "codeberg.page", "raw.codeberg.org", giteaClient, - giteaRoot, "https://docs.codeberg.org/pages/raw-content/", + "https://docs.codeberg.org/pages/raw-content/", []string{"/.well-known/acme-challenge/"}, []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, cache.NewKeyValueCache(), diff --git a/server/helpers.go b/server/handler/hsts.go similarity index 96% rename from server/helpers.go rename to server/handler/hsts.go index 7c898cd..1ab73ae 100644 --- a/server/helpers.go +++ b/server/handler/hsts.go @@ -1,4 +1,4 @@ -package server +package handler import ( "strings" diff --git a/server/handler/try.go b/server/handler/try.go new file mode 100644 index 0000000..b9adb7b --- /dev/null +++ b/server/handler/try.go @@ -0,0 +1,76 @@ +package handler + +import ( + "net/http" + "strings" + + "github.com/rs/zerolog" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/upstream" +) + +// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. +func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix, trimmedHost string, + options *upstream.Options, + canonicalDomainCache cache.SetGetKey, +) { + // check if a canonical domain exists on a request on MainDomain + if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", string(mainDomainSuffix), canonicalDomainCache) + if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { + canonicalPath := ctx.Req.RequestURI + if options.TargetRepo != "pages" { + path := strings.SplitN(canonicalPath, "/", 3) + if len(path) >= 3 { + canonicalPath = "/" + path[2] + } + } + ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect) + return + } + } + + // add host for debugging + options.Host = string(trimmedHost) + + // Try to request the file from the Gitea API + if !options.Upstream(ctx, giteaClient) { + html.ReturnErrorPage(ctx, "", ctx.StatusCode) + } +} + +// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, +// it will also disallow search indexing and add a Link header to the canonical URL. +func tryBranch(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, + targetOptions *upstream.Options, canonicalLink bool, +) (*upstream.Options, bool) { + if targetOptions.TargetOwner == "" || targetOptions.TargetRepo == "" { + log.Debug().Msg("tryBranch: owner or repo is empty") + return nil, false + } + + // Replace "~" to "/" so we can access branch that contains slash character + // Branch name cannot contain "~" so doing this is okay + targetOptions.TargetBranch = strings.ReplaceAll(targetOptions.TargetBranch, "~", "/") + + // Check if the branch exists, otherwise treat it as a file path + branchExist, _ := targetOptions.GetBranchTimestamp(giteaClient) + if !branchExist { + log.Debug().Msg("tryBranch: branch doesn't exist") + return nil, false + } + + if canonicalLink { + // Hide from search machines & add canonical link + ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex") + ctx.RespWriter.Header().Set("Link", targetOptions.ContentWebLink(giteaClient)+"; rel=\"canonical\"") + } + + log.Debug().Msg("tryBranch: true") + return targetOptions, true +} diff --git a/server/try.go b/server/try.go deleted file mode 100644 index 135c1e0..0000000 --- a/server/try.go +++ /dev/null @@ -1,49 +0,0 @@ -package server - -import ( - "net/http" - "strings" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/context" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/upstream" -) - -// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, - mainDomainSuffix, trimmedHost string, - - targetOptions *upstream.Options, - targetOwner, targetRepo, targetBranch, targetPath string, - - canonicalDomainCache cache.SetGetKey, -) { - // check if a canonical domain exists on a request on MainDomain - if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) - if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { - canonicalPath := ctx.Req.RequestURI - if targetRepo != "pages" { - path := strings.SplitN(canonicalPath, "/", 3) - if len(path) >= 3 { - canonicalPath = "/" + path[2] - } - } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect) - return - } - } - - targetOptions.TargetOwner = targetOwner - targetOptions.TargetRepo = targetRepo - targetOptions.TargetBranch = targetBranch - targetOptions.TargetPath = targetPath - targetOptions.Host = string(trimmedHost) - - // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaClient) { - html.ReturnErrorPage(ctx, "", ctx.StatusCode) - } -} diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 6ad6506..0e29673 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -16,12 +16,12 @@ var canonicalDomainCacheTimeout = 15 * time.Minute const canonicalDomainConfig = ".domains" // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( domains []string valid bool ) - if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { + if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { domains = cachedValue.([]string) for _, domain := range domains { if domain == actualDomain { @@ -30,7 +30,7 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta } } } else { - body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig) + body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig) if err == nil { for _, domain := range strings.Split(string(body), "\n") { domain = strings.ToLower(domain) @@ -45,16 +45,16 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta } } } else { - log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo) + log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) } - domains = append(domains, targetOwner+mainDomainSuffix) + domains = append(domains, o.TargetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { valid = true } - if targetRepo != "" && targetRepo != "pages" { - domains[len(domains)-1] += "/" + targetRepo + if o.TargetRepo != "" && o.TargetRepo != "pages" { + domains[len(domains)-1] += "/" + o.TargetRepo } - _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout) + _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout) } return domains[0], valid } diff --git a/server/upstream/header.go b/server/upstream/header.go new file mode 100644 index 0000000..9575a3f --- /dev/null +++ b/server/upstream/header.go @@ -0,0 +1,28 @@ +package upstream + +import ( + "net/http" + "time" + + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" +) + +// setHeader set values to response header +func (o *Options) setHeader(ctx *context.Context, header http.Header) { + if eTag := header.Get(gitea.ETagHeader); eTag != "" { + ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag) + } + if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" { + ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator) + } + if length := header.Get(gitea.ContentLengthHeader); length != "" { + ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length) + } + if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime) + } else { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime) + } + ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) +} diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 6bc23c8..428976b 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -2,35 +2,46 @@ package upstream import ( "errors" + "fmt" "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/gitea" ) -// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch -// (or nil if the branch doesn't exist) -func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp { - log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger() +// GetBranchTimestamp finds the default branch (if branch is "") and save branch and it's last modification time to Options +func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) { + log := log.With().Strs("BranchInfo", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch}).Logger() - if len(branch) == 0 { + if len(o.TargetBranch) == 0 { // Get default branch - defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) + defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(o.TargetOwner, o.TargetRepo) if err != nil { log.Err(err).Msg("Could't fetch default branch from repository") - return nil + return false, err } log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch) - branch = defaultBranch + o.TargetBranch = defaultBranch } - timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch) + timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch) if err != nil { if !errors.Is(err, gitea.ErrorNotFound) { log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch") } - return nil + return false, err } + + if timestamp == nil || timestamp.Branch == "" { + return false, fmt.Errorf("empty response") + } + log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp) - return timestamp + o.BranchTimestamp = timestamp.Timestamp + o.TargetBranch = timestamp.Branch + return true, nil +} + +func (o *Options) ContentWebLink(giteaClient *gitea.Client) string { + return giteaClient.ContentWebLink(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath) + "; rel=\"canonical\"" } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index d37c35e..b76b8e6 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -34,10 +34,10 @@ var upstreamNotFoundPages = []string{ // Options provides various options for the upstream request. type Options struct { - TargetOwner, - TargetRepo, - TargetBranch, - TargetPath, + TargetOwner string + TargetRepo string + TargetBranch string + TargetPath string // Used for debugging purposes. Host string @@ -62,16 +62,22 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin // Check if the branch exists and when it was modified if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) - - if branch == nil || branch.Branch == "" { + branchExist, err := o.GetBranchTimestamp(giteaClient) + // handle 404 + if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist { html.ReturnErrorPage(ctx, - fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch), + fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo), + http.StatusNotFound) + return true + } + + // handle unexpected errors + if err != nil { + html.ReturnErrorPage(ctx, + fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err), http.StatusFailedDependency) return true } - o.TargetBranch = branch.Branch - o.BranchTimestamp = branch.Timestamp } // Check if the browser has a cached version @@ -172,21 +178,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin } // Set ETag & MIME - if eTag := header.Get(gitea.ETagHeader); eTag != "" { - ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag) - } - if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" { - ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator) - } - if length := header.Get(gitea.ContentLengthHeader); length != "" { - ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length) - } - if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw { - ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime) - } else { - ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime) - } - ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) + o.setHeader(ctx, header) log.Debug().Msg("Prepare response") From b6d0a04b217f34705cb8cb6ba89107c76a87bfb1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 21:04:34 +0100 Subject: [PATCH 12/62] refactor: rm not needed type conversion --- server/handler/handler_custom_domain.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index bec3b46..5ea7649 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -21,7 +21,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) { // Serve pages from custom domains - targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, string(mainDomainSuffix), dnsLookupCache) + targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache) if targetOwner == "" { html.ReturnErrorPage(ctx, "could not obtain repo owner from custom domain", @@ -46,13 +46,13 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g TargetBranch: targetBranch, TargetPath: path.Join(pathParts...), }, canonicalLink); works { - canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, string(mainDomainSuffix), canonicalDomainCache) + canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! - targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) + targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache) if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+string(targetOpt.TargetPath), http.StatusTemporaryRedirect) return From aa90356f0a4d48556a749e4e3677aa5e7641c4d8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 21:10:16 +0100 Subject: [PATCH 13/62] use a const for defaultPagesRepo --- server/handler/handler_sub_domain.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index df42d61..8190cd9 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -15,6 +15,8 @@ import ( "codeberg.org/codeberg/pages/server/upstream" ) +const defaultPagesRepo = "pages" + func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix string, trimmedHost string, @@ -36,7 +38,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Check if the first directory is a repo with the second directory as a branch // example.codeberg.page/myrepo/@main/index.html if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { - if targetRepo == "pages" { + if targetRepo == defaultPagesRepo { // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) return @@ -60,14 +62,14 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite return } - // Check if the first directory is a branch for the "pages" repo + // Check if the first directory is a branch for the defaultPagesRepo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { log.Debug().Msg("main domain preparations, now trying with specified branch") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, - TargetRepo: "pages", + TargetRepo: defaultPagesRepo, TargetBranch: pathElements[0][1:], TargetPath: path.Join(pathElements[1:]...), }, true); works { @@ -81,16 +83,16 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite return } - // Check if the first directory is a repo with a "pages" branch + // Check if the first directory is a repo with a defaultPagesRepo branch // example.codeberg.page/myrepo/index.html // example.codeberg.page/pages/... is not allowed here. log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != "pages" { + if pathElements[0] != defaultPagesRepo { if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, TargetRepo: pathElements[0], - TargetBranch: "pages", + TargetBranch: defaultPagesRepo, TargetPath: path.Join(pathElements[1:]...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 5") @@ -99,13 +101,13 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite } } - // Try to use the "pages" repo on its default branch + // Try to use the defaultPagesRepo on its default branch // example.codeberg.page/index.html log.Debug().Msg("main domain preparations, now trying with default repo/branch") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, - TargetRepo: "pages", + TargetRepo: defaultPagesRepo, TargetPath: path.Join(pathElements...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 6") From c827a28dd829974af387665dc7e0db92283ed930 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 21:13:13 +0100 Subject: [PATCH 14/62] defaultPagesBranch --- server/handler/handler_sub_domain.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 8190cd9..59d1bbc 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -16,6 +16,7 @@ import ( ) const defaultPagesRepo = "pages" +const defaultPagesBranch = "pages" func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix string, @@ -92,7 +93,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TryIndexPages: true, TargetOwner: targetOwner, TargetRepo: pathElements[0], - TargetBranch: defaultPagesRepo, + TargetBranch: defaultPagesBranch, TargetPath: path.Join(pathElements[1:]...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 5") From 4565481643dfd9014a674cd5a182a6c42e820fe0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 21:16:11 +0100 Subject: [PATCH 15/62] refactor: finish use default const for defaultPagesBranch and defaultPagesRepo --- server/handler/handler.go | 2 ++ server/handler/handler_sub_domain.go | 3 --- server/handler/try.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/handler/handler.go b/server/handler/handler.go index b42751e..1a0a285 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -16,6 +16,8 @@ import ( const ( headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" headerAccessControlAllowMethods = "Access-Control-Allow-Methods" + defaultPagesRepo = "pages" + defaultPagesBranch = "pages" ) // Handler handles a single HTTP request to the web server. diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 59d1bbc..73177b6 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -15,9 +15,6 @@ import ( "codeberg.org/codeberg/pages/server/upstream" ) -const defaultPagesRepo = "pages" -const defaultPagesBranch = "pages" - func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix string, trimmedHost string, diff --git a/server/handler/try.go b/server/handler/try.go index b9adb7b..bae3c44 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -24,7 +24,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", string(mainDomainSuffix), canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := ctx.Req.RequestURI - if options.TargetRepo != "pages" { + if options.TargetRepo != defaultPagesRepo { path := strings.SplitN(canonicalPath, "/", 3) if len(path) >= 3 { canonicalPath = "/" + path[2] From 3c61a39864f087f1bb6305dbcd512bdd7ffb72fa Mon Sep 17 00:00:00 2001 From: Gusted Date: Sat, 12 Nov 2022 22:25:20 +0100 Subject: [PATCH 16/62] Enable http/2 support (#137) As per [the documentation](https://pkg.go.dev/net/http#Serve), it doesn't enable HTTP2 by-default, unless we enable it via the `NextProtos` option. Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/137 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/certificates/certificates.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 42620b1..11cf0df 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -117,6 +117,7 @@ func TLSConfig(mainDomainSuffix string, }, PreferServerCipherSuites: true, NextProtos: []string{ + "h2", "http/1.1", tlsalpn01.ACMETLS1Protocol, }, From f2f943c0d800c513025a2fecfb1a48729052fdc1 Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 15 Nov 2022 16:15:11 +0100 Subject: [PATCH 17/62] Remove unnecessary conversion (#139) - Remove unnecessary type conversion. - Enforce via CI Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/139 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- .golangci.yml | 20 ++++++++++ cmd/main.go | 2 +- integration/get_test.go | 2 +- integration/main_test.go | 2 +- server/certificates/certificates.go | 50 +++++++++++++------------ server/database/setup.go | 2 +- server/gitea/cache.go | 5 +-- server/handler/handler.go | 2 +- server/handler/handler_custom_domain.go | 2 +- server/handler/handler_sub_domain.go | 2 +- server/handler/try.go | 8 ++-- server/setup.go | 4 +- server/upstream/helper.go | 2 +- server/upstream/upstream.go | 4 +- 14 files changed, 64 insertions(+), 43 deletions(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6d9b95a --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,20 @@ +linters-settings: + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - importShadow + - ifElseChain + - hugeParam + +linters: + enable: + - unconvert + - gocritic + +run: + timeout: 5m diff --git a/cmd/main.go b/cmd/main.go index 6ad1aa8..b72013a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -65,7 +65,7 @@ func Serve(ctx *cli.Context) error { } allowedCorsDomains := AllowedCorsDomains - if len(rawDomain) != 0 { + if rawDomain != "" { allowedCorsDomains = append(allowedCorsDomains, rawDomain) } diff --git a/integration/get_test.go b/integration/get_test.go index 8794651..81d8488 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -124,7 +124,7 @@ func TestLFSSupport(t *testing.T) { func TestGetOptions(t *testing.T) { log.Println("=== TestGetOptions ===") - req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", nil) + req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", http.NoBody) resp, err := getTestHTTPSClient().Do(req) assert.NoError(t, err) if !assert.NotNil(t, resp) { diff --git a/integration/main_test.go b/integration/main_test.go index 06d553f..406b33a 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -28,7 +28,7 @@ func TestMain(m *testing.M) { time.Sleep(10 * time.Second) - os.Exit(m.Run()) + m.Run() } func startServer(ctx context.Context) error { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 11cf0df..8af4be5 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -53,17 +53,19 @@ func TLSConfig(mainDomainSuffix string, if info.SupportedProtos != nil { for _, proto := range info.SupportedProtos { - if proto == tlsalpn01.ACMETLS1Protocol { - challenge, ok := challengeCache.Get(sni) - if !ok { - return nil, errors.New("no challenge for this domain") - } - cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) - if err != nil { - return nil, err - } - return cert, nil + if proto != tlsalpn01.ACMETLS1Protocol { + continue } + + challenge, ok := challengeCache.Get(sni) + if !ok { + return nil, errors.New("no challenge for this domain") + } + cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) + if err != nil { + return nil, err + } + return cert, nil } } @@ -195,7 +197,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { // parse certificate from database - res, err := certDB.Get(string(sni)) + res, err := certDB.Get(sni) if err != nil { panic(err) // TODO: no panic } @@ -216,7 +218,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi } // renew certificates 7 days before they expire - if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(7 * 24 * time.Hour)) { + if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) { // TODO: add ValidUntil to custom res struct if res.CSR != nil && len(res.CSR) > 0 { // CSR stores the time when the renewal shall be tried again @@ -227,9 +229,9 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi } go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { - log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err) + log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err) } })() } @@ -262,7 +264,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re defer obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase), nil } // request actual cert @@ -305,12 +307,12 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) if err := keyDatabase.Put(name, renew); err != nil { - return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err + return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err } return tlsCertificate, nil } } - return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err + return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err } log.Debug().Msgf("Obtained certificate for %v", domains) @@ -408,7 +410,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits - mainCertBytes, err := certDB.Get(string(mainDomainSuffix)) + mainCertBytes, err := certDB.Get(mainDomainSuffix) if err != nil { return fmt.Errorf("cert database is not working") } @@ -452,7 +454,7 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co } if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only") } @@ -479,7 +481,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - if err != nil || !tlsCertificates[0].NotAfter.After(now) { + if err != nil || tlsCertificates[0].NotAfter.Before(now) { err := certDB.Delete(string(key)) if err != nil { log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key)) @@ -501,18 +503,18 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } // update main cert - res, err := certDB.Get(string(mainDomainSuffix)) + res, err := certDB.Get(mainDomainSuffix) if err != nil { log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix) } else if res == nil { - log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix)) + log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", mainDomainSuffix) } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) // renew main certificate 30 days before it expires - if !tlsCertificates[0].NotAfter.After(time.Now().Add(30 * 24 * time.Hour)) { + if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) { go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { log.Error().Err(err).Msg("Couldn't renew certificate for main domain") } diff --git a/server/database/setup.go b/server/database/setup.go index 1c5a0af..097c63e 100644 --- a/server/database/setup.go +++ b/server/database/setup.go @@ -44,7 +44,7 @@ func (p aDB) Get(name string) (*certificate.Resource, error) { if resBytes == nil { return nil, nil } - if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil { + if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil { return nil, err } return cert, nil diff --git a/server/gitea/cache.go b/server/gitea/cache.go index b11a370..85cbcde 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -42,9 +42,8 @@ func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } -func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) { - header := make(http.Header) - var statusCode int +func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) { + header = make(http.Header) if f.Exists { statusCode = http.StatusOK diff --git a/server/handler/handler.go b/server/handler/handler.go index 1a0a285..78301e9 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -28,7 +28,7 @@ func Handler(mainDomainSuffix, rawDomain string, dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger() + log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() ctx := context.New(w, req) ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 5ea7649..2f98085 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -54,7 +54,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g // only redirect if the target is also a codeberg page! targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache) if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(targetOpt.TargetPath), http.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+targetOpt.TargetPath, http.StatusTemporaryRedirect) return } diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 73177b6..1d769d4 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -29,7 +29,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite if targetOwner == "www" { // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? - ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect) + ctx.Redirect("https://"+mainDomainSuffix[1:]+ctx.Path(), http.StatusPermanentRedirect) return } diff --git a/server/handler/try.go b/server/handler/try.go index bae3c44..5a09b91 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -21,8 +21,8 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", string(mainDomainSuffix), canonicalDomainCache) - if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { + canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache) + if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) { canonicalPath := ctx.Req.RequestURI if options.TargetRepo != defaultPagesRepo { path := strings.SplitN(canonicalPath, "/", 3) @@ -35,8 +35,8 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, } } - // add host for debugging - options.Host = string(trimmedHost) + // Add host for debugging. + options.Host = trimmedHost // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient) { diff --git a/server/setup.go b/server/setup.go index e7194ed..282e692 100644 --- a/server/setup.go +++ b/server/setup.go @@ -15,13 +15,13 @@ func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFu return func(w http.ResponseWriter, req *http.Request) { ctx := context.New(w, req) if strings.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + string(strings.TrimPrefix(ctx.Path(), challengePath))) + challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) if !ok || challenge == nil { ctx.String("no challenge for this token", http.StatusNotFound) } ctx.String(challenge.(string)) } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.Path()), http.StatusMovedPermanently) + ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently) } } } diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 428976b..a84d4f0 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -13,7 +13,7 @@ import ( func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) { log := log.With().Strs("BranchInfo", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch}).Logger() - if len(o.TargetBranch) == 0 { + if o.TargetBranch == "" { // Get default branch defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(o.TargetOwner, o.TargetRepo) if err != nil { diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index b76b8e6..7c3c848 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -82,8 +82,8 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin // Check if the browser has a cached version if ctx.Response() != nil { - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil { - if !ifModifiedSince.Before(o.BranchTimestamp) { + if ifModifiedSince, err := time.Parse(time.RFC1123, ctx.Response().Header.Get(headerIfModifiedSince)); err == nil { + if ifModifiedSince.After(o.BranchTimestamp) { ctx.RespWriter.WriteHeader(http.StatusNotModified) log.Trace().Msg("check response against last modified: valid") return true From caeb1a4acb0d35c78988fcd4e30d9ac51847f13a Mon Sep 17 00:00:00 2001 From: jklippel Date: Tue, 22 Nov 2022 21:26:10 +0000 Subject: [PATCH 18/62] Return a 404 if there is no repository (#141) If no repository is found the user expects a 404 status code instead of a dependency failed status code (as it was before). Signed-off-by: Jan Klippel Fixes: https://codeberg.org/Codeberg/Community/issues/809 Co-authored-by: Jan Klippel Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/141 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: jklippel Co-committed-by: jklippel --- server/handler/handler_sub_domain.go | 4 ++-- server/handler/handler_test.go | 35 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 1d769d4..2a75e9f 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -115,6 +115,6 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Couldn't find a valid repo/branch html.ReturnErrorPage(ctx, - fmt.Sprintf("couldn't find a valid repo[%s]", targetRepo), - http.StatusFailedDependency) + fmt.Sprintf("could not find a valid repository[%s]", targetRepo), + http.StatusNotFound) } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index f5538c9..626564a 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -23,26 +23,27 @@ func TestHandlerPerformance(t *testing.T) { ) testCase := func(uri string, status int) { - req := httptest.NewRequest("GET", uri, nil) - w := httptest.NewRecorder() + t.Run(uri, func(t *testing.T) { + req := httptest.NewRequest("GET", uri, nil) + w := httptest.NewRecorder() - log.Printf("Start: %v\n", time.Now()) - start := time.Now() - testHandler(w, req) - end := time.Now() - log.Printf("Done: %v\n", time.Now()) + log.Printf("Start: %v\n", time.Now()) + start := time.Now() + testHandler(w, req) + end := time.Now() + log.Printf("Done: %v\n", time.Now()) - resp := w.Result() + resp := w.Result() - if resp.StatusCode != status { - t.Errorf("request failed with status code %d", resp.StatusCode) - } else { - t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds()) - } + if resp.StatusCode != status { + t.Errorf("request failed with status code %d", resp.StatusCode) + } else { + t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds()) + } + }) } - testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200 - testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200 - testCase("https://example.momar.xyz/", 424) // TODO: expect 200 - testCase("https://codeberg.page/", 424) // TODO: expect 200 + testCase("https://mondstern.codeberg.page/", 404) // TODO: expect 200 + testCase("https://codeberg.page/", 404) // TODO: expect 200 + testCase("https://example.momar.xyz/", 424) } From 5e72753e916d1076cfd772d3d6c7f86474c6c87a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 22 Nov 2022 22:30:53 +0100 Subject: [PATCH 19/62] ci: "docker-tag" use `tags` --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 20254fe..7e6d694 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -102,7 +102,7 @@ pipeline: registry: codeberg.org dockerfile: Dockerfile repo: codeberg.org/codeberg/pages-server - tag: [ latest, "${CI_COMMIT_TAG}" ] + tags: [ latest, "${CI_COMMIT_TAG}" ] username: from_secret: bot_user password: From dcf03fc078228a286384757c3501ab5df5306062 Mon Sep 17 00:00:00 2001 From: crapStone Date: Fri, 2 Dec 2022 15:25:25 +0000 Subject: [PATCH 20/62] Fix error page (#144) Co-authored-by: crapStone Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/144 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- html/error.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/html/error.go b/html/error.go index 826c42b..ac222c4 100644 --- a/html/error.go +++ b/html/error.go @@ -1,6 +1,7 @@ package html import ( + "html/template" "net/http" "strconv" "strings" @@ -39,7 +40,8 @@ func errorMessage(statusCode int) string { // TODO: use template engine func errorBody(statusCode int) string { - return strings.ReplaceAll(NotFoundPage, - "%status%", - strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) + return template.HTMLEscapeString( + strings.ReplaceAll(NotFoundPage, + "%status%", + strconv.Itoa(statusCode)+" "+errorMessage(statusCode))) } From 9d769aeee7bdbdce03bbb0f6fb213d4b7d45a701 Mon Sep 17 00:00:00 2001 From: crapStone Date: Sun, 4 Dec 2022 21:24:58 +0000 Subject: [PATCH 21/62] Fix error page generation (#145) Co-authored-by: crapStone Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/145 Reviewed-by: Gusted Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- Justfile | 4 ++-- html/error.go | 31 +++++++++++++++++-------------- html/error_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 html/error_test.go diff --git a/Justfile b/Justfile index f7ea414..7d72fe8 100644 --- a/Justfile +++ b/Justfile @@ -38,10 +38,10 @@ tool-gofumpt: fi test: - go test -race codeberg.org/codeberg/pages/server/... + go test -race codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ test-run TEST: - go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... + go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ integration: go test -race -tags integration codeberg.org/codeberg/pages/integration/... diff --git a/html/error.go b/html/error.go index ac222c4..206b123 100644 --- a/html/error.go +++ b/html/error.go @@ -15,16 +15,27 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") ctx.RespWriter.WriteHeader(statusCode) - if msg == "" { - msg = errorBody(statusCode) - } else { - // TODO: use template engine - msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) - } + msg = generateResponse(msg, statusCode) _, _ = ctx.RespWriter.Write([]byte(msg)) } +// TODO: use template engine +func generateResponse(msg string, statusCode int) string { + if msg == "" { + msg = strings.ReplaceAll(NotFoundPage, + "%status%", + strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) + } else { + msg = strings.ReplaceAll( + strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)), + "%status%", + http.StatusText(statusCode)) + } + + return msg +} + func errorMessage(statusCode int) string { message := http.StatusText(statusCode) @@ -37,11 +48,3 @@ func errorMessage(statusCode int) string { return message } - -// TODO: use template engine -func errorBody(statusCode int) string { - return template.HTMLEscapeString( - strings.ReplaceAll(NotFoundPage, - "%status%", - strconv.Itoa(statusCode)+" "+errorMessage(statusCode))) -} diff --git a/html/error_test.go b/html/error_test.go new file mode 100644 index 0000000..f5da08c --- /dev/null +++ b/html/error_test.go @@ -0,0 +1,38 @@ +package html + +import ( + "net/http" + "strings" + "testing" +) + +func TestValidMessage(t *testing.T) { + testString := "requested blacklisted path" + statusCode := http.StatusForbidden + + expected := strings.ReplaceAll( + strings.ReplaceAll(ErrorPage, "%message%", testString), + "%status%", + http.StatusText(statusCode)) + actual := generateResponse(testString, statusCode) + + if expected != actual { + t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual) + } +} + +func TestMessageWithHtml(t *testing.T) { + testString := `abc Date: Wed, 4 Jan 2023 04:51:27 +0000 Subject: [PATCH 22/62] Safely get certificate's leaf (#150) - It's not guaranteed that `tls.X509KeyPair` will set `c.Leaf`. - This patch fixes this by using a wrapper that parses the leaf certificate(in bytes) if `c.Leaf` wasn't set. - Resolves #149 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/150 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/certificates/certificates.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 8af4be5..e3cdbfb 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -303,7 +303,11 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains) if renew != nil && renew.CertURL != "" { tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey) - if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { + if err != nil { + return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err + } + leaf, err := leaf(&tlsCertificate) + if err == nil && leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) if err := keyDatabase.Put(name, renew); err != nil { @@ -529,3 +533,12 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } } } + +// leaf returns the parsed leaf certificate, either from c.leaf or by parsing +// the corresponding c.Certificate[0]. +func leaf(c *tls.Certificate) (*x509.Certificate, error) { + if c.Leaf != nil { + return c.Leaf, nil + } + return x509.ParseCertificate(c.Certificate[0]) +} From f7fad2a5ae0e4942848604a77199f4f1f8a1c3b4 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 4 Jan 2023 06:08:06 +0100 Subject: [PATCH 23/62] Integration Tests use https://codeberg.org/cb_pages_tests --- integration/get_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/get_test.go b/integration/get_test.go index 81d8488..f13ce8b 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -31,7 +31,7 @@ func TestGetRedirect(t *testing.T) { func TestGetContent(t *testing.T) { log.Println("=== TestGetContent ===") // test get image - resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg") + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg") assert.NoError(t, err) if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { t.FailNow() @@ -42,7 +42,7 @@ func TestGetContent(t *testing.T) { assert.Len(t, resp.Header.Get("ETag"), 42) // specify branch - resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/") + resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/pag/@master/") assert.NoError(t, err) if !assert.NotNil(t, resp) { t.FailNow() @@ -53,7 +53,7 @@ func TestGetContent(t *testing.T) { assert.Len(t, resp.Header.Get("ETag"), 44) // access branch name contains '/' - resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/") + resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/blumia/@docs~main/") assert.NoError(t, err) if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { t.FailNow() @@ -81,7 +81,7 @@ func TestCustomDomain(t *testing.T) { func TestGetNotFound(t *testing.T) { log.Println("=== TestGetNotFound ===") // test custom not found pages - resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah") + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/pages-404-demo/blah") assert.NoError(t, err) if !assert.NotNil(t, resp) { t.FailNow() @@ -95,7 +95,7 @@ func TestGetNotFound(t *testing.T) { func TestFollowSymlink(t *testing.T) { log.Printf("=== TestFollowSymlink ===\n") - resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link") + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link") assert.NoError(t, err) if !assert.NotNil(t, resp) { t.FailNow() @@ -111,7 +111,7 @@ func TestFollowSymlink(t *testing.T) { func TestLFSSupport(t *testing.T) { log.Printf("=== TestLFSSupport ===\n") - resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") assert.NoError(t, err) if !assert.NotNil(t, resp) { t.FailNow() From c286b3b1d0a21bc5cc447df7744da9894d857e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20Leopoldo=20Sologuren=20Guti=C3=A9rrez?= Date: Wed, 4 Jan 2023 05:26:14 +0000 Subject: [PATCH 24/62] Added TokenBucket to limit the rate of validation failures (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added new TockenBucket named `acmeClientFailLimit` to avoid being banned because of the [Failed validation limit](https://letsencrypt.org/docs/failed-validation-limit/) of Let's Encrypt. The behaviour is similar to the other limiters blocking the `obtainCert` func ensuring rate under limit. Co-authored-by: fsologureng Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/151 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Felipe Leopoldo Sologuren Gutiérrez Co-committed-by: Felipe Leopoldo Sologuren Gutiérrez --- server/certificates/certificates.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index e3cdbfb..1aa90a0 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -163,6 +163,9 @@ var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute) // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second) +// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/ +var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour) + type AcmeTLSChallengeProvider struct { challengeCache cache.SetGetKey } @@ -278,6 +281,9 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re res, err = acmeClient.Certificate.Renew(*renew, true, false, "") if err != nil { log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains) + if acmeUseRateLimits { + acmeClientFailLimit.Take() + } res = nil } } @@ -298,6 +304,9 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re Bundle: true, MustStaple: false, }) + if acmeUseRateLimits && err != nil { + acmeClientFailLimit.Take() + } } if err != nil { log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains) From bd538abd372c41336c355a7d0890a289bad256a5 Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 11 Jan 2023 00:00:37 +0000 Subject: [PATCH 25/62] Fix wrong redirect on custom domain with path (#154) closes #153 Co-authored-by: crapStone Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/154 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- integration/get_test.go | 18 ++++++++++++++++++ server/handler/handler_custom_domain.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/integration/get_test.go b/integration/get_test.go index f13ce8b..556ac53 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -78,6 +78,24 @@ func TestCustomDomain(t *testing.T) { assert.EqualValues(t, 106, getSize(resp.Body)) } +func TestCustomDomainRedirects(t *testing.T) { + log.Println("=== TestCustomDomainRedirects ===") + // test redirect from default pages domain to custom domain + resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/test_pages-server_custom-mock-domain/@main/README.md") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) + assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) + // TODO: custom port is not evaluated (witch does hurt tests & dev env only) + assert.EqualValues(t, "https://mock-pages.codeberg-test.org/@main/README.md", resp.Header.Get("Location")) + assert.EqualValues(t, `https:/codeberg.org/6543/test_pages-server_custom-mock-domain/src/branch/main/README.md; rel="canonical"; rel="canonical"`, resp.Header.Get("Link")) + + // TODO: test redirect from an custom domain to the primary custom domain (www.example.com -> example.com) + // (cover bug https://codeberg.org/Codeberg/pages-server/issues/153) +} + func TestGetNotFound(t *testing.T) { log.Println("=== TestGetNotFound ===") // test custom not found pages diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 2f98085..a541b74 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -54,7 +54,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g // only redirect if the target is also a codeberg page! targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache) if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+targetOpt.TargetPath, http.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect) return } From 513e79832ae5bd8e825eb4f5a2605b1c585eccd6 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 22 Jan 2023 18:52:21 +0000 Subject: [PATCH 26/62] Use correct log level for `CheckCanonicalDomain` (#162) - Currently any error generated by requesting the `.domains` file of a repository would be logged under the info log level, which isn't the correct log level when we exclude the not found error. - Use warn log level if the error isn't the not found error. Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/162 Reviewed-by: Otto --- server/upstream/domains.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 0e29673..ba1c494 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -45,7 +45,11 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, } } } else { - log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) + if err != gitea.ErrorNotFound { + log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) + } else { + log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) + } } domains = append(domains, o.TargetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { From 2c2087953d975c8cc9b57f6b7c94cb18a7894170 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Feb 2023 17:30:06 +0000 Subject: [PATCH 27/62] Add Integration test for custom domain redirect to another custom domain (#172) address #155 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/172 --- integration/get_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/integration/get_test.go b/integration/get_test.go index 556ac53..6dd57bc 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -89,11 +89,22 @@ func TestCustomDomainRedirects(t *testing.T) { assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) // TODO: custom port is not evaluated (witch does hurt tests & dev env only) + // assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/@main/README.md", resp.Header.Get("Location")) assert.EqualValues(t, "https://mock-pages.codeberg-test.org/@main/README.md", resp.Header.Get("Location")) assert.EqualValues(t, `https:/codeberg.org/6543/test_pages-server_custom-mock-domain/src/branch/main/README.md; rel="canonical"; rel="canonical"`, resp.Header.Get("Link")) - // TODO: test redirect from an custom domain to the primary custom domain (www.example.com -> example.com) - // (cover bug https://codeberg.org/Codeberg/pages-server/issues/153) + // test redirect from an custom domain to the primary custom domain (www.example.com -> example.com) + // regression test to https://codeberg.org/Codeberg/pages-server/issues/153 + resp, err = getTestHTTPSClient().Get("https://mock-pages-redirect.codeberg-test.org:4430/README.md") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) + assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) + // TODO: custom port is not evaluated (witch does hurt tests & dev env only) + // assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location")) + assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location")) } func TestGetNotFound(t *testing.T) { From 8b1f497bc4699a76c8323a1466a93de4fc0f8b4d Mon Sep 17 00:00:00 2001 From: Gusted Date: Fri, 10 Feb 2023 01:38:15 +0000 Subject: [PATCH 28/62] Allow to use certificate even if domain validation fails (#160) - Currently if the canonical domain validations fails(either for legitimate reasons or for bug reasons like the request to Gitea/Forgejo failing) it will use main domain certificate, which in the case for custom domains will warrant a security error as the certificate isn't issued to the custom domain. - This patch handles this situation more gracefully and instead only disallow obtaining a certificate if the domain validation fails, so in the case that a certificate still exists it can still be used even if the canonical domain validation fails. There's a small side effect, legitimate users that remove domains from `.domain` will still be able to use the removed domain(as long as the DNS records exists) as long as the certificate currently hold by pages-server isn't expired. - Given the increased usage in custom domains that are resulting in errors, I think it ways more than the side effect. - In order to future-proof against future slowdowns of instances, add a retry mechanism to the domain validation function, such that it's more likely to succeed even if the instance is not responding. - Refactor the code a bit and add some comments. Co-authored-by: Gusted Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/160 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/certificates/certificates.go | 9 +++- server/upstream/domains.go | 71 +++++++++++++++-------------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 1aa90a0..0bf5672 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -70,6 +70,7 @@ func TLSConfig(mainDomainSuffix string, } targetOwner := "" + mayObtainCert := true if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) { // deliver default certificate for the main domain (*.codeberg.page) sni = mainDomainSuffix @@ -87,7 +88,9 @@ func TLSConfig(mainDomainSuffix string, } _, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache) if !valid { - sni = mainDomainSuffix + // We shouldn't obtain a certificate when we cannot check if the + // repository has specified this domain in the `.domains` file. + mayObtainCert = false } } } @@ -106,6 +109,10 @@ func TLSConfig(mainDomainSuffix string, return nil, errors.New("won't request certificate for main domain, something really bad has happened") } + if !mayObtainCert { + return nil, fmt.Errorf("won't request certificate for %q", sni) + } + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { return nil, err diff --git a/server/upstream/domains.go b/server/upstream/domains.go index ba1c494..5b274b6 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -16,49 +16,54 @@ var canonicalDomainCacheTimeout = 15 * time.Minute const canonicalDomainConfig = ".domains" // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { - var ( - domains []string - valid bool - ) +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) { + // Check if this request is cached. if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { - domains = cachedValue.([]string) + domains := cachedValue.([]string) for _, domain := range domains { if domain == actualDomain { valid = true break } } - } else { - body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig) - if err == nil { - for _, domain := range strings.Split(string(body), "\n") { - domain = strings.ToLower(domain) - domain = strings.TrimSpace(domain) - domain = strings.TrimPrefix(domain, "http://") - domain = strings.TrimPrefix(domain, "https://") - if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { - domains = append(domains, domain) - } - if domain == actualDomain { - valid = true - } - } - } else { - if err != gitea.ErrorNotFound { - log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) - } else { - log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) - } + return domains[0], valid + } + + body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig) + if err == nil || err == gitea.ErrorNotFound { + log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) + } + + var domains []string + for _, domain := range strings.Split(string(body), "\n") { + domain = strings.ToLower(domain) + domain = strings.TrimSpace(domain) + domain = strings.TrimPrefix(domain, "http://") + domain = strings.TrimPrefix(domain, "https://") + if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { + domains = append(domains, domain) } - domains = append(domains, o.TargetOwner+mainDomainSuffix) - if domains[len(domains)-1] == actualDomain { + if domain == actualDomain { valid = true } - if o.TargetRepo != "" && o.TargetRepo != "pages" { - domains[len(domains)-1] += "/" + o.TargetRepo - } - _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout) } + + // Add [owner].[pages-domain] as valid domnain. + domains = append(domains, o.TargetOwner+mainDomainSuffix) + if domains[len(domains)-1] == actualDomain { + valid = true + } + + // If the target repository isn't called pages, add `/[repository]` to the + // previous valid domain. + if o.TargetRepo != "" && o.TargetRepo != "pages" { + domains[len(domains)-1] += "/" + o.TargetRepo + } + + // Add result to cache. + _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout) + + // Return the first domain from the list and return if any of the domains + // matched the requested domain. return domains[0], valid } From 7fce7cf68bcc35cd5d054b0434d6943d77f7053a Mon Sep 17 00:00:00 2001 From: foehammer Date: Fri, 10 Feb 2023 01:44:44 +0000 Subject: [PATCH 29/62] Added Whitespace Trimming TXT DNS Records (#152) Solves https://codeberg.org/Codeberg/Community/issues/823 and https://codeberg.org/Codeberg/pages-server/issues/143 Co-authored-by: foehammer127 Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/152 Reviewed-by: Otto Reviewed-by: Gusted Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: foehammer Co-committed-by: foehammer --- server/dns/dns.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dns/dns.go b/server/dns/dns.go index 818e29a..2719d4d 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -28,7 +28,7 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG names, err := net.LookupTXT(domain) if err == nil { for _, name := range names { - name = strings.TrimSuffix(name, ".") + name = strings.TrimSuffix(strings.TrimSpace(name), ".") if strings.HasSuffix(name, mainDomainSuffix) { cname = name break From 7b35a192bf4f87ca93c7d3a8a3d3dd2672c22fec Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 10 Feb 2023 03:00:14 +0000 Subject: [PATCH 30/62] Add cert store option based on sqlite3, mysql & postgres (#173) Deprecate **pogreb**! close #169 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/173 --- .gitignore | 2 + .golangci.yml | 14 + .woodpecker.yml | 15 +- Dockerfile | 5 +- Justfile | 20 +- README.md | 3 +- cmd/certs.go | 97 ++++-- cmd/flags.go | 259 ++++++++-------- cmd/main.go | 17 +- cmd/setup.go | 45 +++ go.mod | 18 +- go.sum | 377 +++++++++++++++++++++++- integration/main_test.go | 3 +- main.go | 2 +- server/certificates/certificates.go | 133 +++++---- server/certificates/mock.go | 15 +- server/certificates/mock_test.go | 3 +- server/database/interface.go | 62 +++- server/database/mock.go | 5 +- server/database/{setup.go => pogreb.go} | 31 +- server/database/xorm.go | 121 ++++++++ server/upstream/helper.go | 8 +- 22 files changed, 1000 insertions(+), 255 deletions(-) create mode 100644 cmd/setup.go rename server/database/{setup.go => pogreb.go} (76%) create mode 100644 server/database/xorm.go diff --git a/.gitignore b/.gitignore index dfe69ac..8745935 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ .idea/ +.cache/ *.iml key-database.pogreb/ acme-account.json build/ vendor/ pages +certs.sqlite diff --git a/.golangci.yml b/.golangci.yml index 6d9b95a..488ca09 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,9 +12,23 @@ linters-settings: - hugeParam linters: + disable-all: true enable: - unconvert - gocritic + - gofumpt + - bidichk + - errcheck + - gofmt + - goimports + - gosimple + - govet + - ineffassign + - misspell + - staticcheck + - typecheck + - unused + - whitespace run: timeout: 5m diff --git a/.woodpecker.yml b/.woodpecker.yml index 7e6d694..ba44f82 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,7 +1,7 @@ pipeline: # use vendor to cache dependencies vendor: - image: golang:1.18 + image: golang:1.20 commands: - go mod vendor @@ -65,6 +65,19 @@ pipeline: - RAW_DOMAIN=raw.localhost.mock.directory - PORT=4430 + # TODO: remove in next version + integration-tests-legacy: + group: test + image: codeberg.org/6543/docker-images/golang_just + commands: + - just integration + environment: + - ACME_API=https://acme.mock.directory + - PAGES_DOMAIN=localhost.mock.directory + - RAW_DOMAIN=raw.localhost.mock.directory + - PORT=4430 + - DB_TYPE= + release: image: plugins/gitea-release settings: diff --git a/Dockerfile b/Dockerfile index 904d6f4..eec97de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,9 @@ -FROM golang:alpine as build +FROM techknowlogick/xgo as build WORKDIR /workspace -RUN apk add ca-certificates COPY . . -RUN CGO_ENABLED=0 go build . +RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' . FROM scratch COPY --from=build /workspace/pages /pages diff --git a/Justfile b/Justfile index 7d72fe8..683222d 100644 --- a/Justfile +++ b/Justfile @@ -1,3 +1,6 @@ +CGO_FLAGS := '-extldflags "-static" -linkmode external' +TAGS := 'sqlite sqlite_unlock_notify netgo' + dev: #!/usr/bin/env bash set -euxo pipefail @@ -7,16 +10,15 @@ dev: export RAW_DOMAIN=raw.localhost.mock.directory export PORT=4430 export LOG_LEVEL=trace - go run . + go run -tags '{{TAGS}}' . build: - CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./ build-tag VERSION: - CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./ lint: tool-golangci tool-gofumpt - [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ golangci-lint run --timeout 5m --build-tags integration # TODO: run editorconfig-checker @@ -25,7 +27,7 @@ fmt: tool-gofumpt clean: go clean ./... - rm -rf build/ + rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json tool-golangci: @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ @@ -38,13 +40,13 @@ tool-gofumpt: fi test: - go test -race codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ test-run TEST: - go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ integration: - go test -race -tags integration codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/... integration-run TEST: - go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... diff --git a/README.md b/README.md index 50cdc3c..6d5a294 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ Thank you very much. run `just dev` now this pages should work: - - https://magiclike.localhost.mock.directory:4430/ + - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg - https://momar.localhost.mock.directory:4430/ci-testing/ - https://momar.localhost.mock.directory:4430/pag/@master/ + - https://mock-pages.codeberg-test.org:4430/README.md diff --git a/cmd/certs.go b/cmd/certs.go index d93fe13..4adf076 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -2,8 +2,10 @@ package cmd import ( "fmt" + "time" - "github.com/akrylysov/pogreb" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "codeberg.org/codeberg/pages/server/database" @@ -23,25 +25,85 @@ var Certs = &cli.Command{ Usage: "remove a certificate from the database", Action: removeCert, }, + { + Name: "migrate", + Usage: "migrate from \"pogreb\" driver to dbms driver", + Action: migrateCerts, + }, }, + Flags: append(CertStorageFlags, []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Usage: "print trace info", + EnvVars: []string{"VERBOSE"}, + Value: false, + }, + }...), +} + +func migrateCerts(ctx *cli.Context) error { + dbType := ctx.String("db-type") + if dbType == "" { + dbType = "sqlite3" + } + dbConn := ctx.String("db-conn") + dbPogrebConn := ctx.String("db-pogreb") + verbose := ctx.Bool("verbose") + + log.Level(zerolog.InfoLevel) + if verbose { + log.Level(zerolog.TraceLevel) + } + + xormDB, err := database.NewXormDB(dbType, dbConn) + if err != nil { + return fmt.Errorf("could not connect to database: %w", err) + } + defer xormDB.Close() + + pogrebDB, err := database.NewPogreb(dbPogrebConn) + if err != nil { + return fmt.Errorf("could not open database: %w", err) + } + defer pogrebDB.Close() + + fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn) + + certs, err := pogrebDB.Items(0, 0) + if err != nil { + return err + } + + for _, cert := range certs { + if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil { + return err + } + } + + fmt.Println("... done") + return nil } func listCerts(ctx *cli.Context) error { - // TODO: make "key-database.pogreb" set via flag - keyDatabase, err := database.New("key-database.pogreb") + certDB, closeFn, err := openCertDB(ctx) if err != nil { - return fmt.Errorf("could not create database: %v", err) + return err + } + defer closeFn() + + items, err := certDB.Items(0, 0) + if err != nil { + return err } - items := keyDatabase.Items() - for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() { - if err != nil { - return err + fmt.Printf("Domain\tValidTill\n\n") + for _, cert := range items { + if cert.Domain[0] == '.' { + cert.Domain = "*" + cert.Domain } - if domain[0] == '.' { - fmt.Printf("*") - } - fmt.Printf("%s\n", domain) + fmt.Printf("%s\t%s\n", + cert.Domain, + time.Unix(cert.ValidTill, 0).Format(time.RFC3339)) } return nil } @@ -53,20 +115,17 @@ func removeCert(ctx *cli.Context) error { domains := ctx.Args().Slice() - // TODO: make "key-database.pogreb" set via flag - keyDatabase, err := database.New("key-database.pogreb") + certDB, closeFn, err := openCertDB(ctx) if err != nil { - return fmt.Errorf("could not create database: %v", err) + return err } + defer closeFn() for _, domain := range domains { fmt.Printf("Removing domain %s from the database...\n", domain) - if err := keyDatabase.Delete(domain); err != nil { + if err := certDB.Delete(domain); err != nil { return err } } - if err := keyDatabase.Close(); err != nil { - return err - } return nil } diff --git a/cmd/flags.go b/cmd/flags.go index 8ac09ec..40bb053 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -4,120 +4,149 @@ import ( "github.com/urfave/cli/v2" ) -var ServeFlags = []cli.Flag{ - // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static - // pages, or used for comparison in CNAME lookups. Static pages can be accessed through - // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". - &cli.StringFlag{ - Name: "pages-domain", - Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", - EnvVars: []string{"PAGES_DOMAIN"}, - Value: "codeberg.page", - }, - // GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. - &cli.StringFlag{ - Name: "gitea-root", - Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", - EnvVars: []string{"GITEA_ROOT"}, - Value: "https://codeberg.org", - }, - // GiteaApiToken specifies an api token for the Gitea instance - &cli.StringFlag{ - Name: "gitea-api-token", - Usage: "specifies an api token for the Gitea instance", - EnvVars: []string{"GITEA_API_TOKEN"}, - Value: "", - }, - // RawDomain specifies the domain from which raw repository content shall be served in the following format: - // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} - // (set to []byte(nil) to disable raw content hosting) - &cli.StringFlag{ - Name: "raw-domain", - Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", - EnvVars: []string{"RAW_DOMAIN"}, - Value: "raw.codeberg.page", - }, - // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). - &cli.StringFlag{ - Name: "raw-info-page", - Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", - EnvVars: []string{"RAW_INFO_PAGE"}, - Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", - }, +var ( + CertStorageFlags = []cli.Flag{ + &cli.StringFlag{ + // TODO: remove in next version + // DEPRICATED + Name: "db-pogreb", + Value: "key-database.pogreb", + EnvVars: []string{"DB_POGREB"}, + }, + &cli.StringFlag{ + Name: "db-type", + Value: "", // TODO: "sqlite3" in next version + EnvVars: []string{"DB_TYPE"}, + }, + &cli.StringFlag{ + Name: "db-conn", + Value: "certs.sqlite", + EnvVars: []string{"DB_CONN"}, + }, + } - // Server - &cli.StringFlag{ - Name: "host", - Usage: "specifies host of listening address", - EnvVars: []string{"HOST"}, - Value: "[::]", - }, - &cli.StringFlag{ - Name: "port", - Usage: "specifies port of listening address", - EnvVars: []string{"PORT"}, - Value: "443", - }, - &cli.BoolFlag{ - Name: "enable-http-server", - // TODO: desc - EnvVars: []string{"ENABLE_HTTP_SERVER"}, - }, - // Server Options - &cli.BoolFlag{ - Name: "enable-lfs-support", - Usage: "enable lfs support, require gitea v1.17.0 as backend", - EnvVars: []string{"ENABLE_LFS_SUPPORT"}, - Value: true, - }, - &cli.BoolFlag{ - Name: "enable-symlink-support", - Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend", - EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, - Value: true, - }, - &cli.StringFlag{ - Name: "log-level", - Value: "warn", - Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", - EnvVars: []string{"LOG_LEVEL"}, - }, + ServerFlags = append(CertStorageFlags, []cli.Flag{ + // ############# + // ### Gitea ### + // ############# + // GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. + &cli.StringFlag{ + Name: "gitea-root", + Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", + EnvVars: []string{"GITEA_ROOT"}, + Value: "https://codeberg.org", + }, + // GiteaApiToken specifies an api token for the Gitea instance + &cli.StringFlag{ + Name: "gitea-api-token", + Usage: "specifies an api token for the Gitea instance", + EnvVars: []string{"GITEA_API_TOKEN"}, + Value: "", + }, - // ACME - &cli.StringFlag{ - Name: "acme-api-endpoint", - EnvVars: []string{"ACME_API"}, - Value: "https://acme-v02.api.letsencrypt.org/directory", - }, - &cli.StringFlag{ - Name: "acme-email", - EnvVars: []string{"ACME_EMAIL"}, - Value: "noreply@example.email", - }, - &cli.BoolFlag{ - Name: "acme-use-rate-limits", - // TODO: Usage - EnvVars: []string{"ACME_USE_RATE_LIMITS"}, - Value: true, - }, - &cli.BoolFlag{ - Name: "acme-accept-terms", - // TODO: Usage - EnvVars: []string{"ACME_ACCEPT_TERMS"}, - }, - &cli.StringFlag{ - Name: "acme-eab-kid", - // TODO: Usage - EnvVars: []string{"ACME_EAB_KID"}, - }, - &cli.StringFlag{ - Name: "acme-eab-hmac", - // TODO: Usage - EnvVars: []string{"ACME_EAB_HMAC"}, - }, - &cli.StringFlag{ - Name: "dns-provider", - // TODO: Usage - EnvVars: []string{"DNS_PROVIDER"}, - }, -} + // ########################### + // ### Page Server Domains ### + // ########################### + // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static + // pages, or used for comparison in CNAME lookups. Static pages can be accessed through + // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". + &cli.StringFlag{ + Name: "pages-domain", + Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", + EnvVars: []string{"PAGES_DOMAIN"}, + Value: "codeberg.page", + }, + // RawDomain specifies the domain from which raw repository content shall be served in the following format: + // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} + // (set to []byte(nil) to disable raw content hosting) + &cli.StringFlag{ + Name: "raw-domain", + Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", + EnvVars: []string{"RAW_DOMAIN"}, + Value: "raw.codeberg.page", + }, + // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). + &cli.StringFlag{ + Name: "raw-info-page", + Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", + EnvVars: []string{"RAW_INFO_PAGE"}, + Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", + }, + + // Server + &cli.StringFlag{ + Name: "host", + Usage: "specifies host of listening address", + EnvVars: []string{"HOST"}, + Value: "[::]", + }, + &cli.StringFlag{ + Name: "port", + Usage: "specifies port of listening address", + EnvVars: []string{"PORT"}, + Value: "443", + }, + &cli.BoolFlag{ + Name: "enable-http-server", + // TODO: desc + EnvVars: []string{"ENABLE_HTTP_SERVER"}, + }, + // Server Options + &cli.BoolFlag{ + Name: "enable-lfs-support", + Usage: "enable lfs support, require gitea v1.17.0 as backend", + EnvVars: []string{"ENABLE_LFS_SUPPORT"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "enable-symlink-support", + Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend", + EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, + Value: true, + }, + &cli.StringFlag{ + Name: "log-level", + Value: "warn", + Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", + EnvVars: []string{"LOG_LEVEL"}, + }, + + // ACME + &cli.StringFlag{ + Name: "acme-api-endpoint", + EnvVars: []string{"ACME_API"}, + Value: "https://acme-v02.api.letsencrypt.org/directory", + }, + &cli.StringFlag{ + Name: "acme-email", + EnvVars: []string{"ACME_EMAIL"}, + Value: "noreply@example.email", + }, + &cli.BoolFlag{ + Name: "acme-use-rate-limits", + // TODO: Usage + EnvVars: []string{"ACME_USE_RATE_LIMITS"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "acme-accept-terms", + // TODO: Usage + EnvVars: []string{"ACME_ACCEPT_TERMS"}, + }, + &cli.StringFlag{ + Name: "acme-eab-kid", + // TODO: Usage + EnvVars: []string{"ACME_EAB_KID"}, + }, + &cli.StringFlag{ + Name: "acme-eab-hmac", + // TODO: Usage + EnvVars: []string{"ACME_EAB_HMAC"}, + }, + &cli.StringFlag{ + Name: "dns-provider", + // TODO: Usage + EnvVars: []string{"DNS_PROVIDER"}, + }, + }...) +) diff --git a/cmd/main.go b/cmd/main.go index b72013a..af4d2ce 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,7 +18,6 @@ import ( "codeberg.org/codeberg/pages/server" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/handler" ) @@ -38,7 +37,7 @@ var BlacklistedPaths = []string{ // Serve sets up and starts the web server. func Serve(ctx *cli.Context) error { - // Initalize the logger. + // Initialize the logger. logLevel, err := zerolog.ParseLevel(ctx.String("log-level")) if err != nil { return err @@ -74,6 +73,13 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix = "." + mainDomainSuffix } + // Init ssl cert database + certDB, closeFn, err := openCertDB(ctx) + if err != nil { + return err + } + defer closeFn() + keyCache := cache.NewKeyValueCache() challengeCache := cache.NewKeyValueCache() // canonicalDomainCache stores canonical domains @@ -104,13 +110,6 @@ func Serve(ctx *cli.Context) error { return fmt.Errorf("couldn't create listener: %v", err) } - // TODO: make "key-database.pogreb" set via flag - certDB, err := database.New("key-database.pogreb") - if err != nil { - return fmt.Errorf("could not create database: %v", err) - } - defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it - listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaClient, dnsProvider, diff --git a/cmd/setup.go b/cmd/setup.go new file mode 100644 index 0000000..3d1d6ee --- /dev/null +++ b/cmd/setup.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + "codeberg.org/codeberg/pages/server/database" +) + +func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { + if ctx.String("db-type") != "" { + log.Trace().Msg("use xorm mode") + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) + } + } else { + // TODO: remove in next version + fmt.Println(` +###################### +## W A R N I N G !!! # +###################### + +You use "pogreb" witch is deprecated and will be removed in the next version. +Please switch to sqlite, mysql or postgres !!! + +The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`) + log.Error().Msg("depricated \"pogreb\" used\n") + + certDB, err = database.NewPogreb(ctx.String("db-pogreb")) + if err != nil { + return nil, nil, fmt.Errorf("could not create database: %w", err) + } + } + + closeFn = func() { + if err := certDB.Close(); err != nil { + log.Error().Err(err) + } + } + + return certDB, closeFn, nil +} diff --git a/go.mod b/go.mod index 77ed762..2e75c8e 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,21 @@ module codeberg.org/codeberg/pages -go 1.19 +go 1.20 require ( code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 + github.com/go-sql-driver/mysql v1.6.0 github.com/joho/godotenv v1.4.0 + github.com/lib/pq v1.10.7 + github.com/mattn/go-sqlite3 v1.14.16 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 + xorm.io/xorm v1.3.2 ) require ( @@ -47,11 +51,13 @@ require ( github.com/go-errors/errors v1.0.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect - github.com/gofrs/uuid v3.2.0+incompatible // indirect + github.com/goccy/go-json v0.8.1 // indirect + github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.1.1 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/gophercloud/gophercloud v0.16.0 // indirect github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect @@ -62,7 +68,7 @@ require ( github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect github.com/jarcoal/httpmock v1.0.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.7 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect @@ -78,7 +84,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect github.com/nrdcg/auroradns v1.0.1 // indirect github.com/nrdcg/desec v0.6.0 // indirect @@ -103,6 +109,7 @@ require ( github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect github.com/spf13/cast v1.3.1 // indirect github.com/stretchr/objx v0.3.0 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect github.com/vultr/govultr/v2 v2.7.1 // indirect @@ -124,4 +131,5 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + xorm.io/builder v0.3.12 // indirect ) diff --git a/go.sum b/go.sum index a44001c..c1042c9 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= +gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= +gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -55,24 +58,38 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 h1:xPMsUicZ3iosVPSIP7bW5EcGUzjiiMl1OYTe14y/R24= github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87/go.mod h1:iGLljf5n9GjT6kc0HBvyI1nOKnGQbNB66VzSNbK5iks= github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a h1:Cf4CrDeyrIcuIiJZEZJAH5dapqQ6J3OmP/vHPbDjaFA= github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a/go.mod h1:ig6eVXkYn/9dz0Vm8UdLf+E0u1bE6kBSn3n2hqk6jas= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbxuNdz3FeyvHFnRjmo= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo= github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -81,27 +98,39 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.20.0 h1:y2a6KwYHTFxhw+8PLhz0q5hpTGj6Un3W1pbpQLhzFaE= github.com/cloudflare/cloudflare-go v0.20.0/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF1FhM7N60fuNiFdYTI= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4= github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -110,6 +139,7 @@ github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454Wv github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/deepmap/oapi-codegen v1.6.1 h1:2BvsmRb6pogGNtr8Ann+esAbSKFXx2CZN18VpAMecnw= github.com/deepmap/oapi-codegen v1.6.1/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= @@ -118,6 +148,13 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnsimple/dnsimple-go v0.70.1 h1:cSZndVjttLpgplDuesY4LFIvfKf/zRA1J7mCATBbzSM= github.com/dnsimple/dnsimple-go v0.70.1/go.mod h1:F9WHww9cC76hrnwGFfAfrqdW99j3MOYasQcIwTS/aUk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/exoscale/egoscale v0.67.0 h1:qgWh7T5IZGrNWtg6ib4dr+76WThvB+odTtGG+DGbXF8= @@ -127,6 +164,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -144,24 +183,37 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw= +github.com/goccy/go-json v0.8.1 h1:4/Wjm0JIJaTDm8K1KcGrLHJoa8EsJ13YWeX+6Kfq6uI= +github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -186,6 +238,10 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -194,6 +250,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= @@ -208,8 +265,10 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -220,16 +279,23 @@ github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae h1:Hi3IgB9RQDE15 github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae/go.mod h1:wx8HMD8oQD0Ryhz6+6ykq75PJ79iPyEqYHfwZ4l7OsA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -246,6 +312,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -258,12 +325,61 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df h1:MZf03xP9WdakyXhOWuAD5uPK3wHh96wCsqe3hCMKh8E= github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/infobloxopen/infoblox-go-client v1.1.1 h1:728A6LbLjptj/7kZjHyIxQnm768PWHfGFm0HH8FnbtU= github.com/infobloxopen/infoblox-go-client v1.1.1/go.mod h1:BXiw7S2b9qJoM8MS40vfgCNB2NLHGusk1DtO16BD9zI= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= +github.com/jackc/pgtype v1.8.0/go.mod h1:PqDKcEBtllAtk/2p6z6SHdXW5UB+MhE75tUol2OKexE= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= +github.com/jackc/pgx/v4 v4.12.0/go.mod h1:fE547h6VulLPA3kySjfnSG/e2D861g/50JlVUa/ub60= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jarcoal/httpmock v1.0.6 h1:e81vOSexXU3mJuJ4l//geOmKIt+Vkxerk1feQBC8D0g= github.com/jarcoal/httpmock v1.0.6/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= @@ -277,8 +393,10 @@ github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -286,6 +404,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc= @@ -298,6 +418,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labbsr0x/bindman-dns-webhook v1.0.2 h1:I7ITbmQPAVwrDdhd6dHKi+MYJTJqPCK0jE6YNBAevnk= @@ -306,6 +427,15 @@ github.com/labbsr0x/goh v1.0.1 h1:97aBJkDjpyBZGPbQuOK5/gHcSFbcr5aRsq3RSRJFpPk= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.31.1 h1:dBtjKo7J9UhNFhTOclEXb12RRyQDaRBxISdONVuU+DA= github.com/linode/linodego v0.31.1/go.mod h1:BR0gVkCJffEdIGJSl6bHR80Ty+Uvg/2jkjmrWaFectM= github.com/liquidweb/go-lwApi v0.0.0-20190605172801-52a4864d2738/go.mod h1:0sYF9rMXb0vlG+4SzdiGMXHheCZxjguMq+Zb4S2BfBs= @@ -315,19 +445,25 @@ github.com/liquidweb/liquidweb-cli v0.6.9 h1:acbIvdRauiwbxIsOCEMXGwF75aSJDbDiyAW github.com/liquidweb/liquidweb-cli v0.6.9/go.mod h1:cE1uvQ+x24NGUL75D0QagOFCG8Wdvmwu8aL9TLmA/eQ= github.com/liquidweb/liquidweb-go v1.6.3 h1:NVHvcnX3eb3BltiIoA+gLYn15nOpkYkdizOEYGSKrk4= github.com/liquidweb/liquidweb-go v1.6.3/go.mod h1:SuXXp+thr28LnjEw18AYtWwIbWMHSUiajPQs8T9c/Rc= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= @@ -337,6 +473,9 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -360,11 +499,19 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 h1:o6uBwrhM5C8Ll3MAAxrQxRHEu7FkapwTuI2WmL1rw4g= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nrdcg/auroradns v1.0.1 h1:m/kBq83Xvy3cU261MOknd8BdnOk12q4lAWM+kOdsC2Y= github.com/nrdcg/auroradns v1.0.1/go.mod h1:y4pc0i9QXYlFCWrhWrUSIETnZgrf4KuwjDIWmmXo3JI= @@ -383,29 +530,49 @@ github.com/nrdcg/porkbun v0.1.1/go.mod h1:JWl/WKnguWos4mjfp4YizvvToigk9qpQwrodOk github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/oracle/oci-go-sdk v24.3.0+incompatible h1:x4mcfb4agelf1O4/1/auGlZ1lr97jXRSSN5MxTgG/zU= github.com/oracle/oci-go-sdk v24.3.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= github.com/ovh/go-ovh v1.1.0 h1:bHXZmw8nTgZin4Nv7JuaLs0KG5x54EQR7migYTd1zrk= github.com/ovh/go-ovh v1.1.0/go.mod h1:AxitLZ5HBRPyUd+Zl60Ajaag+rNTdVXWIkzfrVuTXWA= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -414,27 +581,41 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/pquerna/otp v1.3.0 h1:oJV/SkzR33anKXwQU3Of42rL4wbrffP4uvUf1SvS5Xs= github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjIi3L6QmL/V9vpJPbc/j/F2u55d3fs= github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -442,12 +623,18 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI= github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= @@ -466,21 +653,28 @@ github.com/softlayer/softlayer-go v1.0.3/go.mod h1:6HepcfAXROz0Rf63krk5hPZyHT6qy github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e h1:3OgWYFw7jxCZPcvAg+4R8A50GZ+CCkARF10lxu2qDsQ= github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e/go.mod h1:fKZCUVdirrxrBpwd9wb+lSoVixvpwAu8eHzbQB2tums= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -490,11 +684,16 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggsC4564= github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= @@ -511,7 +710,13 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -519,23 +724,37 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 h1:d9qaMM+ODpCq+9We41//fu/sHsTnXcrqd1en3x+GKy4= go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -570,6 +789,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -579,6 +799,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -589,6 +810,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -629,9 +851,11 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -641,6 +865,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -648,6 +873,7 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -663,13 +889,16 @@ golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -684,6 +913,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -692,6 +922,7 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -700,14 +931,18 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -716,6 +951,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -725,12 +961,17 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -743,6 +984,7 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -753,6 +995,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -767,10 +1010,15 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171 h1:xes2Q2k+d/+YNXVw0FpZkIDJiaux4OVrRKXRAzH6A0U= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= @@ -789,10 +1037,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -806,6 +1057,7 @@ gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -818,12 +1070,129 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U= +modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60= +modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw= +modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI= +modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag= +modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw= +modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ= +modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c= +modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo= +modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg= +modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I= +modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs= +modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8= +modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE= +modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk= +modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w= +modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE= +modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8= +modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc= +modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU= +modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE= +modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk= +modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI= +modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE= +modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg= +modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74= +modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU= +modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU= +modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc= +modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM= +modernc.org/ccgo/v3 v3.12.65/go.mod h1:D6hQtKxPNZiY6wDBtehSGKFKmyXn53F8nGTpH+POmS4= +modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ= +modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84= +modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ= +modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY= +modernc.org/ccgo/v3 v3.12.82 h1:wudcnJyjLj1aQQCXF3IM9Gz2X6UNjw+afIghzdtn0v8= +modernc.org/ccgo/v3 v3.12.82/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w= +modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q= +modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg= +modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M= +modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU= +modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE= +modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso= +modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8= +modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8= +modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I= +modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk= +modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY= +modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE= +modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg= +modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM= +modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg= +modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo= +modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8= +modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ= +modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA= +modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM= +modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg= +modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE= +modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM= +modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU= +modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw= +modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M= +modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18= +modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8= +modernc.org/libc v1.11.70/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw= +modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw= +modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0= +modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI= +modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE= +modernc.org/libc v1.11.87 h1:PzIzOqtlzMDDcCzJ5cUP6h/Ku6Fa9iyflP2ccTY64aE= +modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14= +modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM= +modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.14.2 h1:ohsW2+e+Qe2To1W6GNezzKGwjXwSax6R+CrhRxVaFbE= +modernc.org/sqlite v1.14.2/go.mod h1:yqfn85u8wVOE6ub5UT8VI9JjhrwBUUCNyTACN0h6Sx8= +modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY= +modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM= +xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/xorm v1.3.2 h1:uTRRKF2jYzbZ5nsofXVUx6ncMaek+SHjWYtCXyZo1oM= +xorm.io/xorm v1.3.2/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= diff --git a/integration/main_test.go b/integration/main_test.go index 406b33a..3e0e187 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -40,11 +40,12 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") setEnvIfNotSet("PORT", "4430") + setEnvIfNotSet("DB_TYPE", "sqlite3") app := cli.NewApp() app.Name = "pages-server" app.Action = cmd.Serve - app.Flags = cmd.ServeFlags + app.Flags = cmd.ServerFlags go func() { if err := app.RunContext(ctx, args); err != nil { diff --git a/main.go b/main.go index 2836b86..8eac1fb 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,7 @@ func main() { app.Version = version app.Usage = "pages server" app.Action = cmd.Serve - app.Flags = cmd.ServeFlags + app.Flags = cmd.ServerFlags app.Commands = []*cli.Command{ cmd.Certs, } diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 0bf5672..555539e 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -1,14 +1,12 @@ package certificates import ( - "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" - "encoding/gob" "encoding/json" "errors" "fmt" @@ -100,10 +98,9 @@ func TLSConfig(mainDomainSuffix string, return tlsCertificate.(*tls.Certificate), nil } - var tlsCertificate tls.Certificate + var tlsCertificate *tls.Certificate var err error - var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { + if tlsCertificate, err = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); err != nil { // request a new certificate if strings.EqualFold(sni, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") @@ -119,12 +116,11 @@ func TLSConfig(mainDomainSuffix string, } } - if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil { + if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil { return nil, err } - return &tlsCertificate, nil + return tlsCertificate, nil }, - PreferServerCipherSuites: true, NextProtos: []string{ "h2", "http/1.1", @@ -205,54 +201,53 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (*tls.Certificate, error) { // parse certificate from database res, err := certDB.Get(sni) if err != nil { - panic(err) // TODO: no panic - } - if res == nil { - return tls.Certificate{}, false + return nil, err + } else if res == nil { + return nil, database.ErrNotFound } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { - panic(err) + return nil, err } // TODO: document & put into own function if !strings.EqualFold(sni, mainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { - panic(err) + return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err) } // renew certificates 7 days before they expire if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) { - // TODO: add ValidUntil to custom res struct + // TODO: use ValidTill of custom cert struct if res.CSR != nil && len(res.CSR) > 0 { // CSR stores the time when the renewal shall be tried again nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64) if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) { - return tlsCertificate, true + return &tlsCertificate, nil } } + // TODO: make a queue ? go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) - if err != nil { + if _, err := obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB); err != nil { log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err) } })() } } - return tlsCertificate, true + return &tlsCertificate, nil } var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -265,16 +260,16 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) - if !ok { - return tls.Certificate{}, errors.New("certificate failed in synchronous request") + cert, err := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + if err != nil { + return nil, fmt.Errorf("certificate failed in synchronous request: %w", err) } return cert, nil } defer obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase), nil + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) } // request actual cert @@ -297,7 +292,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if res == nil { if user != "" { if err := checkUserLimit(user); err != nil { - return tls.Certificate{}, err + return nil, err } } @@ -320,33 +315,42 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if renew != nil && renew.CertURL != "" { tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey) if err != nil { - return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err + mockC, err2 := mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase) + if err2 != nil { + return nil, errors.Join(err, err2) + } + return mockC, err } leaf, err := leaf(&tlsCertificate) if err == nil && leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) if err := keyDatabase.Put(name, renew); err != nil { - return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err + mockC, err2 := mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase) + if err2 != nil { + return nil, errors.Join(err, err2) + } + return mockC, err } - return tlsCertificate, nil + return &tlsCertificate, nil } } - return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err + return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase) } log.Debug().Msgf("Obtained certificate for %v", domains) if err := keyDatabase.Put(name, res); err != nil { - return tls.Certificate{}, err + return nil, err } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { - return tls.Certificate{}, err + return nil, err } - return tlsCertificate, nil + return &tlsCertificate, nil } func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { + // TODO: make it a config flag const configFile = "acme-account.json" var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config @@ -431,8 +435,8 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits mainCertBytes, err := certDB.Get(mainDomainSuffix) - if err != nil { - return fmt.Errorf("cert database is not working") + if err != nil && !errors.Is(err, database.ErrNotFound) { + return fmt.Errorf("cert database is not working: %w", err) } acmeClient, err = lego.NewClient(acmeConfig) @@ -485,41 +489,35 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { for { - // clean up expired certs - now := time.Now() + // delete expired certs that will be invalid until next clean up + threshold := time.Now().Add(interval) expiredCertCount := 0 - keyDatabaseIterator := certDB.Items() - key, resBytes, err := keyDatabaseIterator.Next() - for err == nil { - if !strings.EqualFold(string(key), mainDomainSuffix) { - resGob := bytes.NewBuffer(resBytes) - resDec := gob.NewDecoder(resGob) - res := &certificate.Resource{} - err = resDec.Decode(res) - if err != nil { - panic(err) - } - tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - if err != nil || tlsCertificates[0].NotAfter.Before(now) { - err := certDB.Delete(string(key)) - if err != nil { - log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key)) - } else { - expiredCertCount++ + certs, err := certDB.Items(0, 0) + if err != nil { + log.Error().Err(err).Msg("could not get certs from list") + } else { + for _, cert := range certs { + if !strings.EqualFold(cert.Domain, strings.TrimPrefix(mainDomainSuffix, ".")) { + if time.Unix(cert.ValidTill, 0).Before(threshold) { + err := certDB.Delete(cert.Domain) + if err != nil { + log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Domain) + } else { + expiredCertCount++ + } } } } - key, resBytes, err = keyDatabaseIterator.Next() - } - log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) + log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) - // compact the database - msg, err := certDB.Compact() - if err != nil { - log.Error().Err(err).Msg("Compacting key database failed") - } else { - log.Debug().Msgf("Compacted key database: %s", msg) + // compact the database + msg, err := certDB.Compact() + if err != nil { + log.Error().Err(err).Msg("Compacting key database failed") + } else { + log.Debug().Msgf("Compacted key database: %s", msg) + } } // update main cert @@ -530,9 +528,10 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", mainDomainSuffix) } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - - // renew main certificate 30 days before it expires - if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) { + if err != nil { + log.Error().Err(fmt.Errorf("could not parse cert for mainDomainSuffix: %w", err)) + } else if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) { + // renew main certificate 30 days before it expires go (func() { _, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { diff --git a/server/certificates/mock.go b/server/certificates/mock.go index 0e87e6e..a28d0f4 100644 --- a/server/certificates/mock.go +++ b/server/certificates/mock.go @@ -13,14 +13,15 @@ import ( "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certificate" + "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/database" ) -func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate { +func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) if err != nil { - panic(err) + return nil, err } template := x509.Certificate{ @@ -52,7 +53,7 @@ func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) key, ) if err != nil { - panic(err) + return nil, err } out := &bytes.Buffer{} @@ -61,7 +62,7 @@ func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) Type: "CERTIFICATE", }) if err != nil { - panic(err) + return nil, err } outBytes := out.Bytes() res := &certificate.Resource{ @@ -75,12 +76,12 @@ func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) databaseName = mainDomainSuffix } if err := keyDatabase.Put(databaseName, res); err != nil { - panic(err) + log.Error().Err(err) } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { - panic(err) + return nil, err } - return tlsCertificate + return &tlsCertificate, nil } diff --git a/server/certificates/mock_test.go b/server/certificates/mock_test.go index 1cbd1f6..00e1b21 100644 --- a/server/certificates/mock_test.go +++ b/server/certificates/mock_test.go @@ -10,7 +10,8 @@ import ( func TestMockCert(t *testing.T) { db, err := database.NewTmpDB() assert.NoError(t, err) - cert := mockCert("example.com", "some error msg", "codeberg.page", db) + cert, err := mockCert("example.com", "some error msg", "codeberg.page", db) + assert.NoError(t, err) if assert.NotEmpty(t, cert) { assert.NotEmpty(t, cert.Certificate) } diff --git a/server/database/interface.go b/server/database/interface.go index 3ba3efc..56537a4 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -1,8 +1,11 @@ package database import ( - "github.com/akrylysov/pogreb" + "fmt" + + "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certificate" + "github.com/rs/zerolog/log" ) type CertDB interface { @@ -10,6 +13,61 @@ type CertDB interface { Put(name string, cert *certificate.Resource) error Get(name string) (*certificate.Resource, error) Delete(key string) error + Items(page, pageSize int) ([]*Cert, error) + // Compact deprecated // TODO: remove in next version Compact() (string, error) - Items() *pogreb.ItemIterator +} + +type Cert struct { + Domain string `xorm:"pk NOT NULL UNIQUE 'domain'"` + Created int64 `xorm:"created NOT NULL DEFAULT 0 'created'"` + Updated int64 `xorm:"updated NOT NULL DEFAULT 0 'updated'"` + ValidTill int64 `xorm:" NOT NULL DEFAULT 0 'valid_till'"` + // certificate.Resource + CertURL string `xorm:"'cert_url'"` + CertStableURL string `xorm:"'cert_stable_url'"` + PrivateKey []byte `xorm:"'private_key'"` + Certificate []byte `xorm:"'certificate'"` + IssuerCertificate []byte `xorm:"'issuer_certificate'"` +} + +func (c Cert) Raw() *certificate.Resource { + return &certificate.Resource{ + Domain: c.Domain, + CertURL: c.CertURL, + CertStableURL: c.CertStableURL, + PrivateKey: c.PrivateKey, + Certificate: c.Certificate, + IssuerCertificate: c.IssuerCertificate, + } +} + +func toCert(name string, c *certificate.Resource) (*Cert, error) { + tlsCertificates, err := certcrypto.ParsePEMBundle(c.Certificate) + if err != nil { + return nil, err + } + if len(tlsCertificates) == 0 || tlsCertificates[0] == nil { + err := fmt.Errorf("parsed cert resource has no cert") + log.Error().Err(err).Str("domain", c.Domain).Msgf("cert: %v", c) + return nil, err + } + validTill := tlsCertificates[0].NotAfter.Unix() + + // TODO: do we need this or can we just go with domain name for wildcard cert + // default *.mock cert is prefixed with '.' + if name != c.Domain && name[1:] != c.Domain && name[0] != '.' { + return nil, fmt.Errorf("domain key and cert domain not equal") + } + + return &Cert{ + Domain: c.Domain, + ValidTill: validTill, + + CertURL: c.CertURL, + CertStableURL: c.CertStableURL, + PrivateKey: c.PrivateKey, + Certificate: c.Certificate, + IssuerCertificate: c.IssuerCertificate, + }, nil } diff --git a/server/database/mock.go b/server/database/mock.go index dfe2316..7c3735e 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -5,7 +5,6 @@ import ( "time" "github.com/OrlovEvgeny/go-mcache" - "github.com/akrylysov/pogreb" "github.com/go-acme/lego/v4/certificate" ) @@ -43,8 +42,8 @@ func (p tmpDB) Compact() (string, error) { return "Truncate done", nil } -func (p tmpDB) Items() *pogreb.ItemIterator { - panic("ItemIterator not implemented for tmpDB") +func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { + return nil, fmt.Errorf("items not implemented for tmpDB") } func NewTmpDB() (CertDB, error) { diff --git a/server/database/setup.go b/server/database/pogreb.go similarity index 76% rename from server/database/setup.go rename to server/database/pogreb.go index 097c63e..9a53faf 100644 --- a/server/database/setup.go +++ b/server/database/pogreb.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/gob" + "errors" "fmt" "time" @@ -62,8 +63,32 @@ func (p aDB) Compact() (string, error) { return fmt.Sprintf("%+v", result), nil } -func (p aDB) Items() *pogreb.ItemIterator { - return p.intern.Items() +func (p aDB) Items(_, _ int) ([]*Cert, error) { + items := make([]*Cert, 0, p.intern.Count()) + iterator := p.intern.Items() + for { + key, resBytes, err := iterator.Next() + if err != nil { + if errors.Is(err, pogreb.ErrIterationDone) { + break + } + return nil, err + } + + res := &certificate.Resource{} + if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil { + return nil, err + } + + cert, err := toCert(string(key), res) + if err != nil { + return nil, err + } + + items = append(items, cert) + } + + return items, nil } var _ CertDB = &aDB{} @@ -82,7 +107,7 @@ func (p aDB) sync() { } } -func New(path string) (CertDB, error) { +func NewPogreb(path string) (CertDB, error) { if path == "" { return nil, fmt.Errorf("path not set") } diff --git a/server/database/xorm.go b/server/database/xorm.go new file mode 100644 index 0000000..68ff18c --- /dev/null +++ b/server/database/xorm.go @@ -0,0 +1,121 @@ +package database + +import ( + "errors" + "fmt" + "strings" + + "github.com/rs/zerolog/log" + + "github.com/go-acme/lego/v4/certificate" + "xorm.io/xorm" + + // register sql driver + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" +) + +var _ CertDB = xDB{} + +var ErrNotFound = errors.New("entry not found") + +type xDB struct { + engine *xorm.Engine +} + +func NewXormDB(dbType, dbConn string) (CertDB, error) { + if !supportedDriver(dbType) { + return nil, fmt.Errorf("not supported db type '%s'", dbType) + } + if dbConn == "" { + return nil, fmt.Errorf("no db connection provided") + } + + e, err := xorm.NewEngine(dbType, dbConn) + if err != nil { + return nil, err + } + + if err := e.Sync2(new(Cert)); err != nil { + return nil, fmt.Errorf("cound not sync db model :%w", err) + } + + return &xDB{ + engine: e, + }, nil +} + +func (x xDB) Close() error { + return x.engine.Close() +} + +func (x xDB) Put(domain string, cert *certificate.Resource) error { + log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db") + c, err := toCert(domain, cert) + if err != nil { + return err + } + + _, err = x.engine.Insert(c) + return err +} + +func (x xDB) Get(domain string) (*certificate.Resource, error) { + // TODO: do we need this or can we just go with domain name for wildcard cert + domain = strings.TrimPrefix(domain, ".") + + cert := new(Cert) + log.Trace().Str("domain", domain).Msg("get cert from db") + if found, err := x.engine.ID(domain).Get(cert); err != nil { + return nil, err + } else if !found { + return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, domain) + } + return cert.Raw(), nil +} + +func (x xDB) Delete(domain string) error { + log.Trace().Str("domain", domain).Msg("delete cert from db") + _, err := x.engine.ID(domain).Delete(new(Cert)) + return err +} + +func (x xDB) Compact() (string, error) { + // not needed + return "", nil +} + +// Items return al certs from db, if pageSize is 0 it does not use limit +func (x xDB) Items(page, pageSize int) ([]*Cert, error) { + // paginated return + if pageSize > 0 { + certs := make([]*Cert, 0, pageSize) + if page >= 0 { + page = 1 + } + err := x.engine.Limit(pageSize, (page-1)*pageSize).Find(&certs) + return certs, err + } + + // return all + certs := make([]*Cert, 0, 64) + err := x.engine.Find(&certs) + return certs, err +} + +// Supported database drivers +const ( + DriverSqlite = "sqlite3" + DriverMysql = "mysql" + DriverPostgres = "postgres" +) + +func supportedDriver(driver string) bool { + switch driver { + case DriverMysql, DriverPostgres, DriverSqlite: + return true + default: + return false + } +} diff --git a/server/upstream/helper.go b/server/upstream/helper.go index a84d4f0..ac0ab3f 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -17,17 +17,17 @@ func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) { // Get default branch defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(o.TargetOwner, o.TargetRepo) if err != nil { - log.Err(err).Msg("Could't fetch default branch from repository") + log.Err(err).Msg("Couldn't fetch default branch from repository") return false, err } - log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch) + log.Debug().Msgf("Successfully fetched default branch %q from Gitea", defaultBranch) o.TargetBranch = defaultBranch } timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch) if err != nil { if !errors.Is(err, gitea.ErrorNotFound) { - log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch") + log.Error().Err(err).Msg("Could not get latest commit timestamp from branch") } return false, err } @@ -36,7 +36,7 @@ func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) { return false, fmt.Errorf("empty response") } - log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp) + log.Debug().Msgf("Successfully fetched latest commit timestamp from branch: %#v", timestamp) o.BranchTimestamp = timestamp.Timestamp o.TargetBranch = timestamp.Branch return true, nil From 1b6ea4b6e1b0b1fb79f3e2887f61f92c4c00330c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 10 Feb 2023 04:33:28 +0100 Subject: [PATCH 31/62] use same version var on cli app as header --- main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 8eac1fb..6c1d0cc 100644 --- a/main.go +++ b/main.go @@ -8,15 +8,13 @@ import ( "github.com/urfave/cli/v2" "codeberg.org/codeberg/pages/cmd" + "codeberg.org/codeberg/pages/server/version" ) -// can be changed with -X on compile -var version = "dev" - func main() { app := cli.NewApp() app.Name = "pages-server" - app.Version = version + app.Version = version.Version app.Usage = "pages server" app.Action = cmd.Serve app.Flags = cmd.ServerFlags From d8d119b0b3a4d557b3c1e1fd6cecb6efc5289901 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 00:31:56 +0000 Subject: [PATCH 32/62] Fix Cache Bug (#178) error io.EOF is gracefully end of file read. so we don't need to cancel cache saving Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/178 --- cmd/main.go | 4 ++-- server/gitea/cache.go | 2 +- server/upstream/domains.go | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index af4d2ce..4f0c019 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -44,7 +44,7 @@ func Serve(ctx *cli.Context) error { } log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) - giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") + giteaRoot := ctx.String("gitea-root") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") mainDomainSuffix := ctx.String("pages-domain") @@ -68,7 +68,7 @@ func Serve(ctx *cli.Context) error { allowedCorsDomains = append(allowedCorsDomains, rawDomain) } - // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash + // Make sure MainDomain has a trailing dot if !strings.HasPrefix(mainDomainSuffix, ".") { mainDomainSuffix = "." + mainDomainSuffix } diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 85cbcde..af61edf 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -80,7 +80,7 @@ type writeCacheReader struct { func (t *writeCacheReader) Read(p []byte) (n int, err error) { n, err = t.originalReader.Read(p) - if err != nil { + if err != nil && err != io.EOF { log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) t.hasError = true } else if n > 0 { diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 5b274b6..bb4b57a 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -1,6 +1,7 @@ package upstream import ( + "errors" "strings" "time" @@ -30,8 +31,8 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, } body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig) - if err == nil || err == gitea.ErrorNotFound { - log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) + if err != nil && !errors.Is(err, gitea.ErrorNotFound) { + log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) } var domains []string From 272c7ca76fcc75e60ad9841b17f396dd5df63e6a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 01:26:21 +0000 Subject: [PATCH 33/62] Fix xorm regressions by handle wildcard certs correctly (#177) close #176 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/177 --- cmd/certs.go | 3 -- cmd/flags.go | 33 +++++++------ integration/get_test.go | 4 +- server/database/interface.go | 12 +++-- server/database/xorm.go | 45 ++++++++++++++++-- server/database/xorm_test.go | 92 ++++++++++++++++++++++++++++++++++++ server/upstream/domains.go | 2 +- 7 files changed, 162 insertions(+), 29 deletions(-) create mode 100644 server/database/xorm_test.go diff --git a/cmd/certs.go b/cmd/certs.go index 4adf076..96244b7 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -98,9 +98,6 @@ func listCerts(ctx *cli.Context) error { fmt.Printf("Domain\tValidTill\n\n") for _, cert := range items { - if cert.Domain[0] == '.' { - cert.Domain = "*" + cert.Domain - } fmt.Printf("%s\t%s\n", cert.Domain, time.Unix(cert.ValidTill, 0).Format(time.RFC3339)) diff --git a/cmd/flags.go b/cmd/flags.go index 40bb053..da3febc 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -43,6 +43,18 @@ var ( EnvVars: []string{"GITEA_API_TOKEN"}, Value: "", }, + &cli.BoolFlag{ + Name: "enable-lfs-support", + Usage: "enable lfs support, require gitea >= v1.17.0 as backend", + EnvVars: []string{"ENABLE_LFS_SUPPORT"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "enable-symlink-support", + Usage: "follow symlinks if enabled, require gitea >= v1.18.0 as backend", + EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, + Value: true, + }, // ########################### // ### Page Server Domains ### @@ -73,7 +85,9 @@ var ( Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", }, - // Server + // ######################### + // ### Page Server Setup ### + // ######################### &cli.StringFlag{ Name: "host", Usage: "specifies host of listening address", @@ -91,19 +105,6 @@ var ( // TODO: desc EnvVars: []string{"ENABLE_HTTP_SERVER"}, }, - // Server Options - &cli.BoolFlag{ - Name: "enable-lfs-support", - Usage: "enable lfs support, require gitea v1.17.0 as backend", - EnvVars: []string{"ENABLE_LFS_SUPPORT"}, - Value: true, - }, - &cli.BoolFlag{ - Name: "enable-symlink-support", - Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend", - EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, - Value: true, - }, &cli.StringFlag{ Name: "log-level", Value: "warn", @@ -111,7 +112,9 @@ var ( EnvVars: []string{"LOG_LEVEL"}, }, - // ACME + // ############################ + // ### ACME Client Settings ### + // ############################ &cli.StringFlag{ Name: "acme-api-endpoint", EnvVars: []string{"ACME_API"}, diff --git a/integration/get_test.go b/integration/get_test.go index 6dd57bc..55e6d12 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -20,7 +20,9 @@ func TestGetRedirect(t *testing.T) { log.Println("=== TestGetRedirect ===") // test custom domain redirect resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430") - assert.NoError(t, err) + if !assert.NoError(t, err) { + t.FailNow() + } if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) { t.FailNow() } diff --git a/server/database/interface.go b/server/database/interface.go index 56537a4..a0edc91 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -54,10 +54,14 @@ func toCert(name string, c *certificate.Resource) (*Cert, error) { } validTill := tlsCertificates[0].NotAfter.Unix() - // TODO: do we need this or can we just go with domain name for wildcard cert - // default *.mock cert is prefixed with '.' - if name != c.Domain && name[1:] != c.Domain && name[0] != '.' { - return nil, fmt.Errorf("domain key and cert domain not equal") + // handle wildcard certs + if name[:1] == "." { + name = "*" + name + } + if name != c.Domain { + err := fmt.Errorf("domain key '%s' and cert domain '%s' not equal", name, c.Domain) + log.Error().Err(err).Msg("toCert conversion did discover mismatch") + // TODO: fail hard: return nil, err } return &Cert{ diff --git a/server/database/xorm.go b/server/database/xorm.go index 68ff18c..a14f887 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -3,7 +3,6 @@ package database import ( "errors" "fmt" - "strings" "github.com/rs/zerolog/log" @@ -52,18 +51,38 @@ func (x xDB) Close() error { func (x xDB) Put(domain string, cert *certificate.Resource) error { log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db") + + domain = integrationTestReplacements(domain) c, err := toCert(domain, cert) if err != nil { return err } - _, err = x.engine.Insert(c) - return err + sess := x.engine.NewSession() + if err := sess.Begin(); err != nil { + return err + } + defer sess.Close() + + if exist, _ := sess.ID(c.Domain).Exist(); exist { + if _, err := sess.ID(c.Domain).Update(c); err != nil { + return err + } + } else { + if _, err = sess.Insert(c); err != nil { + return err + } + } + + return sess.Commit() } func (x xDB) Get(domain string) (*certificate.Resource, error) { - // TODO: do we need this or can we just go with domain name for wildcard cert - domain = strings.TrimPrefix(domain, ".") + // handle wildcard certs + if domain[:1] == "." { + domain = "*" + domain + } + domain = integrationTestReplacements(domain) cert := new(Cert) log.Trace().Str("domain", domain).Msg("get cert from db") @@ -76,6 +95,12 @@ func (x xDB) Get(domain string) (*certificate.Resource, error) { } func (x xDB) Delete(domain string) error { + // handle wildcard certs + if domain[:1] == "." { + domain = "*" + domain + } + domain = integrationTestReplacements(domain) + log.Trace().Str("domain", domain).Msg("delete cert from db") _, err := x.engine.ID(domain).Delete(new(Cert)) return err @@ -119,3 +144,13 @@ func supportedDriver(driver string) bool { return false } } + +// integrationTestReplacements is needed because integration tests use a single domain cert, +// while production use a wildcard cert +// TODO: find a better way to handle this +func integrationTestReplacements(domainKey string) string { + if domainKey == "*.localhost.mock.directory" { + return "localhost.mock.directory" + } + return domainKey +} diff --git a/server/database/xorm_test.go b/server/database/xorm_test.go new file mode 100644 index 0000000..9c032ee --- /dev/null +++ b/server/database/xorm_test.go @@ -0,0 +1,92 @@ +package database + +import ( + "errors" + "testing" + + "github.com/go-acme/lego/v4/certificate" + "github.com/stretchr/testify/assert" + "xorm.io/xorm" +) + +func newTestDB(t *testing.T) *xDB { + e, err := xorm.NewEngine("sqlite3", ":memory:") + assert.NoError(t, err) + assert.NoError(t, e.Sync2(new(Cert))) + return &xDB{engine: e} +} + +func TestSanitizeWildcardCerts(t *testing.T) { + certDB := newTestDB(t) + + _, err := certDB.Get(".not.found") + assert.True(t, errors.Is(err, ErrNotFound)) + + // TODO: cert key and domain mismatch are don not fail hard jet + // https://codeberg.org/Codeberg/pages-server/src/commit/d8595cee882e53d7f44f1ddc4ef8a1f7b8f31d8d/server/database/interface.go#L64 + // + // assert.Error(t, certDB.Put(".wildcard.de", &certificate.Resource{ + // Domain: "*.localhost.mock.directory", + // Certificate: localhost_mock_directory_certificate, + // })) + + // insert new wildcard cert + assert.NoError(t, certDB.Put(".wildcard.de", &certificate.Resource{ + Domain: "*.wildcard.de", + Certificate: localhost_mock_directory_certificate, + })) + + // update existing cert + assert.Error(t, certDB.Put(".wildcard.de", &certificate.Resource{ + Domain: "*.wildcard.de", + Certificate: localhost_mock_directory_certificate, + })) + + c1, err := certDB.Get(".wildcard.de") + assert.NoError(t, err) + c2, err := certDB.Get("*.wildcard.de") + assert.NoError(t, err) + assert.EqualValues(t, c1, c2) +} + +var localhost_mock_directory_certificate = []byte(`-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgIIJyBaXHmLk6gwDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE +AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSA0OWE0ZmIwHhcNMjMwMjEwMDEwOTA2 +WhcNMjgwMjEwMDEwOTA2WjAjMSEwHwYDVQQDExhsb2NhbGhvc3QubW9jay5kaXJl +Y3RvcnkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDIU/CjzS7t62Gj +neEMqvP7sn99ULT7AEUzEfWL05fWG2z714qcUg1hXkZLgdVDgmsCpplyddip7+2t +ZH/9rLPLMqJphzvOL4CF6jDLbeifETtKyjnt9vUZFnnNWcP3tu8lo8iYSl08qsUI +Pp/hiEriAQzCDjTbR5m9xUPNPYqxzcS4ALzmmCX9Qfc4CuuhMkdv2G4TT7rylWrA +SCSRPnGjeA7pCByfNrO/uXbxmzl3sMO3k5sqgMkx1QIHEN412V8+vtx88mt2sM6k +xjzGZWWKXlRq+oufIKX9KPplhsCjMH6E3VNAzgOPYDqXagtUcGmLWghURltO8Mt2 +zwM6OgjjAgMBAAGjgaUwgaIwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsG +AQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBSMQvlJ1755 +sarf8i1KNqj7s5o/aDAfBgNVHSMEGDAWgBTcZcxJMhWdP7MecHCCpNkFURC/YzAj +BgNVHREEHDAaghhsb2NhbGhvc3QubW9jay5kaXJlY3RvcnkwDQYJKoZIhvcNAQEL +BQADggEBACcd7TT28OWwzQN2PcH0aG38JX5Wp2iOS/unDCfWjNAztXHW7nBDMxza +VtyebkJfccexpuVuOsjOX+bww0vtEYIvKX3/GbkhogksBrNkE0sJZtMnZWMR33wa +YxAy/kJBTmLi02r8fX9ZhwjldStHKBav4USuP7DXZjrgX7LFQhR4LIDrPaYqQRZ8 +ltC3mM9LDQ9rQyIFP5cSBMO3RUAm4I8JyLoOdb/9G2uxjHr7r6eG1g8DmLYSKBsQ +mWGQDOYgR3cGltDe2yMxM++yHY+b1uhxGOWMrDA1+1k7yI19LL8Ifi2FMovDfu/X +JxYk1NNNtdctwaYJFenmGQvDaIq1KgE= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDUDCCAjigAwIBAgIIKBJ7IIA6W1swDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVUGViYmxlIFJvb3QgQ0EgNTdmZjE2MCAXDTIzMDIwOTA1MzMxMloYDzIwNTMw +MjA5MDUzMzEyWjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDQ5 +YTRmYjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANOvlqRx8SXQFWo2 +gFCiXxls53eENcyr8+meFyjgnS853eEvplaPxoa2MREKd+ZYxM8EMMfj2XGvR3UI +aqR5QyLQ9ihuRqvQo4fG91usBHgH+vDbGPdMX8gDmm9HgnmtOVhSKJU+M2jfE1SW +UuWB9xOa3LMreTXbTNfZEMoXf+GcWZMbx5WPgEga3DvfmV+RsfNvB55eD7YAyZgF +ZnQ3Dskmnxxlkz0EGgd7rqhFHHNB9jARlL22gITADwoWZidlr3ciM9DISymRKQ0c +mRN15fQjNWdtuREgJlpXecbYQMGhdTOmFrqdHkveD1o63rGSC4z+s/APV6xIbcRp +aNpO7L8CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNxlzEky +FZ0/sx5wcIKk2QVREL9jMB8GA1UdIwQYMBaAFOqfkm9rebIz4z0SDIKW5edLg5JM +MA0GCSqGSIb3DQEBCwUAA4IBAQBRG9AHEnyj2fKzVDDbQaKHjAF5jh0gwyHoIeRK +FkP9mQNSWxhvPWI0tK/E49LopzmVuzSbDd5kZsaii73rAs6f6Rf9W5veo3AFSEad +stM+Zv0f2vWB38nuvkoCRLXMX+QUeuL65rKxdEpyArBju4L3/PqAZRgMLcrH+ak8 +nvw5RdAq+Km/ZWyJgGikK6cfMmh91YALCDFnoWUWrCjkBaBFKrG59ONV9f0IQX07 +aNfFXFCF5l466xw9dHjw5iaFib10cpY3iq4kyPYIMs6uaewkCtxWKKjiozM4g4w3 +HqwyUyZ52WUJOJ/6G9DJLDtN3fgGR+IAp8BhYd5CqOscnt3h +-----END CERTIFICATE-----`) diff --git a/server/upstream/domains.go b/server/upstream/domains.go index bb4b57a..eb30394 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -49,7 +49,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, } } - // Add [owner].[pages-domain] as valid domnain. + // Add [owner].[pages-domain] as valid domain. domains = append(domains, o.TargetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { valid = true From fd643d15f0f44081a6469be7613b1162b8a29258 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 02:04:57 +0000 Subject: [PATCH 34/62] Drop: pogreb support (#175) followup of #173 close #95 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/175 --- .woodpecker.yml | 13 --- Justfile | 2 +- cmd/certs.go | 61 +------------ cmd/flags.go | 9 +- cmd/setup.go | 26 +----- go.mod | 1 - go.sum | 2 - server/certificates/certificates.go | 8 -- server/database/interface.go | 2 - server/database/mock.go | 5 -- server/database/pogreb.go | 134 ---------------------------- server/database/xorm.go | 5 -- 12 files changed, 6 insertions(+), 262 deletions(-) delete mode 100644 server/database/pogreb.go diff --git a/.woodpecker.yml b/.woodpecker.yml index ba44f82..5b07053 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -65,19 +65,6 @@ pipeline: - RAW_DOMAIN=raw.localhost.mock.directory - PORT=4430 - # TODO: remove in next version - integration-tests-legacy: - group: test - image: codeberg.org/6543/docker-images/golang_just - commands: - - just integration - environment: - - ACME_API=https://acme.mock.directory - - PAGES_DOMAIN=localhost.mock.directory - - RAW_DOMAIN=raw.localhost.mock.directory - - PORT=4430 - - DB_TYPE= - release: image: plugins/gitea-release settings: diff --git a/Justfile b/Justfile index 683222d..0db7845 100644 --- a/Justfile +++ b/Justfile @@ -27,7 +27,7 @@ fmt: tool-gofumpt clean: go clean ./... - rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json + rm -rf build/ integration/certs.sqlite integration/acme-account.json tool-golangci: @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \ diff --git a/cmd/certs.go b/cmd/certs.go index 96244b7..6012b6e 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -4,11 +4,7 @@ import ( "fmt" "time" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/database" ) var Certs = &cli.Command{ @@ -25,63 +21,8 @@ var Certs = &cli.Command{ Usage: "remove a certificate from the database", Action: removeCert, }, - { - Name: "migrate", - Usage: "migrate from \"pogreb\" driver to dbms driver", - Action: migrateCerts, - }, }, - Flags: append(CertStorageFlags, []cli.Flag{ - &cli.BoolFlag{ - Name: "verbose", - Usage: "print trace info", - EnvVars: []string{"VERBOSE"}, - Value: false, - }, - }...), -} - -func migrateCerts(ctx *cli.Context) error { - dbType := ctx.String("db-type") - if dbType == "" { - dbType = "sqlite3" - } - dbConn := ctx.String("db-conn") - dbPogrebConn := ctx.String("db-pogreb") - verbose := ctx.Bool("verbose") - - log.Level(zerolog.InfoLevel) - if verbose { - log.Level(zerolog.TraceLevel) - } - - xormDB, err := database.NewXormDB(dbType, dbConn) - if err != nil { - return fmt.Errorf("could not connect to database: %w", err) - } - defer xormDB.Close() - - pogrebDB, err := database.NewPogreb(dbPogrebConn) - if err != nil { - return fmt.Errorf("could not open database: %w", err) - } - defer pogrebDB.Close() - - fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn) - - certs, err := pogrebDB.Items(0, 0) - if err != nil { - return err - } - - for _, cert := range certs { - if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil { - return err - } - } - - fmt.Println("... done") - return nil + Flags: CertStorageFlags, } func listCerts(ctx *cli.Context) error { diff --git a/cmd/flags.go b/cmd/flags.go index da3febc..a37b179 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -6,16 +6,9 @@ import ( var ( CertStorageFlags = []cli.Flag{ - &cli.StringFlag{ - // TODO: remove in next version - // DEPRICATED - Name: "db-pogreb", - Value: "key-database.pogreb", - EnvVars: []string{"DB_POGREB"}, - }, &cli.StringFlag{ Name: "db-type", - Value: "", // TODO: "sqlite3" in next version + Value: "sqlite3", EnvVars: []string{"DB_TYPE"}, }, &cli.StringFlag{ diff --git a/cmd/setup.go b/cmd/setup.go index 3d1d6ee..bccba03 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -10,29 +10,9 @@ import ( ) func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { - if ctx.String("db-type") != "" { - log.Trace().Msg("use xorm mode") - certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to database: %w", err) - } - } else { - // TODO: remove in next version - fmt.Println(` -###################### -## W A R N I N G !!! # -###################### - -You use "pogreb" witch is deprecated and will be removed in the next version. -Please switch to sqlite, mysql or postgres !!! - -The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`) - log.Error().Msg("depricated \"pogreb\" used\n") - - certDB, err = database.NewPogreb(ctx.String("db-pogreb")) - if err != nil { - return nil, nil, fmt.Errorf("could not create database: %w", err) - } + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) } closeFn = func() { diff --git a/go.mod b/go.mod index 2e75c8e..944e2ad 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a - github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 github.com/go-sql-driver/mysql v1.6.0 github.com/joho/godotenv v1.4.0 diff --git a/go.sum b/go.sum index c1042c9..b5a7568 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,6 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= -github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= -github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 555539e..23fad17 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -510,14 +510,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } } log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount) - - // compact the database - msg, err := certDB.Compact() - if err != nil { - log.Error().Err(err).Msg("Compacting key database failed") - } else { - log.Debug().Msgf("Compacted key database: %s", msg) - } } // update main cert diff --git a/server/database/interface.go b/server/database/interface.go index a0edc91..92068c5 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -14,8 +14,6 @@ type CertDB interface { Get(name string) (*certificate.Resource, error) Delete(key string) error Items(page, pageSize int) ([]*Cert, error) - // Compact deprecated // TODO: remove in next version - Compact() (string, error) } type Cert struct { diff --git a/server/database/mock.go b/server/database/mock.go index 7c3735e..6148287 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -37,11 +37,6 @@ func (p tmpDB) Delete(key string) error { return nil } -func (p tmpDB) Compact() (string, error) { - p.intern.Truncate() - return "Truncate done", nil -} - func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { return nil, fmt.Errorf("items not implemented for tmpDB") } diff --git a/server/database/pogreb.go b/server/database/pogreb.go deleted file mode 100644 index 9a53faf..0000000 --- a/server/database/pogreb.go +++ /dev/null @@ -1,134 +0,0 @@ -package database - -import ( - "bytes" - "context" - "encoding/gob" - "errors" - "fmt" - "time" - - "github.com/akrylysov/pogreb" - "github.com/akrylysov/pogreb/fs" - "github.com/go-acme/lego/v4/certificate" - "github.com/rs/zerolog/log" -) - -var _ CertDB = aDB{} - -type aDB struct { - ctx context.Context - cancel context.CancelFunc - intern *pogreb.DB - syncInterval time.Duration -} - -func (p aDB) Close() error { - p.cancel() - return p.intern.Sync() -} - -func (p aDB) Put(name string, cert *certificate.Resource) error { - var resGob bytes.Buffer - if err := gob.NewEncoder(&resGob).Encode(cert); err != nil { - return err - } - return p.intern.Put([]byte(name), resGob.Bytes()) -} - -func (p aDB) Get(name string) (*certificate.Resource, error) { - cert := &certificate.Resource{} - resBytes, err := p.intern.Get([]byte(name)) - if err != nil { - return nil, err - } - if resBytes == nil { - return nil, nil - } - if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil { - return nil, err - } - return cert, nil -} - -func (p aDB) Delete(key string) error { - return p.intern.Delete([]byte(key)) -} - -func (p aDB) Compact() (string, error) { - result, err := p.intern.Compact() - if err != nil { - return "", err - } - return fmt.Sprintf("%+v", result), nil -} - -func (p aDB) Items(_, _ int) ([]*Cert, error) { - items := make([]*Cert, 0, p.intern.Count()) - iterator := p.intern.Items() - for { - key, resBytes, err := iterator.Next() - if err != nil { - if errors.Is(err, pogreb.ErrIterationDone) { - break - } - return nil, err - } - - res := &certificate.Resource{} - if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil { - return nil, err - } - - cert, err := toCert(string(key), res) - if err != nil { - return nil, err - } - - items = append(items, cert) - } - - return items, nil -} - -var _ CertDB = &aDB{} - -func (p aDB) sync() { - for { - err := p.intern.Sync() - if err != nil { - log.Error().Err(err).Msg("Syncing cert database failed") - } - select { - case <-p.ctx.Done(): - return - case <-time.After(p.syncInterval): - } - } -} - -func NewPogreb(path string) (CertDB, error) { - if path == "" { - return nil, fmt.Errorf("path not set") - } - db, err := pogreb.Open(path, &pogreb.Options{ - BackgroundSyncInterval: 30 * time.Second, - BackgroundCompactionInterval: 6 * time.Hour, - FileSystem: fs.OSMMap, - }) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithCancel(context.Background()) - result := &aDB{ - ctx: ctx, - cancel: cancel, - intern: db, - syncInterval: 5 * time.Minute, - } - - go result.sync() - - return result, nil -} diff --git a/server/database/xorm.go b/server/database/xorm.go index a14f887..4b43cbb 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -106,11 +106,6 @@ func (x xDB) Delete(domain string) error { return err } -func (x xDB) Compact() (string, error) { - // not needed - return "", nil -} - // Items return al certs from db, if pageSize is 0 it does not use limit func (x xDB) Items(page, pageSize int) ([]*Cert, error) { // paginated return From 5753f7136deb4cc29bae91640f0ae1ef1472b8f6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 02:29:08 +0000 Subject: [PATCH 35/62] Move acmeClient creation into own file & struct (#179) get rid of gobal vars and make make functions with less args :) tldr: collect funcs and create a own ACME client to manage that stuff Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/179 --- cmd/flags.go | 10 +- cmd/main.go | 35 +--- cmd/setup.go | 34 +++ server/certificates/acme_client.go | 95 +++++++++ server/certificates/acme_config.go | 100 +++++++++ server/certificates/cached_challengers.go | 41 ++++ server/certificates/certificates.go | 242 +++------------------- server/certificates/mock_test.go | 3 +- 8 files changed, 323 insertions(+), 237 deletions(-) create mode 100644 server/certificates/acme_client.go create mode 100644 server/certificates/acme_config.go create mode 100644 server/certificates/cached_challengers.go diff --git a/cmd/flags.go b/cmd/flags.go index a37b179..8052421 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -140,9 +140,15 @@ var ( EnvVars: []string{"ACME_EAB_HMAC"}, }, &cli.StringFlag{ - Name: "dns-provider", - // TODO: Usage + Name: "dns-provider", + Usage: "Use DNS-Challenge for main domain\n\nRead more at: https://go-acme.github.io/lego/dns/", EnvVars: []string{"DNS_PROVIDER"}, }, + &cli.StringFlag{ + Name: "acme-account-config", + Usage: "json file of acme account", + Value: "acme-account.json", + EnvVars: []string{"ACME_ACCOUNT_CONFIG"}, + }, }...) ) diff --git a/cmd/main.go b/cmd/main.go index 4f0c019..a1c3b97 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,7 +3,6 @@ package cmd import ( "context" "crypto/tls" - "errors" "fmt" "net" "net/http" @@ -52,17 +51,6 @@ func Serve(ctx *cli.Context) error { listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") - acmeAPI := ctx.String("acme-api-endpoint") - acmeMail := ctx.String("acme-email") - acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") - acmeAcceptTerms := ctx.Bool("acme-accept-terms") - acmeEabKID := ctx.String("acme-eab-kid") - acmeEabHmac := ctx.String("acme-eab-hmac") - dnsProvider := ctx.String("dns-provider") - if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { - return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory") - } - allowedCorsDomains := AllowedCorsDomains if rawDomain != "" { allowedCorsDomains = append(allowedCorsDomains, rawDomain) @@ -94,6 +82,15 @@ func Serve(ctx *cli.Context) error { return fmt.Errorf("could not create new gitea client: %v", err) } + acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache) + if err != nil { + return err + } + + if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil { + return err + } + // Create handler based on settings httpsHandler := handler.Handler(mainDomainSuffix, rawDomain, giteaClient, @@ -112,24 +109,14 @@ func Serve(ctx *cli.Context) error { listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaClient, - dnsProvider, - acmeUseRateLimits, + acmeClient, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB)) - acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) - if err != nil { - return err - } - - if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil { - return err - } - interval := 12 * time.Hour certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB) + go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB) if enableHTTPServer { go func() { diff --git a/cmd/setup.go b/cmd/setup.go index bccba03..bb9f8cb 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -1,14 +1,19 @@ package cmd import ( + "errors" "fmt" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" ) +var ErrAcmeMissConfig = errors.New("ACME client has wrong config") + func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) if err != nil { @@ -23,3 +28,32 @@ func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err e return certDB, closeFn, nil } + +func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) { + acmeAPI := ctx.String("acme-api-endpoint") + acmeMail := ctx.String("acme-email") + acmeEabHmac := ctx.String("acme-eab-hmac") + acmeEabKID := ctx.String("acme-eab-kid") + acmeAcceptTerms := ctx.Bool("acme-accept-terms") + dnsProvider := ctx.String("dns-provider") + acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") + acmeAccountConf := ctx.String("acme-account-config") + + // check config + if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + } + + return certificates.NewAcmeClient( + acmeAccountConf, + acmeAPI, + acmeMail, + acmeEabHmac, + acmeEabKID, + dnsProvider, + acmeAcceptTerms, + enableHTTPServer, + acmeUseRateLimits, + challengeCache, + ) +} diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go new file mode 100644 index 0000000..7737396 --- /dev/null +++ b/server/certificates/acme_client.go @@ -0,0 +1,95 @@ +package certificates + +import ( + "fmt" + "sync" + "time" + + "github.com/go-acme/lego/v4/lego" + "github.com/go-acme/lego/v4/providers/dns" + "github.com/reugn/equalizer" + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/server/cache" +) + +type AcmeClient struct { + legoClient *lego.Client + dnsChallengerLegoClient *lego.Client + + obtainLocks sync.Map + + acmeUseRateLimits bool + + // limiter + acmeClientOrderLimit *equalizer.TokenBucket + acmeClientRequestLimit *equalizer.TokenBucket + acmeClientFailLimit *equalizer.TokenBucket + acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket +} + +func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) { + acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) + if err != nil { + return nil, err + } + + acmeClient, err := lego.NewClient(acmeConfig) + if err != nil { + log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only") + } else { + err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) + if err != nil { + log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") + } + if enableHTTPServer { + err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache}) + if err != nil { + log.Error().Err(err).Msg("Can't create HTTP-01 provider") + } + } + } + + mainDomainAcmeClient, err := lego.NewClient(acmeConfig) + if err != nil { + log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") + } else { + if dnsProvider == "" { + // using mock server, don't use wildcard certs + err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) + if err != nil { + log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") + } + } else { + // use DNS-Challenge https://go-acme.github.io/lego/dns/ + provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) + if err != nil { + return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err) + } + if err := mainDomainAcmeClient.Challenge.SetDNS01Provider(provider); err != nil { + return nil, fmt.Errorf("can not create DNS-01 provider: %w", err) + } + } + } + + return &AcmeClient{ + legoClient: acmeClient, + dnsChallengerLegoClient: mainDomainAcmeClient, + + acmeUseRateLimits: acmeUseRateLimits, + + obtainLocks: sync.Map{}, + + // limiter + + // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes + // TODO: when this is used a lot, we probably have to think of a somewhat better solution? + acmeClientOrderLimit: equalizer.NewTokenBucket(25, 15*time.Minute), + // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) + acmeClientRequestLimit: equalizer.NewTokenBucket(5, 1*time.Second), + // rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/ + acmeClientFailLimit: equalizer.NewTokenBucket(5, 1*time.Hour), + // checkUserLimit() use this to rate als per user + acmeClientCertificateLimitPerUser: map[string]*equalizer.TokenBucket{}, + }, nil +} diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go new file mode 100644 index 0000000..69568e6 --- /dev/null +++ b/server/certificates/acme_config.go @@ -0,0 +1,100 @@ +package certificates + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/json" + "fmt" + "os" + + "github.com/go-acme/lego/v4/certcrypto" + "github.com/go-acme/lego/v4/lego" + "github.com/go-acme/lego/v4/registration" + "github.com/rs/zerolog/log" +) + +func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { + var myAcmeAccount AcmeAccount + var myAcmeConfig *lego.Config + + if account, err := os.ReadFile(configFile); err == nil { + log.Info().Msgf("found existing acme account config file '%s'", configFile) + if err := json.Unmarshal(account, &myAcmeAccount); err != nil { + return nil, err + } + myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) + if err != nil { + return nil, err + } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) + myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 + + // Validate Config + _, err := lego.NewClient(myAcmeConfig) + if err != nil { + log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate") + return nil, fmt.Errorf("acme config validation failed: %w", err) + } + return myAcmeConfig, nil + } else if !os.IsNotExist(err) { + return nil, err + } + + log.Info().Msgf("no existing acme account config found, try to create a new one") + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + myAcmeAccount = AcmeAccount{ + Email: acmeMail, + Key: privateKey, + KeyPEM: string(certcrypto.PEMEncode(privateKey)), + } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) + myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 + tempClient, err := lego.NewClient(myAcmeConfig) + if err != nil { + log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") + } else { + // accept terms & log in to EAB + if acmeEabKID == "" || acmeEabHmac == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) + if err != nil { + log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") + } else { + myAcmeAccount.Registration = reg + } + } else { + reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ + TermsOfServiceAgreed: acmeAcceptTerms, + Kid: acmeEabKID, + HmacEncoded: acmeEabHmac, + }) + if err != nil { + log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") + } else { + myAcmeAccount.Registration = reg + } + } + + if myAcmeAccount.Registration != nil { + acmeAccountJSON, err := json.Marshal(myAcmeAccount) + if err != nil { + log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits") + select {} + } + log.Info().Msgf("new acme account created. write to config file '%s'", configFile) + err = os.WriteFile(configFile, acmeAccountJSON, 0o600) + if err != nil { + log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits") + select {} + } + } + } + + return myAcmeConfig, nil +} diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go new file mode 100644 index 0000000..6ce6e67 --- /dev/null +++ b/server/certificates/cached_challengers.go @@ -0,0 +1,41 @@ +package certificates + +import ( + "time" + + "github.com/go-acme/lego/v4/challenge" + + "codeberg.org/codeberg/pages/server/cache" +) + +type AcmeTLSChallengeProvider struct { + challengeCache cache.SetGetKey +} + +// make sure AcmeTLSChallengeProvider match Provider interface +var _ challenge.Provider = AcmeTLSChallengeProvider{} + +func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { + return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) +} + +func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { + a.challengeCache.Remove(domain) + return nil +} + +type AcmeHTTPChallengeProvider struct { + challengeCache cache.SetGetKey +} + +// make sure AcmeHTTPChallengeProvider match Provider interface +var _ challenge.Provider = AcmeHTTPChallengeProvider{} + +func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { + return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) +} + +func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { + a.challengeCache.Remove(domain + "/" + token) + return nil +} diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 23fad17..3ea440f 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -2,27 +2,18 @@ package certificates import ( "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" - "os" "strconv" "strings" - "sync" "time" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certificate" - "github.com/go-acme/lego/v4/challenge" "github.com/go-acme/lego/v4/challenge/tlsalpn01" "github.com/go-acme/lego/v4/lego" - "github.com/go-acme/lego/v4/providers/dns" - "github.com/go-acme/lego/v4/registration" "github.com/reugn/equalizer" "github.com/rs/zerolog/log" @@ -33,11 +24,12 @@ import ( "codeberg.org/codeberg/pages/server/upstream" ) +var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours") + // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, - dnsProvider string, - acmeUseRateLimits bool, + acmeClient *AcmeClient, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, certDB database.CertDB, ) *tls.Config { @@ -100,7 +92,7 @@ func TLSConfig(mainDomainSuffix string, var tlsCertificate *tls.Certificate var err error - if tlsCertificate, err = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); err != nil { + if tlsCertificate, err = acmeClient.retrieveCertFromDB(sni, mainDomainSuffix, false, certDB); err != nil { // request a new certificate if strings.EqualFold(sni, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") @@ -110,7 +102,7 @@ func TLSConfig(mainDomainSuffix string, return nil, fmt.Errorf("won't request certificate for %q", sni) } - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{sni}, nil, targetOwner, false, mainDomainSuffix, certDB) if err != nil { return nil, err } @@ -141,67 +133,20 @@ func TLSConfig(mainDomainSuffix string, } } -func checkUserLimit(user string) error { - userLimit, ok := acmeClientCertificateLimitPerUser[user] +func (c *AcmeClient) checkUserLimit(user string) error { + userLimit, ok := c.acmeClientCertificateLimitPerUser[user] if !ok { - // Each Codeberg user can only add 10 new domains per day. + // Each user can only add 10 new domains per day. userLimit = equalizer.NewTokenBucket(10, time.Hour*24) - acmeClientCertificateLimitPerUser[user] = userLimit + c.acmeClientCertificateLimitPerUser[user] = userLimit } if !userLimit.Ask() { - return errors.New("rate limit exceeded: 10 certificates per user per 24 hours") + return fmt.Errorf("user '%s' error: %w", user, ErrUserRateLimitExceeded) } return nil } -var ( - acmeClient, mainDomainAcmeClient *lego.Client - acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} -) - -// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes -// TODO: when this is used a lot, we probably have to think of a somewhat better solution? -var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute) - -// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) -var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second) - -// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/ -var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour) - -type AcmeTLSChallengeProvider struct { - challengeCache cache.SetGetKey -} - -// make sure AcmeTLSChallengeProvider match Provider interface -var _ challenge.Provider = AcmeTLSChallengeProvider{} - -func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { - return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) -} - -func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { - a.challengeCache.Remove(domain) - return nil -} - -type AcmeHTTPChallengeProvider struct { - challengeCache cache.SetGetKey -} - -// make sure AcmeHTTPChallengeProvider match Provider interface -var _ challenge.Provider = AcmeHTTPChallengeProvider{} - -func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { - return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) -} - -func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { - a.challengeCache.Remove(domain + "/" + token) - return nil -} - -func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (*tls.Certificate, error) { +func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProvider bool, certDB database.CertDB) (*tls.Certificate, error) { // parse certificate from database res, err := certDB.Get(sni) if err != nil { @@ -235,7 +180,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi // TODO: make a queue ? go (func() { res.CSR = nil // acme client doesn't like CSR to be set - if _, err := obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB); err != nil { + if _, err := c.obtainCert(c.legoClient, []string{sni}, res, "", useDnsProvider, mainDomainSuffix, certDB); err != nil { log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err) } })() @@ -245,28 +190,26 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi return &tlsCertificate, nil } -var obtainLocks = sync.Map{} - -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (*tls.Certificate, error) { +func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { + if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] } // lock to avoid simultaneous requests - _, working := obtainLocks.LoadOrStore(name, struct{}{}) + _, working := c.obtainLocks.LoadOrStore(name, struct{}{}) if working { for working { time.Sleep(100 * time.Millisecond) - _, working = obtainLocks.Load(name) + _, working = c.obtainLocks.Load(name) } - cert, err := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + cert, err := c.retrieveCertFromDB(name, mainDomainSuffix, useDnsProvider, keyDatabase) if err != nil { return nil, fmt.Errorf("certificate failed in synchronous request: %w", err) } return cert, nil } - defer obtainLocks.Delete(name) + defer c.obtainLocks.Delete(name) if acmeClient == nil { return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) @@ -276,29 +219,29 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re var res *certificate.Resource var err error if renew != nil && renew.CertURL != "" { - if acmeUseRateLimits { - acmeClientRequestLimit.Take() + if c.acmeUseRateLimits { + c.acmeClientRequestLimit.Take() } log.Debug().Msgf("Renewing certificate for: %v", domains) res, err = acmeClient.Certificate.Renew(*renew, true, false, "") if err != nil { log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains) - if acmeUseRateLimits { - acmeClientFailLimit.Take() + if c.acmeUseRateLimits { + c.acmeClientFailLimit.Take() } res = nil } } if res == nil { if user != "" { - if err := checkUserLimit(user); err != nil { + if err := c.checkUserLimit(user); err != nil { return nil, err } } - if acmeUseRateLimits { - acmeClientOrderLimit.Take() - acmeClientRequestLimit.Take() + if c.acmeUseRateLimits { + c.acmeClientOrderLimit.Take() + c.acmeClientRequestLimit.Take() } log.Debug().Msgf("Re-requesting new certificate for %v", domains) res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{ @@ -306,8 +249,8 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re Bundle: true, MustStaple: false, }) - if acmeUseRateLimits && err != nil { - acmeClientFailLimit.Take() + if c.acmeUseRateLimits && err != nil { + c.acmeClientFailLimit.Take() } } if err != nil { @@ -349,136 +292,15 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re return &tlsCertificate, nil } -func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { - // TODO: make it a config flag - const configFile = "acme-account.json" - var myAcmeAccount AcmeAccount - var myAcmeConfig *lego.Config - - if account, err := os.ReadFile(configFile); err == nil { - if err := json.Unmarshal(account, &myAcmeAccount); err != nil { - return nil, err - } - myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) - if err != nil { - return nil, err - } - myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI - myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 - - // Validate Config - _, err := lego.NewClient(myAcmeConfig) - if err != nil { - // TODO: should we fail hard instead? - log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") - } - return myAcmeConfig, nil - } else if !os.IsNotExist(err) { - return nil, err - } - - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return nil, err - } - myAcmeAccount = AcmeAccount{ - Email: acmeMail, - Key: privateKey, - KeyPEM: string(certcrypto.PEMEncode(privateKey)), - } - myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI - myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 - tempClient, err := lego.NewClient(myAcmeConfig) - if err != nil { - log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") - } else { - // accept terms & log in to EAB - if acmeEabKID == "" || acmeEabHmac == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) - if err != nil { - log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") - } else { - myAcmeAccount.Registration = reg - } - } else { - reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: acmeAcceptTerms, - Kid: acmeEabKID, - HmacEncoded: acmeEabHmac, - }) - if err != nil { - log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") - } else { - myAcmeAccount.Registration = reg - } - } - - if myAcmeAccount.Registration != nil { - acmeAccountJSON, err := json.Marshal(myAcmeAccount) - if err != nil { - log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits") - select {} - } - err = os.WriteFile(configFile, acmeAccountJSON, 0o600) - if err != nil { - log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits") - select {} - } - } - } - - return myAcmeConfig, nil -} - -func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { +func SetupMainDomainCertificates(mainDomainSuffix string, acmeClient *AcmeClient, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits mainCertBytes, err := certDB.Get(mainDomainSuffix) if err != nil && !errors.Is(err, database.ErrNotFound) { return fmt.Errorf("cert database is not working: %w", err) } - acmeClient, err = lego.NewClient(acmeConfig) - if err != nil { - log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only") - } else { - err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) - if err != nil { - log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") - } - if enableHTTPServer { - err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache}) - if err != nil { - log.Error().Err(err).Msg("Can't create HTTP-01 provider") - } - } - } - - mainDomainAcmeClient, err = lego.NewClient(acmeConfig) - if err != nil { - log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") - } else { - if dnsProvider == "" { - // using mock server, don't use wildcard certs - err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) - if err != nil { - log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") - } - } else { - provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) - if err != nil { - log.Error().Err(err).Msg("Can't create DNS Challenge provider") - } - err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider) - if err != nil { - log.Error().Err(err).Msg("Can't create DNS-01 provider") - } - } - } - if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + _, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", true, mainDomainSuffix, certDB) if err != nil { log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only") } @@ -487,7 +309,7 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co return nil } -func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { +func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *AcmeClient, mainDomainSuffix string, certDB database.CertDB) { for { // delete expired certs that will be invalid until next clean up threshold := time.Now().Add(interval) @@ -525,7 +347,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } else if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) { // renew main certificate 30 days before it expires go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) + _, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", true, mainDomainSuffix, certDB) if err != nil { log.Error().Err(err).Msg("Couldn't renew certificate for main domain") } diff --git a/server/certificates/mock_test.go b/server/certificates/mock_test.go index 00e1b21..5d0dde0 100644 --- a/server/certificates/mock_test.go +++ b/server/certificates/mock_test.go @@ -3,8 +3,9 @@ package certificates import ( "testing" - "codeberg.org/codeberg/pages/server/database" "github.com/stretchr/testify/assert" + + "codeberg.org/codeberg/pages/server/database" ) func TestMockCert(t *testing.T) { From 08d4e70cfd4df37748b29a859b1e5d408d93dd59 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Feb 2023 03:39:38 +0100 Subject: [PATCH 36/62] Update Readme to point out new Architecture --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6d5a294..5707d65 100644 --- a/README.md +++ b/README.md @@ -87,20 +87,20 @@ You can also contact the maintainers of this project: ### First steps -The code of this repository is split in several modules. -While heavy refactoring work is currently undergo, you can easily understand the basic structure: +The code of this repository is split in several modules. +The [Architecture is explained](https://codeberg.org/Codeberg/pages-server/wiki/Architecture) in the wiki. + The `cmd` folder holds the data necessary for interacting with the service via the cli. -If you are considering to deploy the service yourself, make sure to check it out. The heart of the software lives in the `server` folder and is split in several modules. -After scanning the code, you should quickly be able to understand their function and start hacking on them. Again: Feel free to get in touch with us for any questions that might arise. Thank you very much. - ### Test Server -run `just dev` +Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://just.systems/man/en/) installed. + +run `just dev` now this pages should work: - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg - https://momar.localhost.mock.directory:4430/ci-testing/ From 46316f9e2f472b070d187ee1e06742dea0739413 Mon Sep 17 00:00:00 2001 From: crystal Date: Sat, 11 Feb 2023 03:12:42 +0000 Subject: [PATCH 37/62] Fix raw domain for branches with custom domains and index.html (#159) fix #156 fix #157 Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/159 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crystal Co-committed-by: crystal --- integration/get_test.go | 28 ++++++++++++++++++++++++++++ server/handler/try.go | 2 +- server/upstream/upstream.go | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/integration/get_test.go b/integration/get_test.go index 55e6d12..b70eeed 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -109,6 +109,34 @@ func TestCustomDomainRedirects(t *testing.T) { assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location")) } +func TestRawCustomDomain(t *testing.T) { + log.Println("=== TestRawCustomDomain ===") + // test raw domain response for custom domain branch + resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) + assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "76", resp.Header.Get("Content-Length")) + assert.EqualValues(t, 76, getSize(resp.Body)) +} + +func TestRawIndex(t *testing.T) { + log.Println("=== TestRawCustomDomain ===") + // test raw domain response for index.html + resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) + assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "597", resp.Header.Get("Content-Length")) + assert.EqualValues(t, 597, getSize(resp.Body)) +} + func TestGetNotFound(t *testing.T) { log.Println("=== TestGetNotFound ===") // test custom not found pages diff --git a/server/handler/try.go b/server/handler/try.go index 5a09b91..5c65138 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -20,7 +20,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, canonicalDomainCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain - if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) { canonicalPath := ctx.Req.RequestURI diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 7c3c848..3845969 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -168,7 +168,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) return true } - if strings.HasSuffix(ctx.Path(), "/index.html") { + if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw { ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) return true } From 9a3d1c36dce7c7d9c460dfdb684f6e4bae5848f9 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 13 Feb 2023 20:14:45 +0000 Subject: [PATCH 38/62] Document more flags & make http port customizable (#183) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/183 --- .gitignore | 1 + Justfile | 3 ++ cmd/flags.go | 34 +++++++++++------- cmd/main.go | 40 +++++++++++---------- cmd/setup.go | 5 +++ server/certificates/acme_config.go | 2 ++ server/certificates/cached_challengers.go | 19 ++++++++++ server/certificates/certificates.go | 43 +++++++++++++---------- server/setup.go | 27 -------------- 9 files changed, 97 insertions(+), 77 deletions(-) delete mode 100644 server/setup.go diff --git a/.gitignore b/.gitignore index 8745935..3035107 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ vendor/ pages certs.sqlite +.bash_history diff --git a/Justfile b/Justfile index 0db7845..9ee7eb3 100644 --- a/Justfile +++ b/Justfile @@ -50,3 +50,6 @@ integration: integration-run TEST: go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... + +docker: + docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just diff --git a/cmd/flags.go b/cmd/flags.go index 8052421..5bc638b 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -8,11 +8,13 @@ var ( CertStorageFlags = []cli.Flag{ &cli.StringFlag{ Name: "db-type", + Usage: "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io", Value: "sqlite3", EnvVars: []string{"DB_TYPE"}, }, &cli.StringFlag{ Name: "db-conn", + Usage: "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access", Value: "certs.sqlite", EnvVars: []string{"DB_CONN"}, }, @@ -87,15 +89,21 @@ var ( EnvVars: []string{"HOST"}, Value: "[::]", }, - &cli.StringFlag{ + &cli.UintFlag{ Name: "port", - Usage: "specifies port of listening address", - EnvVars: []string{"PORT"}, - Value: "443", + Usage: "specifies the https port to listen to ssl requests", + EnvVars: []string{"PORT", "HTTPS_PORT"}, + Value: 443, + }, + &cli.UintFlag{ + Name: "http-port", + Usage: "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true", + EnvVars: []string{"HTTP_PORT"}, + Value: 80, }, &cli.BoolFlag{ - Name: "enable-http-server", - // TODO: desc + Name: "enable-http-server", + Usage: "start a http server to redirect to https and respond to http acme challenges", EnvVars: []string{"ENABLE_HTTP_SERVER"}, }, &cli.StringFlag{ @@ -125,23 +133,23 @@ var ( Value: true, }, &cli.BoolFlag{ - Name: "acme-accept-terms", - // TODO: Usage + Name: "acme-accept-terms", + Usage: "To accept the ACME ToS", EnvVars: []string{"ACME_ACCEPT_TERMS"}, }, &cli.StringFlag{ - Name: "acme-eab-kid", - // TODO: Usage + Name: "acme-eab-kid", + Usage: "Register the current account to the ACME server with external binding.", EnvVars: []string{"ACME_EAB_KID"}, }, &cli.StringFlag{ - Name: "acme-eab-hmac", - // TODO: Usage + Name: "acme-eab-hmac", + Usage: "Register the current account to the ACME server with external binding.", EnvVars: []string{"ACME_EAB_HMAC"}, }, &cli.StringFlag{ Name: "dns-provider", - Usage: "Use DNS-Challenge for main domain\n\nRead more at: https://go-acme.github.io/lego/dns/", + Usage: "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/", EnvVars: []string{"DNS_PROVIDER"}, }, &cli.StringFlag{ diff --git a/cmd/main.go b/cmd/main.go index a1c3b97..8a65d43 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -14,7 +14,6 @@ import ( "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "codeberg.org/codeberg/pages/server" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/gitea" @@ -48,7 +47,9 @@ func Serve(ctx *cli.Context) error { rawDomain := ctx.String("raw-domain") mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") - listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) + listeningHost := ctx.String("host") + listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("port")) + listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port")) enableHTTPServer := ctx.Bool("enable-http-server") allowedCorsDomains := AllowedCorsDomains @@ -91,22 +92,14 @@ func Serve(ctx *cli.Context) error { return err } - // Create handler based on settings - httpsHandler := handler.Handler(mainDomainSuffix, rawDomain, - giteaClient, - rawInfoPage, - BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache) - - httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache) - - // Setup listener and TLS - log.Info().Msgf("Listening on https://%s", listeningAddress) - listener, err := net.Listen("tcp", listeningAddress) + // Create listener for SSL connections + log.Info().Msgf("Listening on https://%s", listeningSSLAddress) + listener, err := net.Listen("tcp", listeningSSLAddress) if err != nil { return fmt.Errorf("couldn't create listener: %v", err) } + // Setup listener for SSL connections listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaClient, acmeClient, @@ -119,18 +112,29 @@ func Serve(ctx *cli.Context) error { go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB) if enableHTTPServer { + // Create handler for http->https redirect and http acme challenges + httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache) + + // Create listener for http and start listening go func() { - log.Info().Msg("Start HTTP server listening on :80") - err := http.ListenAndServe("[::]:80", httpHandler) + log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) + err := http.ListenAndServe(listeningHTTPAddress, httpHandler) if err != nil { log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") } }() } - // Start the web fastServer + // Create ssl handler based on settings + sslHandler := handler.Handler(mainDomainSuffix, rawDomain, + giteaClient, + rawInfoPage, + BlacklistedPaths, allowedCorsDomains, + dnsLookupCache, canonicalDomainCache) + + // Start the ssl listener log.Info().Msgf("Start listening on %s", listener.Addr()) - if err := http.Serve(listener, httpsHandler); err != nil { + if err := http.Serve(listener, sslHandler); err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") } diff --git a/cmd/setup.go b/cmd/setup.go index bb9f8cb..cde4bc9 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -43,6 +43,11 @@ func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache ca if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) } + if acmeEabHmac != "" && acmeEabKID == "" { + return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) + } else if acmeEabHmac == "" && acmeEabKID != "" { + return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) + } return certificates.NewAcmeClient( acmeAccountConf, diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go index 69568e6..12ad7c6 100644 --- a/server/certificates/acme_config.go +++ b/server/certificates/acme_config.go @@ -14,6 +14,8 @@ import ( "github.com/rs/zerolog/log" ) +const challengePath = "/.well-known/acme-challenge/" + func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index 6ce6e67..02474b3 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -1,11 +1,15 @@ package certificates import ( + "net/http" + "strings" "time" "github.com/go-acme/lego/v4/challenge" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/utils" ) type AcmeTLSChallengeProvider struct { @@ -39,3 +43,18 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { a.challengeCache.Remove(domain + "/" + token) return nil } + +func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + ctx := context.New(w, req) + if strings.HasPrefix(ctx.Path(), challengePath) { + challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) + if !ok || challenge == nil { + ctx.String("no challenge for this token", http.StatusNotFound) + } + ctx.String(challenge.(string)) + } else { + ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently) + } + } +} diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 3ea440f..707672c 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -36,22 +36,23 @@ func TLSConfig(mainDomainSuffix string, return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - sni := strings.ToLower(strings.TrimSpace(info.ServerName)) - if len(sni) < 1 { - return nil, errors.New("missing sni") + domain := strings.ToLower(strings.TrimSpace(info.ServerName)) + if len(domain) < 1 { + return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)") } + // https request init is actually a acme challenge if info.SupportedProtos != nil { for _, proto := range info.SupportedProtos { if proto != tlsalpn01.ACMETLS1Protocol { continue } - challenge, ok := challengeCache.Get(sni) + challenge, ok := challengeCache.Get(domain) if !ok { return nil, errors.New("no challenge for this domain") } - cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) + cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string)) if err != nil { return nil, err } @@ -61,22 +62,22 @@ func TLSConfig(mainDomainSuffix string, targetOwner := "" mayObtainCert := true - if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) { + if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) { // deliver default certificate for the main domain (*.codeberg.page) - sni = mainDomainSuffix + domain = mainDomainSuffix } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs - sni = mainDomainSuffix + domain = mainDomainSuffix } else { targetOpt := &upstream.Options{ TargetOwner: targetOwner, TargetRepo: targetRepo, TargetBranch: targetBranch, } - _, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache) + _, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache) if !valid { // We shouldn't obtain a certificate when we cannot check if the // repository has specified this domain in the `.domains` file. @@ -85,30 +86,34 @@ func TLSConfig(mainDomainSuffix string, } } - if tlsCertificate, ok := keyCache.Get(sni); ok { + if tlsCertificate, ok := keyCache.Get(domain); ok { // we can use an existing certificate object return tlsCertificate.(*tls.Certificate), nil } var tlsCertificate *tls.Certificate var err error - if tlsCertificate, err = acmeClient.retrieveCertFromDB(sni, mainDomainSuffix, false, certDB); err != nil { - // request a new certificate - if strings.EqualFold(sni, mainDomainSuffix) { + if tlsCertificate, err = acmeClient.retrieveCertFromDB(domain, mainDomainSuffix, false, certDB); err != nil { + if !errors.Is(err, database.ErrNotFound) { + return nil, err + } + // we could not find a cert in db, request a new certificate + + // first check if we are allowed to obtain a cert for this domain + if strings.EqualFold(domain, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } - if !mayObtainCert { - return nil, fmt.Errorf("won't request certificate for %q", sni) + return nil, fmt.Errorf("won't request certificate for %q", domain) } - tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{sni}, nil, targetOwner, false, mainDomainSuffix, certDB) + tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB) if err != nil { return nil, err } } - if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil { + if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil { return nil, err } return tlsCertificate, nil @@ -164,7 +169,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv if !strings.EqualFold(sni, mainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { - return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err) + return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err) } // renew certificates 7 days before they expire diff --git a/server/setup.go b/server/setup.go deleted file mode 100644 index 282e692..0000000 --- a/server/setup.go +++ /dev/null @@ -1,27 +0,0 @@ -package server - -import ( - "net/http" - "strings" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/context" - "codeberg.org/codeberg/pages/server/utils" -) - -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc { - challengePath := "/.well-known/acme-challenge/" - - return func(w http.ResponseWriter, req *http.Request) { - ctx := context.New(w, req) - if strings.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) - if !ok || challenge == nil { - ctx.String("no challenge for this token", http.StatusNotFound) - } - ctx.String(challenge.(string)) - } else { - ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently) - } - } -} From 42b3f8d1b7933e67a2a24867d1189d522f2fb884 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 13 Feb 2023 23:13:30 +0000 Subject: [PATCH 39/62] use mockery for mock code generation (#185) close #181 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/185 --- server/certificates/mock_test.go | 6 +- server/database/interface.go | 3 + server/database/mock.go | 139 +++++++++++++++++++++++-------- 3 files changed, 113 insertions(+), 35 deletions(-) diff --git a/server/certificates/mock_test.go b/server/certificates/mock_test.go index 5d0dde0..644e8a9 100644 --- a/server/certificates/mock_test.go +++ b/server/certificates/mock_test.go @@ -4,13 +4,15 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "codeberg.org/codeberg/pages/server/database" ) func TestMockCert(t *testing.T) { - db, err := database.NewTmpDB() - assert.NoError(t, err) + db := database.NewMockCertDB(t) + db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil) + cert, err := mockCert("example.com", "some error msg", "codeberg.page", db) assert.NoError(t, err) if assert.NotEmpty(t, cert) { diff --git a/server/database/interface.go b/server/database/interface.go index 92068c5..7fdbae7 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -8,6 +8,9 @@ import ( "github.com/rs/zerolog/log" ) +//go:generate go install github.com/vektra/mockery/v2@latest +//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore + type CertDB interface { Close() error Put(name string, cert *certificate.Resource) error diff --git a/server/database/mock.go b/server/database/mock.go index 6148287..e7e2c38 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -1,49 +1,122 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + package database import ( - "fmt" - "time" - - "github.com/OrlovEvgeny/go-mcache" - "github.com/go-acme/lego/v4/certificate" + certificate "github.com/go-acme/lego/v4/certificate" + mock "github.com/stretchr/testify/mock" ) -var _ CertDB = tmpDB{} - -type tmpDB struct { - intern *mcache.CacheDriver - ttl time.Duration +// MockCertDB is an autogenerated mock type for the CertDB type +type MockCertDB struct { + mock.Mock } -func (p tmpDB) Close() error { - _ = p.intern.Close() - return nil -} +// Close provides a mock function with given fields: +func (_m *MockCertDB) Close() error { + ret := _m.Called() -func (p tmpDB) Put(name string, cert *certificate.Resource) error { - return p.intern.Set(name, cert, p.ttl) -} - -func (p tmpDB) Get(name string) (*certificate.Resource, error) { - cert, has := p.intern.Get(name) - if !has { - return nil, fmt.Errorf("cert for %q not found", name) + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) } - return cert.(*certificate.Resource), nil + + return r0 } -func (p tmpDB) Delete(key string) error { - p.intern.Remove(key) - return nil +// Delete provides a mock function with given fields: key +func (_m *MockCertDB) Delete(key string) error { + ret := _m.Called(key) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(key) + } else { + r0 = ret.Error(0) + } + + return r0 } -func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) { - return nil, fmt.Errorf("items not implemented for tmpDB") +// Get provides a mock function with given fields: name +func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) { + ret := _m.Called(name) + + var r0 *certificate.Resource + var r1 error + if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok { + return rf(name) + } + if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*certificate.Resource) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -func NewTmpDB() (CertDB, error) { - return &tmpDB{ - intern: mcache.New(), - ttl: time.Minute, - }, nil +// Items provides a mock function with given fields: page, pageSize +func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) { + ret := _m.Called(page, pageSize) + + var r0 []*Cert + var r1 error + if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok { + return rf(page, pageSize) + } + if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok { + r0 = rf(page, pageSize) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*Cert) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(page, pageSize) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Put provides a mock function with given fields: name, cert +func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error { + ret := _m.Called(name, cert) + + var r0 error + if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok { + r0 = rf(name, cert) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewMockCertDB interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB { + mock := &MockCertDB{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } From 0adac9a5b15746a139e76e1de3996d4a20e791fa Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 14 Feb 2023 02:23:28 +0000 Subject: [PATCH 40/62] fix http -> https redirect and add integration tests for it (#184) and more logging Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/184 --- Justfile | 2 ++ cmd/main.go | 9 ++++--- integration/get_test.go | 12 +++++++++ integration/main_test.go | 2 ++ server/certificates/cached_challengers.go | 33 +++++++++++++++++++---- server/certificates/certificates.go | 1 + 6 files changed, 50 insertions(+), 9 deletions(-) diff --git a/Justfile b/Justfile index 9ee7eb3..0b8f814 100644 --- a/Justfile +++ b/Justfile @@ -9,6 +9,8 @@ dev: export PAGES_DOMAIN=localhost.mock.directory export RAW_DOMAIN=raw.localhost.mock.directory export PORT=4430 + export HTTP_PORT=8880 + export ENABLE_HTTP_SERVER=true export LOG_LEVEL=trace go run -tags '{{TAGS}}' . diff --git a/cmd/main.go b/cmd/main.go index 8a65d43..aa00f54 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -48,7 +48,8 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningHost := ctx.String("host") - listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("port")) + listeningSSLPort := ctx.Uint("port") + listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port")) enableHTTPServer := ctx.Bool("enable-http-server") @@ -93,7 +94,7 @@ func Serve(ctx *cli.Context) error { } // Create listener for SSL connections - log.Info().Msgf("Listening on https://%s", listeningSSLAddress) + log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) listener, err := net.Listen("tcp", listeningSSLAddress) if err != nil { return fmt.Errorf("couldn't create listener: %v", err) @@ -113,7 +114,7 @@ func Serve(ctx *cli.Context) error { if enableHTTPServer { // Create handler for http->https redirect and http acme challenges - httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache) + httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort) // Create listener for http and start listening go func() { @@ -133,7 +134,7 @@ func Serve(ctx *cli.Context) error { dnsLookupCache, canonicalDomainCache) // Start the ssl listener - log.Info().Msgf("Start listening on %s", listener.Addr()) + log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) if err := http.Serve(listener, sslHandler); err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") } diff --git a/integration/get_test.go b/integration/get_test.go index b70eeed..3a7190a 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -193,6 +193,18 @@ func TestGetOptions(t *testing.T) { assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow")) } +func TestHttpRedirect(t *testing.T) { + log.Println("=== TestHttpRedirect ===") + resp, err := getTestHTTPSClient().Get("http://mock-pages.codeberg-test.org:8880/README.md") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode) + assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location")) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ diff --git a/integration/main_test.go b/integration/main_test.go index 3e0e187..a6579fd 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -40,6 +40,8 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") setEnvIfNotSet("PORT", "4430") + setEnvIfNotSet("HTTP_PORT", "8880") + setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") setEnvIfNotSet("DB_TYPE", "sqlite3") app := cli.NewApp() diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index 02474b3..bc9ea67 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -1,15 +1,17 @@ package certificates import ( + "fmt" "net/http" + "net/url" "strings" "time" "github.com/go-acme/lego/v4/challenge" + "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" - "codeberg.org/codeberg/pages/server/utils" ) type AcmeTLSChallengeProvider struct { @@ -44,17 +46,38 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc { +func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc { + // handle custom-ssl-ports to be added on https redirects + portPart := "" + if sslPort != 443 { + portPart = fmt.Sprintf(":%d", sslPort) + } + return func(w http.ResponseWriter, req *http.Request) { ctx := context.New(w, req) + domain := ctx.TrimHostPort() + + // it's an acme request if strings.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) + challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) if !ok || challenge == nil { + log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain) ctx.String("no challenge for this token", http.StatusNotFound) } + log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain) ctx.String(challenge.(string)) - } else { - ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently) + return } + + // it's a normal http request that needs to be redirected + u, err := url.Parse(fmt.Sprintf("https://%s%s%s", domain, portPart, ctx.Path())) + if err != nil { + log.Error().Err(err).Msg("could not craft http to https redirect") + ctx.String("", http.StatusInternalServerError) + } + + newURL := u.String() + log.Debug().Msgf("redirect http to https: %s", newURL) + ctx.Redirect(newURL, http.StatusMovedPermanently) } } diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 707672c..ce1420d 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -47,6 +47,7 @@ func TLSConfig(mainDomainSuffix string, if proto != tlsalpn01.ACMETLS1Protocol { continue } + log.Info().Msgf("Detect ACME-TLS1 challenge for '%s'", domain) challenge, ok := challengeCache.Get(domain) if !ok { From 42d5802b9ba77556dfa529e20caf0442ac6b1fa4 Mon Sep 17 00:00:00 2001 From: deblan Date: Tue, 14 Feb 2023 03:03:00 +0000 Subject: [PATCH 41/62] Allow to define default branches (#125) This try to address #115 Co-authored-by: Simon Vieille Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/125 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: deblan Co-committed-by: deblan --- cmd/flags.go | 7 ++++ cmd/main.go | 7 ++++ go.mod | 3 +- go.sum | 4 +++ integration/main_test.go | 1 + server/certificates/certificates.go | 3 +- server/dns/dns.go | 10 +++--- server/handler/handler.go | 4 ++- server/handler/handler_custom_domain.go | 5 +-- server/handler/handler_sub_domain.go | 46 ++++++++++++++++++++----- server/handler/handler_test.go | 1 + 11 files changed, 73 insertions(+), 18 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 5bc638b..a71dd35 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -112,6 +112,13 @@ var ( Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", EnvVars: []string{"LOG_LEVEL"}, }, + // Default branches to fetch assets from + &cli.StringSliceFlag{ + Name: "pages-branch", + Usage: "define a branch to fetch assets from", + EnvVars: []string{"PAGES_BRANCHES"}, + Value: cli.NewStringSlice("pages"), + }, // ############################ // ### ACME Client Settings ### diff --git a/cmd/main.go b/cmd/main.go index aa00f54..45e151d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := ctx.String("gitea-root") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") + defaultBranches := ctx.StringSlice("pages-branch") mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningHost := ctx.String("host") @@ -63,6 +64,10 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix = "." + mainDomainSuffix } + if len(defaultBranches) == 0 { + return fmt.Errorf("no default branches set (PAGES_BRANCHES)") + } + // Init ssl cert database certDB, closeFn, err := openCertDB(ctx) if err != nil { @@ -104,6 +109,7 @@ func Serve(ctx *cli.Context) error { listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaClient, acmeClient, + defaultBranches[0], keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB)) @@ -131,6 +137,7 @@ func Serve(ctx *cli.Context) error { giteaClient, rawInfoPage, BlacklistedPaths, allowedCorsDomains, + defaultBranches, dnsLookupCache, canonicalDomainCache) // Start the ssl listener diff --git a/go.mod b/go.mod index 944e2ad..bfde7f7 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 + golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb xorm.io/xorm v1.3.2 ) @@ -117,7 +118,7 @@ require ( golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect diff --git a/go.sum b/go.sum index b5a7568..b10305c 100644 --- a/go.sum +++ b/go.sum @@ -768,6 +768,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -899,6 +901,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/integration/main_test.go b/integration/main_test.go index a6579fd..a397110 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -39,6 +39,7 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("ACME_API", "https://acme.mock.directory") setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") + setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master") setEnvIfNotSet("PORT", "4430") setEnvIfNotSet("HTTP_PORT", "8880") setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index ce1420d..3ae891a 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -30,6 +30,7 @@ var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, + firstDefaultBranch string, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, certDB database.CertDB, ) *tls.Config { @@ -68,7 +69,7 @@ func TLSConfig(mainDomainSuffix string, domain = mainDomainSuffix } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs domain = mainDomainSuffix diff --git a/server/dns/dns.go b/server/dns/dns.go index 2719d4d..c11b278 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -11,9 +11,11 @@ import ( // lookupCacheTimeout specifies the timeout for the DNS lookup cache. var lookupCacheTimeout = 15 * time.Minute +var defaultPagesRepo = "pages" + // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { +func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error @@ -50,10 +52,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG targetBranch = cnameParts[len(cnameParts)-3] } if targetRepo == "" { - targetRepo = "pages" + targetRepo = defaultPagesRepo } - if targetBranch == "" && targetRepo != "pages" { - targetBranch = "pages" + if targetBranch == "" && targetRepo != defaultPagesRepo { + targetBranch = firstDefaultBranch } // if targetBranch is still empty, the caller must find the default branch return diff --git a/server/handler/handler.go b/server/handler/handler.go index 78301e9..a944c7e 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -17,7 +17,6 @@ const ( headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" headerAccessControlAllowMethods = "Access-Control-Allow-Methods" defaultPagesRepo = "pages" - defaultPagesBranch = "pages" ) // Handler handles a single HTTP request to the web server. @@ -25,6 +24,7 @@ func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, rawInfoPage string, blacklistedPaths, allowedCorsDomains []string, + defaultPagesBranches []string, dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { @@ -98,6 +98,7 @@ func Handler(mainDomainSuffix, rawDomain string, log.Debug().Msg("subdomain request detecded") handleSubDomain(log, ctx, giteaClient, mainDomainSuffix, + defaultPagesBranches, trimmedHost, pathElements, canonicalDomainCache) @@ -107,6 +108,7 @@ func Handler(mainDomainSuffix, rawDomain string, mainDomainSuffix, trimmedHost, pathElements, + defaultPagesBranches[0], dnsLookupCache, canonicalDomainCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index a541b74..1b85f62 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -18,10 +18,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g mainDomainSuffix string, trimmedHost string, pathElements []string, + firstDefaultBranch string, dnsLookupCache, canonicalDomainCache cache.SetGetKey, ) { // Serve pages from custom domains - targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache) + targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner == "" { html.ReturnErrorPage(ctx, "could not obtain repo owner from custom domain", @@ -52,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! - targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache) + targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache) if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect) return diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 2a75e9f..68f4822 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/rs/zerolog" + "golang.org/x/exp/slices" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" @@ -17,6 +18,7 @@ import ( func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix string, + defaultPagesBranches []string, trimmedHost string, pathElements []string, canonicalDomainCache cache.SetGetKey, @@ -63,12 +65,21 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Check if the first directory is a branch for the defaultPagesRepo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { + targetBranch := pathElements[0][1:] + + // if the default pages branch can be determined exactly, it does not need to be set + if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) { + // example.codeberg.org/@pages/... redirects to example.codeberg.org/... + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) + return + } + log.Debug().Msg("main domain preparations, now trying with specified branch") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, TargetRepo: defaultPagesRepo, - TargetBranch: pathElements[0][1:], + TargetBranch: targetBranch, TargetPath: path.Join(pathElements[1:]...), }, true); works { log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") @@ -81,19 +92,36 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite return } - // Check if the first directory is a repo with a defaultPagesRepo branch - // example.codeberg.page/myrepo/index.html - // example.codeberg.page/pages/... is not allowed here. - log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != defaultPagesRepo { + for _, defaultPagesBranch := range defaultPagesBranches { + // Check if the first directory is a repo with a default pages branch + // example.codeberg.page/myrepo/index.html + // example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here. + log.Debug().Msg("main domain preparations, now trying with specified repo") + if pathElements[0] != defaultPagesBranch { + if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ + TryIndexPages: true, + TargetOwner: targetOwner, + TargetRepo: pathElements[0], + TargetBranch: defaultPagesBranch, + TargetPath: path.Join(pathElements[1:]...), + }, false); works { + log.Debug().Msg("tryBranch, now trying upstream 5") + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + return + } + } + + // Try to use the defaultPagesRepo on an default pages branch + // example.codeberg.page/index.html + log.Debug().Msg("main domain preparations, now trying with default repo") if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{ TryIndexPages: true, TargetOwner: targetOwner, - TargetRepo: pathElements[0], + TargetRepo: defaultPagesRepo, TargetBranch: defaultPagesBranch, - TargetPath: path.Join(pathElements[1:]...), + TargetPath: path.Join(pathElements...), }, false); works { - log.Debug().Msg("tryBranch, now trying upstream 5") + log.Debug().Msg("tryBranch, now trying upstream 6") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) return } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index 626564a..ed063b2 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -18,6 +18,7 @@ func TestHandlerPerformance(t *testing.T) { "https://docs.codeberg.org/pages/raw-content/", []string{"/.well-known/acme-challenge/"}, []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, + []string{"pages"}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), ) From c9050e5722120f03fc5c8e632aa785d790f2b513 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 11 Mar 2023 05:07:17 +0000 Subject: [PATCH 42/62] Handle Relative Symlinks (#205) enhance #114 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/205 --- integration/get_test.go | 13 ++++++++++++- integration/main_test.go | 2 +- server/certificates/acme_client.go | 2 +- server/database/xorm.go | 2 +- server/gitea/client.go | 6 +++++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/integration/get_test.go b/integration/get_test.go index 3a7190a..c0a3a47 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -64,7 +64,7 @@ func TestGetContent(t *testing.T) { assert.True(t, getSize(resp.Body) > 100) assert.Len(t, resp.Header.Get("ETag"), 44) - // TODO: test get of non cachable content (content size > fileCacheSizeLimit) + // TODO: test get of non cacheable content (content size > fileCacheSizeLimit) } func TestCustomDomain(t *testing.T) { @@ -154,6 +154,7 @@ func TestGetNotFound(t *testing.T) { func TestFollowSymlink(t *testing.T) { log.Printf("=== TestFollowSymlink ===\n") + // file symlink resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link") assert.NoError(t, err) if !assert.NotNil(t, resp) { @@ -165,6 +166,16 @@ func TestFollowSymlink(t *testing.T) { body := getBytes(resp.Body) assert.EqualValues(t, 4, len(body)) assert.EqualValues(t, "abc\n", string(body)) + + // relative file links (../index.html file in this case) + resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/dir_aim/some/") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) + assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "an index\n", string(getBytes(resp.Body))) } func TestLFSSupport(t *testing.T) { diff --git a/integration/main_test.go b/integration/main_test.go index a397110..6566f78 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -23,7 +23,7 @@ func TestMain(m *testing.M) { } defer func() { serverCancel() - log.Println("=== TestMain: Server STOPED ===") + log.Println("=== TestMain: Server STOPPED ===") }() time.Sleep(10 * time.Second) diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index 7737396..ba83e50 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -89,7 +89,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeClientRequestLimit: equalizer.NewTokenBucket(5, 1*time.Second), // rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/ acmeClientFailLimit: equalizer.NewTokenBucket(5, 1*time.Hour), - // checkUserLimit() use this to rate als per user + // checkUserLimit() use this to rate also per user acmeClientCertificateLimitPerUser: map[string]*equalizer.TokenBucket{}, }, nil } diff --git a/server/database/xorm.go b/server/database/xorm.go index 4b43cbb..fb1dc17 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -37,7 +37,7 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) { } if err := e.Sync2(new(Cert)); err != nil { - return nil, fmt.Errorf("cound not sync db model :%w", err) + return nil, fmt.Errorf("could not sync db model :%w", err) } return &xDB{ diff --git a/server/gitea/client.go b/server/gitea/client.go index 51647ba..7a2bf63 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -112,7 +112,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str if cache, ok := client.responseCache.Get(cacheKey); ok { cache := cache.(FileResponse) cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) - // TODO: check against some timestamp missmatch?!? + // TODO: check against some timestamp mismatch?!? if cache.Exists { if cache.IsSymlink { linkDest := string(cache.Body) @@ -145,6 +145,10 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str } linkDest := strings.TrimSpace(string(linkDestBytes)) + // handle relative links + // we first remove the link from the path, and make a relative join (resolve parent paths like "/../" too) + linkDest = path.Join(path.Dir(resource), linkDest) + // we store symlink not content to reduce duplicates in cache if err := client.responseCache.Set(cacheKey, FileResponse{ Exists: true, From 26d59b71f049d459b663ec885156bfb8b89aa0c9 Mon Sep 17 00:00:00 2001 From: Crystal Date: Mon, 20 Mar 2023 22:52:42 +0000 Subject: [PATCH 43/62] Fix typo in integration test log (#210) I forgot to update the name of this function in the CI log so it looks like it's running the same test twice even though it's not. Co-authored-by: crystal Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/210 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Crystal Co-committed-by: Crystal --- integration/get_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/get_test.go b/integration/get_test.go index c0a3a47..9d97390 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -124,7 +124,7 @@ func TestRawCustomDomain(t *testing.T) { } func TestRawIndex(t *testing.T) { - log.Println("=== TestRawCustomDomain ===") + log.Println("=== TestRawIndex ===") // test raw domain response for index.html resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork assert.NoError(t, err) From c40dddf471277260ee3ca95c5701cf723c1e24cb Mon Sep 17 00:00:00 2001 From: Crystal Date: Mon, 20 Mar 2023 22:57:26 +0000 Subject: [PATCH 44/62] Fix certificate renewal (#209) A database bug in xorm.go prevents the pages-server from saving a renewed certificate for a domain that already has one in the database. Co-authored-by: crystal Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/209 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Crystal Co-committed-by: Crystal --- server/database/xorm.go | 2 +- server/database/xorm_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/database/xorm.go b/server/database/xorm.go index fb1dc17..217b6d1 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -64,7 +64,7 @@ func (x xDB) Put(domain string, cert *certificate.Resource) error { } defer sess.Close() - if exist, _ := sess.ID(c.Domain).Exist(); exist { + if exist, _ := sess.ID(c.Domain).Exist(new(Cert)); exist { if _, err := sess.ID(c.Domain).Update(c); err != nil { return err } diff --git a/server/database/xorm_test.go b/server/database/xorm_test.go index 9c032ee..50d8a7f 100644 --- a/server/database/xorm_test.go +++ b/server/database/xorm_test.go @@ -37,7 +37,7 @@ func TestSanitizeWildcardCerts(t *testing.T) { })) // update existing cert - assert.Error(t, certDB.Put(".wildcard.de", &certificate.Resource{ + assert.NoError(t, certDB.Put(".wildcard.de", &certificate.Resource{ Domain: "*.wildcard.de", Certificate: localhost_mock_directory_certificate, })) From 98d7a771be9783a333a3f9abd8d6e08d1ad4a46f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 21 Mar 2023 01:53:07 +0100 Subject: [PATCH 45/62] Readme.md: add link to chat & main repo --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 5707d65..7e1b2c5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Codeberg Pages +[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue)](https://opensource.org/license/eupl-1-2/) +[![status-badge](https://ci.codeberg.org/api/badges/Codeberg/pages-server/status.svg)](https://ci.codeberg.org/Codeberg/pages-server) + + + + Gitea lacks the ability to host static pages from Git. The Codeberg Pages Server addresses this lack by implementing a standalone service that connects to Gitea via API. @@ -8,6 +14,9 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview) and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/). + + Get It On Codeberg + ## Quickstart This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. @@ -29,6 +38,10 @@ record that points to your repo (just like the CNAME record): Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. +## Chat for admins & devs + +[matrix: #gitea-pages-server:obermui.de](https://matrix.to/#/#gitea-pages-server:obermui.de) + ## Deployment **Warning: Some Caveats Apply** From 970c13cf5c006dfa1011ba9e6721baafa3f1ad55 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 21 Mar 2023 02:32:25 +0100 Subject: [PATCH 46/62] Readme.md: use matrix.org for room alias --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7e1b2c5..b694e29 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ [![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue)](https://opensource.org/license/eupl-1-2/) [![status-badge](https://ci.codeberg.org/api/badges/Codeberg/pages-server/status.svg)](https://ci.codeberg.org/Codeberg/pages-server) - - + + Gitea lacks the ability to host static pages from Git. @@ -40,7 +40,7 @@ Certificates are generated, updated and cleaned up automatically via Let's Encry ## Chat for admins & devs -[matrix: #gitea-pages-server:obermui.de](https://matrix.to/#/#gitea-pages-server:obermui.de) +[matrix: #gitea-pages-server:matrix.org](https://matrix.to/#/#gitea-pages-server:matrix.org) ## Deployment From 974229681f4cc7f1ed31df9b05eabef2df013809 Mon Sep 17 00:00:00 2001 From: video-prize-ranch Date: Thu, 30 Mar 2023 21:36:31 +0000 Subject: [PATCH 47/62] Initial redirects implementation (#148) Adds basic support for `_redirects` files. It supports a subset of what IPFS supports: https://docs.ipfs.tech/how-to/websites-on-ipfs/redirects-and-custom-404s/ Example: ``` /redirect https://example.com/ 301 /another-redirect /page 301 /302 https://example.com/ 302 /app/* /index.html 200 /articles/* /posts/:splat 301 ``` 301 redirect: https://video-prize-ranch.localhost.mock.directory:4430/redirect SPA rewrite: https://video-prize-ranch.localhost.mock.directory:4430/app/path/path Catch-all with splat: https://video-prize-ranch.localhost.mock.directory:4430/articles/path/path Closes #46 Co-authored-by: video-prize-ranch Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/148 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: video-prize-ranch Co-committed-by: video-prize-ranch --- FEATURES.md | 45 +++++++++ cmd/main.go | 4 +- integration/get_test.go | 40 ++++++++ server/handler/handler.go | 8 +- server/handler/handler_custom_domain.go | 4 +- server/handler/handler_raw_domain.go | 6 +- server/handler/handler_sub_domain.go | 12 +-- server/handler/handler_test.go | 1 + server/handler/try.go | 3 +- server/upstream/redirects.go | 117 ++++++++++++++++++++++++ server/upstream/upstream.go | 16 +++- 11 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 FEATURES.md create mode 100644 server/upstream/redirects.go diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000..7560a1d --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,45 @@ +# Features + +## Custom domains + +... + +## Redirects + +Redirects can be created with a `_redirects` file with the following format: + +``` +# Comment +from to [status] +``` + +* Lines starting with `#` are ignored +* `from` - the path to redirect from (Note: repository and branch names are removed from request URLs) +* `to` - the path or URL to redirect to +* `status` - status code to use when redirecting (default 301) + +### Status codes + +* `200` - returns content from specified path (no external URLs) without changing the URL (rewrite) +* `301` - Moved Permanently (Permanent redirect) +* `302` - Found (Temporary redirect) + +### Examples + +#### SPA (single-page application) rewrite + +Redirects all paths to `/index.html` for single-page apps. + +``` +/* /index.html 200 +``` + +#### Splats + +Redirects every path under `/articles` to `/posts` while keeping the path. + +``` +/articles/* /posts/:splat 302 +``` + +Example: `/articles/2022/10/12/post-1/` -> `/posts/2022/10/12/post-1/` diff --git a/cmd/main.go b/cmd/main.go index 45e151d..84915c9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -81,6 +81,8 @@ func Serve(ctx *cli.Context) error { canonicalDomainCache := cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains dnsLookupCache := cache.NewKeyValueCache() + // redirectsCache stores redirects in _redirects files + redirectsCache := cache.NewKeyValueCache() // clientResponseCache stores responses from the Gitea server clientResponseCache := cache.NewKeyValueCache() @@ -138,7 +140,7 @@ func Serve(ctx *cli.Context) error { rawInfoPage, BlacklistedPaths, allowedCorsDomains, defaultBranches, - dnsLookupCache, canonicalDomainCache) + dnsLookupCache, canonicalDomainCache, redirectsCache) // Start the ssl listener log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) diff --git a/integration/get_test.go b/integration/get_test.go index 9d97390..cfb7188 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -151,6 +151,46 @@ func TestGetNotFound(t *testing.T) { assert.EqualValues(t, 37, getSize(resp.Body)) } +func TestRedirect(t *testing.T) { + log.Println("=== TestRedirect ===") + // test redirects + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/redirect") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode) + assert.EqualValues(t, "https://example.com/", resp.Header.Get("Location")) +} + +func TestSPARedirect(t *testing.T) { + log.Println("=== TestSPARedirect ===") + // test SPA redirects + url := "https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/app/aqdjw" + resp, err := getTestHTTPSClient().Get(url) + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) + assert.EqualValues(t, url, resp.Request.URL.String()) + assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "258", resp.Header.Get("Content-Length")) + assert.EqualValues(t, 258, getSize(resp.Body)) +} + +func TestSplatRedirect(t *testing.T) { + log.Println("=== TestSplatRedirect ===") + // test splat redirects + resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/articles/qfopefe") + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode) + assert.EqualValues(t, "/posts/qfopefe", resp.Header.Get("Location")) +} + func TestFollowSymlink(t *testing.T) { log.Printf("=== TestFollowSymlink ===\n") diff --git a/server/handler/handler.go b/server/handler/handler.go index a944c7e..7edcf95 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -25,7 +25,7 @@ func Handler(mainDomainSuffix, rawDomain string, rawInfoPage string, blacklistedPaths, allowedCorsDomains []string, defaultPagesBranches []string, - dnsLookupCache, canonicalDomainCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() @@ -93,7 +93,7 @@ func Handler(mainDomainSuffix, rawDomain string, mainDomainSuffix, rawInfoPage, trimmedHost, pathElements, - canonicalDomainCache) + canonicalDomainCache, redirectsCache) } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { log.Debug().Msg("subdomain request detecded") handleSubDomain(log, ctx, giteaClient, @@ -101,7 +101,7 @@ func Handler(mainDomainSuffix, rawDomain string, defaultPagesBranches, trimmedHost, pathElements, - canonicalDomainCache) + canonicalDomainCache, redirectsCache) } else { log.Debug().Msg("custom domain request detecded") handleCustomDomain(log, ctx, giteaClient, @@ -109,7 +109,7 @@ func Handler(mainDomainSuffix, rawDomain string, trimmedHost, pathElements, defaultPagesBranches[0], - dnsLookupCache, canonicalDomainCache) + dnsLookupCache, canonicalDomainCache, redirectsCache) } } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 1b85f62..8742be4 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g trimmedHost string, pathElements []string, firstDefaultBranch string, - dnsLookupCache, canonicalDomainCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, ) { // Serve pages from custom domains targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) @@ -64,7 +64,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g } log.Debug().Msg("tryBranch, now trying upstream 7") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) return } diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index 5e974da..aa41c52 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie mainDomainSuffix, rawInfoPage string, trimmedHost string, pathElements []string, - canonicalDomainCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.SetGetKey, ) { // Serve raw content from RawDomain log.Debug().Msg("raw domain") @@ -41,7 +41,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie TargetPath: path.Join(pathElements[3:]...), }, true); works { log.Trace().Msg("tryUpstream: serve raw domain with specified branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) return } log.Debug().Msg("missing branch info") @@ -58,7 +58,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie TargetPath: path.Join(pathElements[2:]...), }, true); works { log.Trace().Msg("tryUpstream: serve raw domain with default branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 68f4822..8731bec 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite defaultPagesBranches []string, trimmedHost string, pathElements []string, - canonicalDomainCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.SetGetKey, ) { // Serve pages from subdomains of MainDomainSuffix log.Debug().Msg("main domain suffix") @@ -53,7 +53,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[2:]...), }, true); works { log.Trace().Msg("tryUpstream: serve with specified repo and branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), @@ -83,7 +83,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[1:]...), }, true); works { log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), @@ -106,7 +106,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[1:]...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 5") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) return } } @@ -122,7 +122,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) return } } @@ -137,7 +137,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) return } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index ed063b2..de705ec 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -21,6 +21,7 @@ func TestHandlerPerformance(t *testing.T) { []string{"pages"}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), + cache.NewKeyValueCache(), ) testCase := func(uri string, status int) { diff --git a/server/handler/try.go b/server/handler/try.go index 5c65138..6cfe08e 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -18,6 +18,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost string, options *upstream.Options, canonicalDomainCache cache.SetGetKey, + redirectsCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { @@ -39,7 +40,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, options.Host = trimmedHost // Try to request the file from the Gitea API - if !options.Upstream(ctx, giteaClient) { + if !options.Upstream(ctx, giteaClient, redirectsCache) { html.ReturnErrorPage(ctx, "", ctx.StatusCode) } } diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go new file mode 100644 index 0000000..ab6c971 --- /dev/null +++ b/server/upstream/redirects.go @@ -0,0 +1,117 @@ +package upstream + +import ( + "strconv" + "strings" + "time" + + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" + "github.com/rs/zerolog/log" +) + +type Redirect struct { + From string + To string + StatusCode int +} + +// redirectsCacheTimeout specifies the timeout for the redirects cache. +var redirectsCacheTimeout = 10 * time.Minute + +const redirectsConfig = "_redirects" + +// getRedirects returns redirects specified in the _redirects file. +func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect { + var redirects []Redirect + cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch + + // Check for cached redirects + if cachedValue, ok := redirectsCache.Get(cacheKey); ok { + redirects = cachedValue.([]Redirect) + } else { + // Get _redirects file and parse + body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, redirectsConfig) + if err == nil { + for _, line := range strings.Split(string(body), "\n") { + redirectArr := strings.Fields(line) + + // Ignore comments and invalid lines + if strings.HasPrefix(line, "#") || len(redirectArr) < 2 { + continue + } + + // Get redirect status code + statusCode := 301 + if len(redirectArr) == 3 { + statusCode, err = strconv.Atoi(redirectArr[2]) + if err != nil { + log.Info().Err(err).Msgf("could not read %s of %s/%s", redirectsConfig, o.TargetOwner, o.TargetRepo) + } + } + + redirects = append(redirects, Redirect{ + From: redirectArr[0], + To: redirectArr[1], + StatusCode: statusCode, + }) + } + } + _ = redirectsCache.Set(cacheKey, redirects, redirectsCacheTimeout) + } + return redirects +} + +func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) { + if len(redirects) > 0 { + for _, redirect := range redirects { + reqUrl := ctx.Req.RequestURI + // remove repo and branch from request url + reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo) + reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch) + + // check if from url matches request url + if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") { + // do rewrite if status code is 200 + if redirect.StatusCode == 200 { + o.TargetPath = redirect.To + o.Upstream(ctx, giteaClient, redirectsCache) + return true + } else { + ctx.Redirect(redirect.To, redirect.StatusCode) + return true + } + } + + // handle wildcard redirects + trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*") + if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) { + if strings.Contains(redirect.To, ":splat") { + splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl)) + // do rewrite if status code is 200 + if redirect.StatusCode == 200 { + o.TargetPath = splatUrl + o.Upstream(ctx, giteaClient, redirectsCache) + return true + } else { + ctx.Redirect(splatUrl, redirect.StatusCode) + return true + } + } else { + // do rewrite if status code is 200 + if redirect.StatusCode == 200 { + o.TargetPath = redirect.To + o.Upstream(ctx, giteaClient, redirectsCache) + return true + } else { + ctx.Redirect(redirect.To, redirect.StatusCode) + return true + } + } + } + } + } + + return false +} diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 3845969..f97c6ae 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -11,6 +11,7 @@ import ( "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" ) @@ -52,7 +53,7 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { @@ -103,6 +104,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin // Handle not found error if err != nil && errors.Is(err, gitea.ErrorNotFound) { + // Get and match redirects + redirects := o.getRedirects(giteaClient, redirectsCache) + if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) { + return true + } + if o.TryIndexPages { // copy the o struct & try if an index page exists optionsForIndexPages := *o @@ -110,7 +117,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaClient) { + if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) { return true } } @@ -118,7 +125,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaClient) { + if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) { return true } } @@ -131,11 +138,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin optionsForNotFoundPages.appendTrailingSlash = false for _, notFoundPage := range upstreamNotFoundPages { optionsForNotFoundPages.TargetPath = "/" + notFoundPage - if optionsForNotFoundPages.Upstream(ctx, giteaClient) { + if optionsForNotFoundPages.Upstream(ctx, giteaClient, redirectsCache) { return true } } } + return false } From 7f318f89a6696e9a39d510bb8568ba74c4c0043c Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 16 Jul 2023 22:34:46 +0000 Subject: [PATCH 48/62] Fix escaped error message (#230) - This specific message will [already be generated](https://codeberg.org/Codeberg/pages-server/src/commit/974229681f4cc7f1ed31df9b05eabef2df01380/html/error.go#L44) when `http.StatusMisdirectedRequest` is set as status with [an empty message](https://codeberg.org/Codeberg/pages-server/src/commit/974229681f4cc7f1ed31df9b05eabef2df013809/html/error.go#L25-L28). - Resolves https://codeberg.org/Codeberg/pages-server/issues/228 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/230 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: Gusted Co-committed-by: Gusted --- server/handler/handler_custom_domain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 8742be4..299d7cf 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -49,7 +49,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g }, canonicalLink); works { canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache) if !valid { - html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, "", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! From d720d25e42eb5f1e63462a6665afdd0bebd364ae Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 17 Jul 2023 19:44:58 +0000 Subject: [PATCH 49/62] Use http.NoBody as per linter (#231) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/231 Reviewed-by: Gusted --- server/handler/handler_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index de705ec..6521633 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -1,6 +1,7 @@ package handler import ( + "net/http" "net/http/httptest" "testing" "time" @@ -26,7 +27,7 @@ func TestHandlerPerformance(t *testing.T) { testCase := func(uri string, status int) { t.Run(uri, func(t *testing.T) { - req := httptest.NewRequest("GET", uri, nil) + req := httptest.NewRequest("GET", uri, http.NoBody) w := httptest.NewRecorder() log.Printf("Start: %v\n", time.Now()) From 56d3e291c4579c484b173fbcee07b3334c910c23 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Sun, 27 Aug 2023 10:13:15 +0200 Subject: [PATCH 50/62] Security Fix: clean paths correctly to avoid circumvention of BlacklistedPaths --- server/context/context.go | 6 ++-- server/utils/utils.go | 14 ++++++++++ server/utils/utils_test.go | 56 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/server/context/context.go b/server/context/context.go index 481fee2..6650164 100644 --- a/server/context/context.go +++ b/server/context/context.go @@ -48,11 +48,9 @@ func (c *Context) Redirect(uri string, statusCode int) { http.Redirect(c.RespWriter, c.Req, uri, statusCode) } -// Path returns requested path. -// -// The returned bytes are valid until your request handler returns. +// Path returns the cleaned requested path. func (c *Context) Path() string { - return c.Req.URL.Path + return utils.CleanPath(c.Req.URL.Path) } func (c *Context) Host() string { diff --git a/server/utils/utils.go b/server/utils/utils.go index 30f948d..91ed359 100644 --- a/server/utils/utils.go +++ b/server/utils/utils.go @@ -1,6 +1,8 @@ package utils import ( + "net/url" + "path" "strings" ) @@ -11,3 +13,15 @@ func TrimHostPort(host string) string { } return host } + +func CleanPath(uriPath string) string { + unescapedPath, _ := url.PathUnescape(uriPath) + cleanedPath := path.Join("/", unescapedPath) + + // If the path refers to a directory, add a trailing slash. + if !strings.HasSuffix(cleanedPath, "/") && (strings.HasSuffix(unescapedPath, "/") || strings.HasSuffix(unescapedPath, "/.") || strings.HasSuffix(unescapedPath, "/..")) { + cleanedPath += "/" + } + + return cleanedPath +} diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go index 2532392..b8fcea9 100644 --- a/server/utils/utils_test.go +++ b/server/utils/utils_test.go @@ -11,3 +11,59 @@ func TestTrimHostPort(t *testing.T) { assert.EqualValues(t, "", TrimHostPort(":")) assert.EqualValues(t, "example.com", TrimHostPort("example.com:80")) } + +// TestCleanPath is mostly copied from fasthttp, to keep the behaviour we had before migrating away from it. +// Source (MIT licensed): https://github.com/valyala/fasthttp/blob/v1.48.0/uri_test.go#L154 +// Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia, Kirill Danshin, Erik Dubbelboer, FastHTTP Authors +func TestCleanPath(t *testing.T) { + // double slash + testURIPathNormalize(t, "/aa//bb", "/aa/bb") + + // triple slash + testURIPathNormalize(t, "/x///y/", "/x/y/") + + // multi slashes + testURIPathNormalize(t, "/abc//de///fg////", "/abc/de/fg/") + + // encoded slashes + testURIPathNormalize(t, "/xxxx%2fyyy%2f%2F%2F", "/xxxx/yyy/") + + // dotdot + testURIPathNormalize(t, "/aaa/..", "/") + + // dotdot with trailing slash + testURIPathNormalize(t, "/xxx/yyy/../", "/xxx/") + + // multi dotdots + testURIPathNormalize(t, "/aaa/bbb/ccc/../../ddd", "/aaa/ddd") + + // dotdots separated by other data + testURIPathNormalize(t, "/a/b/../c/d/../e/..", "/a/c/") + + // too many dotdots + testURIPathNormalize(t, "/aaa/../../../../xxx", "/xxx") + testURIPathNormalize(t, "/../../../../../..", "/") + testURIPathNormalize(t, "/../../../../../../", "/") + + // encoded dotdots + testURIPathNormalize(t, "/aaa%2Fbbb%2F%2E.%2Fxxx", "/aaa/xxx") + + // double slash with dotdots + testURIPathNormalize(t, "/aaa////..//b", "/b") + + // fake dotdot + testURIPathNormalize(t, "/aaa/..bbb/ccc/..", "/aaa/..bbb/") + + // single dot + testURIPathNormalize(t, "/a/./b/././c/./d.html", "/a/b/c/d.html") + testURIPathNormalize(t, "./foo/", "/foo/") + testURIPathNormalize(t, "./../.././../../aaa/bbb/../../../././../", "/") + testURIPathNormalize(t, "./a/./.././../b/./foo.html", "/b/foo.html") +} + +func testURIPathNormalize(t *testing.T, requestURI, expectedPath string) { + cleanedPath := CleanPath(requestURI) + if cleanedPath != expectedPath { + t.Fatalf("Unexpected path %q. Expected %q. requestURI=%q", cleanedPath, expectedPath, requestURI) + } +} From ff3cd1ba353f42fefe7ef1c1ba907e9ea2c8bc5e Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Sun, 27 Aug 2023 11:10:55 +0200 Subject: [PATCH 51/62] Fix CI pipeline (replace "pipeline" with "steps") --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5b07053..7de8ac8 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,4 +1,4 @@ -pipeline: +steps: # use vendor to cache dependencies vendor: image: golang:1.20 From b6103c6a1b5b2c0802a75ccead7dc3e3f080da12 Mon Sep 17 00:00:00 2001 From: thepaperpilot Date: Sun, 17 Sep 2023 16:45:20 +0000 Subject: [PATCH 52/62] Chore: Fix env var description (#251) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/251 Reviewed-by: crapStone Co-authored-by: thepaperpilot Co-committed-by: thepaperpilot --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b694e29..1cb6967 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. - `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging). ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet. -- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider. +- `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders. - `ACME_EAB_KID` & `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL. - `ACME_ACCEPT_TERMS` (default: use self-signed certificate): Set this to "true" to accept the Terms of Service of your ACME provider. - `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL. From a8272f0ce9fab991229aad53f512e6aab9c771d2 Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 01:49:29 +0000 Subject: [PATCH 53/62] Don't send server version to client (#254) closes #247 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/254 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- server/handler/handler.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/handler/handler.go b/server/handler/handler.go index 7edcf95..fbdafd7 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -10,7 +10,6 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/version" ) const ( @@ -31,7 +30,7 @@ func Handler(mainDomainSuffix, rawDomain string, log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() ctx := context.New(w, req) - ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) + ctx.RespWriter.Header().Set("Server", "pages-server") // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") @@ -88,14 +87,14 @@ func Handler(mainDomainSuffix, rawDomain string, pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { - log.Debug().Msg("raw domain request detecded") + log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, mainDomainSuffix, rawInfoPage, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - log.Debug().Msg("subdomain request detecded") + log.Debug().Msg("subdomain request detected") handleSubDomain(log, ctx, giteaClient, mainDomainSuffix, defaultPagesBranches, @@ -103,7 +102,7 @@ func Handler(mainDomainSuffix, rawDomain string, pathElements, canonicalDomainCache, redirectsCache) } else { - log.Debug().Msg("custom domain request detecded") + log.Debug().Msg("custom domain request detected") handleCustomDomain(log, ctx, giteaClient, mainDomainSuffix, trimmedHost, From be92f30e6434131fda8a78f31ef6e10ea7e9dfdd Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 15 Nov 2023 10:27:27 +0000 Subject: [PATCH 54/62] Update gitea sdk to e23e8aa3004f (#257) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/257 Reviewed-by: crapStone Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- go.mod | 2 +- go.sum | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bfde7f7..72eb289 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module codeberg.org/codeberg/pages go 1.20 require ( - code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa + code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/go-acme/lego/v4 v4.5.3 github.com/go-sql-driver/mysql v1.6.0 diff --git a/go.sum b/go.sum index b10305c..f78ae18 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= -code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= +code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f h1:nMmwDgUIAWj9XQjzHz5unC3ZMfhhwHRk6rnuwLzdu1o= +code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= @@ -249,8 +249,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -789,8 +789,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -899,8 +899,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -964,14 +962,13 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= From 1e1c67be93f59af3fad68cdc8faad72aac8a742b Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 15:25:14 +0000 Subject: [PATCH 55/62] let gitea client send user-agent with version (#258) closes #255 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/258 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- server/gitea/client.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/server/gitea/client.go b/server/gitea/client.go index 7a2bf63..f3bda54 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -17,12 +17,13 @@ import ( "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/version" ) var ErrorNotFound = errors.New("not found") const ( - // cache key prefixe + // cache key prefixes branchTimestampCacheKeyPrefix = "branchTime" defaultBranchCacheKeyPrefix = "defaultBranch" rawContentCacheKeyPrefix = "rawContent" @@ -76,7 +77,13 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo defaultMimeType = "application/octet-stream" } - sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) + sdk, err := gitea.NewClient( + giteaRoot, + gitea.SetHTTPClient(&stdClient), + gitea.SetToken(giteaAPIToken), + gitea.SetUserAgent("pages-server/"+version.Version), + ) + return &Client{ sdkClient: sdk, responseCache: respCache, @@ -172,7 +179,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str return reader, resp.Response.Header, resp.StatusCode, err } - // now we write to cache and respond at the sime time + // now we write to cache and respond at the same time fileResp := FileResponse{ Exists: true, ETag: resp.Header.Get(ETagHeader), @@ -278,11 +285,11 @@ func shouldRespBeSavedToCache(resp *http.Response) bool { return false } - contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64) + contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64) if err != nil { log.Error().Err(err).Msg("could not parse content length") } // if content to big or could not be determined we not cache it - return contentLeng > 0 && contentLeng < fileCacheSizeLimit + return contentLength > 0 && contentLength < fileCacheSizeLimit } From ea68a82cd22a8a8c1f265260af22b9406f13e3a9 Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 15:25:56 +0000 Subject: [PATCH 56/62] new maintainer (#256) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/256 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1cb6967..9ec3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,13 @@ Since we are working nicely in a team, it might be hard at times to get started (still check out the issues, we always aim to have some things to get you started). If you have any questions, want to work on a feature or could imagine collaborating with us for some time, -feel free to ping us in an issue or in a general Matrix chatgroup. +feel free to ping us in an issue or in a general [Matrix chat room](#chat-for-admins--devs). -You can also contact the maintainers of this project: +You can also contact the maintainer(s) of this project: + +- [crapStone](https://codeberg.org/crapStone) [(Matrix)](https://matrix.to/#/@crapstone:obermui.de) + +Previous maintainers: - [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space) - [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de) From 7f0a4e5ca999189527bf0739467787fe74a51f8a Mon Sep 17 00:00:00 2001 From: crapStone Date: Wed, 15 Nov 2023 17:59:04 +0000 Subject: [PATCH 57/62] small cleanup (#259) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/259 Co-authored-by: crapStone Co-committed-by: crapStone --- FEATURES.md | 14 +++++++--- README.md | 27 +++++++++---------- .../haproxy-sni}/.gitignore | 0 .../haproxy-sni}/README.md | 0 .../haproxy-sni}/dhparam.pem | 0 .../haproxy-sni}/docker-compose.yml | 0 .../haproxy-sni}/gitea-www/index.html | 0 .../haproxy-sni}/gitea.Caddyfile | 0 .../haproxy-certificates/codeberg.org.pem | 0 .../haproxy-certificates/codeberg.org.pem.key | 0 .../haproxy-sni}/haproxy.cfg | 0 .../haproxy-sni}/pages-www/index.html | 0 .../haproxy-sni}/pages.Caddyfile | 0 {haproxy-sni => examples/haproxy-sni}/test.sh | 0 14 files changed, 23 insertions(+), 18 deletions(-) rename {haproxy-sni => examples/haproxy-sni}/.gitignore (100%) rename {haproxy-sni => examples/haproxy-sni}/README.md (100%) rename {haproxy-sni => examples/haproxy-sni}/dhparam.pem (100%) rename {haproxy-sni => examples/haproxy-sni}/docker-compose.yml (100%) rename {haproxy-sni => examples/haproxy-sni}/gitea-www/index.html (100%) rename {haproxy-sni => examples/haproxy-sni}/gitea.Caddyfile (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy-certificates/codeberg.org.pem (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy-certificates/codeberg.org.pem.key (100%) rename {haproxy-sni => examples/haproxy-sni}/haproxy.cfg (100%) rename {haproxy-sni => examples/haproxy-sni}/pages-www/index.html (100%) rename {haproxy-sni => examples/haproxy-sni}/pages.Caddyfile (100%) rename {haproxy-sni => examples/haproxy-sni}/test.sh (100%) diff --git a/FEATURES.md b/FEATURES.md index 7560a1d..3d2f394 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -2,13 +2,19 @@ ## Custom domains -... +Custom domains can be used by creating a `.domains` file with the domain name, e.g.: + +```text +codeberg.page +``` + +You also have to set some DNS records, see the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/using-custom-domain/). ## Redirects Redirects can be created with a `_redirects` file with the following format: -``` +```text # Comment from to [status] ``` @@ -30,7 +36,7 @@ from to [status] Redirects all paths to `/index.html` for single-page apps. -``` +```text /* /index.html 200 ``` @@ -38,7 +44,7 @@ Redirects all paths to `/index.html` for single-page apps. Redirects every path under `/articles` to `/posts` while keeping the path. -``` +```text /articles/* /posts/:splat 302 ``` diff --git a/README.md b/README.md index 9ec3aab..39caa24 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,7 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview) and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/). - - Get It On Codeberg + Get It On Codeberg ## Quickstart @@ -61,18 +60,18 @@ but if you want to run it on a shared IP address (and not a standalone), you'll need to configure your reverse proxy not to terminate the TLS connections, but forward the requests on the IP level to the Pages Server. -You can check out a proof of concept in the `haproxy-sni` folder, -and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38). +You can check out a proof of concept in the `examples/haproxy-sni` folder, +and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/examples/haproxy-sni/haproxy.cfg#L38). -### Environment +### Environment Variables - `HOST` & `PORT` (default: `[::]` & `443`): listen address. - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages. -- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources. +- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources (must be subdomain of `PAGES_DOMAIN`). - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance. - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos. -- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. -- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging). +- `RAW_INFO_PAGE` (default: ): info page for raw resources, shown if no resource is provided. +- `ACME_API` (default: ): set this to to use invalid certificates without any verification (great for debugging). ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet. - `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders. - `ACME_EAB_KID` & `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL. @@ -80,10 +79,9 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL. - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. - See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. + See for available values & additional environment variables. - `LOG_LEVEL` (default: warn): Set this to specify the level of logging. - ## Contributing to the development The Codeberg team is very open to your contribution. @@ -119,7 +117,8 @@ Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://ju run `just dev` now this pages should work: - - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg - - https://momar.localhost.mock.directory:4430/ci-testing/ - - https://momar.localhost.mock.directory:4430/pag/@master/ - - https://mock-pages.codeberg-test.org:4430/README.md + +- +- +- +- diff --git a/haproxy-sni/.gitignore b/examples/haproxy-sni/.gitignore similarity index 100% rename from haproxy-sni/.gitignore rename to examples/haproxy-sni/.gitignore diff --git a/haproxy-sni/README.md b/examples/haproxy-sni/README.md similarity index 100% rename from haproxy-sni/README.md rename to examples/haproxy-sni/README.md diff --git a/haproxy-sni/dhparam.pem b/examples/haproxy-sni/dhparam.pem similarity index 100% rename from haproxy-sni/dhparam.pem rename to examples/haproxy-sni/dhparam.pem diff --git a/haproxy-sni/docker-compose.yml b/examples/haproxy-sni/docker-compose.yml similarity index 100% rename from haproxy-sni/docker-compose.yml rename to examples/haproxy-sni/docker-compose.yml diff --git a/haproxy-sni/gitea-www/index.html b/examples/haproxy-sni/gitea-www/index.html similarity index 100% rename from haproxy-sni/gitea-www/index.html rename to examples/haproxy-sni/gitea-www/index.html diff --git a/haproxy-sni/gitea.Caddyfile b/examples/haproxy-sni/gitea.Caddyfile similarity index 100% rename from haproxy-sni/gitea.Caddyfile rename to examples/haproxy-sni/gitea.Caddyfile diff --git a/haproxy-sni/haproxy-certificates/codeberg.org.pem b/examples/haproxy-sni/haproxy-certificates/codeberg.org.pem similarity index 100% rename from haproxy-sni/haproxy-certificates/codeberg.org.pem rename to examples/haproxy-sni/haproxy-certificates/codeberg.org.pem diff --git a/haproxy-sni/haproxy-certificates/codeberg.org.pem.key b/examples/haproxy-sni/haproxy-certificates/codeberg.org.pem.key similarity index 100% rename from haproxy-sni/haproxy-certificates/codeberg.org.pem.key rename to examples/haproxy-sni/haproxy-certificates/codeberg.org.pem.key diff --git a/haproxy-sni/haproxy.cfg b/examples/haproxy-sni/haproxy.cfg similarity index 100% rename from haproxy-sni/haproxy.cfg rename to examples/haproxy-sni/haproxy.cfg diff --git a/haproxy-sni/pages-www/index.html b/examples/haproxy-sni/pages-www/index.html similarity index 100% rename from haproxy-sni/pages-www/index.html rename to examples/haproxy-sni/pages-www/index.html diff --git a/haproxy-sni/pages.Caddyfile b/examples/haproxy-sni/pages.Caddyfile similarity index 100% rename from haproxy-sni/pages.Caddyfile rename to examples/haproxy-sni/pages.Caddyfile diff --git a/haproxy-sni/test.sh b/examples/haproxy-sni/test.sh similarity index 100% rename from haproxy-sni/test.sh rename to examples/haproxy-sni/test.sh From cbb2ce6d0732bcf2372cbc201fc1c1f2733aadba Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 16 Nov 2023 17:11:35 +0000 Subject: [PATCH 58/62] add go templating engine for error page and make errors more clear (#260) ping #199 closes #213 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/260 Co-authored-by: crapStone Co-committed-by: crapStone --- .woodpecker.yml | 2 +- README.md | 2 +- go.mod | 15 ++++-- go.sum | 30 +++++++---- html/404.html | 37 ------------- html/error.go | 50 ------------------ html/error.html | 38 -------------- html/error_test.go | 38 -------------- html/html.go | 54 +++++++++++++++++-- html/html_test.go | 54 +++++++++++++++++++ html/templates/error.html | 69 +++++++++++++++++++++++++ server/handler/handler.go | 2 +- server/handler/handler_custom_domain.go | 2 +- server/handler/handler_raw_domain.go | 2 +- server/handler/handler_sub_domain.go | 22 +++++--- server/handler/try.go | 2 +- server/upstream/upstream.go | 16 +++--- 17 files changed, 232 insertions(+), 203 deletions(-) delete mode 100644 html/404.html delete mode 100644 html/error.go delete mode 100644 html/error.html delete mode 100644 html/error_test.go create mode 100644 html/html_test.go create mode 100644 html/templates/error.html diff --git a/.woodpecker.yml b/.woodpecker.yml index 7de8ac8..759e2f1 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,7 +1,7 @@ steps: # use vendor to cache dependencies vendor: - image: golang:1.20 + image: golang:1.21 commands: - go mod vendor diff --git a/README.md b/README.md index 39caa24..fb2a4b9 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Thank you very much. ### Test Server -Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://just.systems/man/en/) installed. +Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed. run `just dev` now this pages should work: diff --git a/go.mod b/go.mod index 72eb289..eba292e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module codeberg.org/codeberg/pages -go 1.20 +go 1.21 + +toolchain go1.21.4 require ( code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f @@ -10,6 +12,7 @@ require ( github.com/joho/godotenv v1.4.0 github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 + github.com/microcosm-cc/bluemonday v1.0.26 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 @@ -35,6 +38,7 @@ require ( github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect github.com/aws/aws-sdk-go v1.39.0 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cloudflare/cloudflare-go v0.20.0 // indirect @@ -61,6 +65,7 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/gophercloud/gophercloud v0.16.0 // indirect github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -115,11 +120,11 @@ require ( github.com/vultr/govultr/v2 v2.7.1 // indirect go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect google.golang.org/appengine v1.6.5 // indirect diff --git a/go.sum b/go.sum index f78ae18..7ea8b78 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo= github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -251,6 +253,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -278,6 +281,8 @@ github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae/go.mod h1:wx8HMD github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -477,6 +482,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -756,8 +763,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -790,7 +797,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -826,8 +834,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -899,11 +908,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -911,8 +921,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -963,7 +974,8 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/html/404.html b/html/404.html deleted file mode 100644 index 7c721b5..0000000 --- a/html/404.html +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - %status% - - - - - - - - - -

- Page not found! -

-
- Sorry, but this page couldn't be found or is inaccessible (%status%).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! -
- - - Static pages made easy - Codeberg Pages - - - diff --git a/html/error.go b/html/error.go deleted file mode 100644 index 206b123..0000000 --- a/html/error.go +++ /dev/null @@ -1,50 +0,0 @@ -package html - -import ( - "html/template" - "net/http" - "strconv" - "strings" - - "codeberg.org/codeberg/pages/server/context" -) - -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, -// with "%status%" and %message% replaced with the provided statusCode and msg -func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { - ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") - ctx.RespWriter.WriteHeader(statusCode) - - msg = generateResponse(msg, statusCode) - - _, _ = ctx.RespWriter.Write([]byte(msg)) -} - -// TODO: use template engine -func generateResponse(msg string, statusCode int) string { - if msg == "" { - msg = strings.ReplaceAll(NotFoundPage, - "%status%", - strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) - } else { - msg = strings.ReplaceAll( - strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)), - "%status%", - http.StatusText(statusCode)) - } - - return msg -} - -func errorMessage(statusCode int) string { - message := http.StatusText(statusCode) - - switch statusCode { - case http.StatusMisdirectedRequest: - message += " - domain not specified in .domains file" - case http.StatusFailedDependency: - message += " - target repo/branch doesn't exist or is private" - } - - return message -} diff --git a/html/error.html b/html/error.html deleted file mode 100644 index f1975f7..0000000 --- a/html/error.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - %status% - - - - - - - - - -

- %status%! -

-
- Sorry, but this page couldn't be served.
- We got an "%message%"
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! -
- - - Static pages made easy - Codeberg Pages - - - diff --git a/html/error_test.go b/html/error_test.go deleted file mode 100644 index f5da08c..0000000 --- a/html/error_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package html - -import ( - "net/http" - "strings" - "testing" -) - -func TestValidMessage(t *testing.T) { - testString := "requested blacklisted path" - statusCode := http.StatusForbidden - - expected := strings.ReplaceAll( - strings.ReplaceAll(ErrorPage, "%message%", testString), - "%status%", - http.StatusText(statusCode)) - actual := generateResponse(testString, statusCode) - - if expected != actual { - t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual) - } -} - -func TestMessageWithHtml(t *testing.T) { - testString := `abc +func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { + ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") + ctx.RespWriter.WriteHeader(statusCode) + + templateContext := TemplateContext{ + StatusCode: statusCode, + StatusText: http.StatusText(statusCode), + Message: sanitizer.Sanitize(msg), + } + + err := errorTemplate.Execute(ctx.RespWriter, templateContext) + if err != nil { + log.Err(err).Str("message", msg).Int("status", statusCode).Msg("could not write response") + } +} + +func createBlueMondayPolicy() *bluemonday.Policy { + p := bluemonday.NewPolicy() + + p.AllowElements("code") + + return p +} diff --git a/html/html_test.go b/html/html_test.go new file mode 100644 index 0000000..b395bb2 --- /dev/null +++ b/html/html_test.go @@ -0,0 +1,54 @@ +package html + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizerSimpleString(t *testing.T) { + str := "simple text message without any html elements" + + assert.Equal(t, str, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithCodeTag(t *testing.T) { + str := "simple text message with html tag" + + assert.Equal(t, str, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithCodeTagWithAttribute(t *testing.T) { + str := "simple text message with html tag" + expected := "simple text message with html tag" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithATag(t *testing.T) { + str := "simple text message with a link to another page" + expected := "simple text message with a link to another page" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithATagAndHref(t *testing.T) { + str := "simple text message with a link to another page" + expected := "simple text message with a link to another page" + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithImgTag(t *testing.T) { + str := "simple text message with a \"not" + expected := "simple text message with a " + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} + +func TestSanitizerStringWithImgTagAndOnerrorAttribute(t *testing.T) { + str := "simple text message with a \"not" + expected := "simple text message with a " + + assert.Equal(t, expected, sanitizer.Sanitize(str)) +} diff --git a/html/templates/error.html b/html/templates/error.html new file mode 100644 index 0000000..6094a26 --- /dev/null +++ b/html/templates/error.html @@ -0,0 +1,69 @@ + + + + + + {{.StatusText}} + + + + + + + + + + +

{{.StatusText}} ({{.StatusCode}})!

+
+

Sorry, but this page couldn't be served.

+

"{{.Message}}"

+

+ We hope this isn't a problem on our end ;) - Make sure to check the + troubleshooting section in the Docs! +

+
+ + + Static pages made easy - + Codeberg Pages + + + diff --git a/server/handler/handler.go b/server/handler/handler.go index fbdafd7..a3011f3 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -65,7 +65,7 @@ func Handler(mainDomainSuffix, rawDomain string, // Block blacklisted paths (like ACME challenges) for _, blacklistedPath := range blacklistedPaths { if strings.HasPrefix(ctx.Path(), blacklistedPath) { - html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) + html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden) return } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 299d7cf..8742be4 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -49,7 +49,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g }, canonicalLink); works { canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache) if !valid { - html.ReturnErrorPage(ctx, "", http.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHost { // only redirect if the target is also a codeberg page! diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index aa41c52..b87991e 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -61,7 +61,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, - fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), + fmt.Sprintf("raw domain could not find repo %s/%s or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), http.StatusNotFound) } } diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 8731bec..e7cb3a6 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -55,9 +55,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite log.Trace().Msg("tryUpstream: serve with specified repo and branch") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), - http.StatusFailedDependency) + html.ReturnErrorPage( + ctx, + formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency, + ) } return } @@ -85,9 +87,11 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) } else { - html.ReturnErrorPage(ctx, - fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), - http.StatusFailedDependency) + html.ReturnErrorPage( + ctx, + formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + http.StatusFailedDependency, + ) } return } @@ -143,6 +147,10 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite // Couldn't find a valid repo/branch html.ReturnErrorPage(ctx, - fmt.Sprintf("could not find a valid repository[%s]", targetRepo), + fmt.Sprintf("could not find a valid repository or branch for repository: %s", targetRepo), http.StatusNotFound) } + +func formatSetBranchNotFoundMessage(branch, owner, repo string) string { + return fmt.Sprintf("explicitly set branch %q does not exist at %s/%s", branch, owner, repo) +} diff --git a/server/handler/try.go b/server/handler/try.go index 6cfe08e..838ae27 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -41,7 +41,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "", ctx.StatusCode) + html.ReturnErrorPage(ctx, "gitea client failed", ctx.StatusCode) } } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index f97c6ae..1a444e4 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -53,11 +53,11 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) (final bool) { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { - html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest) + html.ReturnErrorPage(ctx, "gitea client: either repo owner or name info is missing", http.StatusBadRequest) return true } @@ -67,7 +67,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // handle 404 if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist { html.ReturnErrorPage(ctx, - fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo), + fmt.Sprintf("branch %q for %s/%s not found", o.TargetBranch, o.TargetOwner, o.TargetRepo), http.StatusNotFound) return true } @@ -75,7 +75,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // handle unexpected errors if err != nil { html.ReturnErrorPage(ctx, - fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err), + fmt.Sprintf("could not get timestamp of branch %q: '%v'", o.TargetBranch, err), http.StatusFailedDependency) return true } @@ -153,16 +153,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi var msg string if err != nil { - msg = "gitea client returned unexpected error" + msg = "gitea client: returned unexpected error" log.Error().Err(err).Msg(msg) - msg = fmt.Sprintf("%s: %v", msg, err) + msg = fmt.Sprintf("%s: '%v'", msg, err) } if reader == nil { - msg = "gitea client returned no reader" + msg = "gitea client: returned no reader" log.Error().Msg(msg) } if statusCode != http.StatusOK { - msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode) + msg = fmt.Sprintf("gitea client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) log.Error().Msg(msg) } From fffb8ffcb68d44572830bf03f90b4fa5b9db262e Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 16 Nov 2023 17:33:39 +0000 Subject: [PATCH 59/62] remove use of rawInfoPage redirect (#261) closes #244 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/261 Co-authored-by: crapStone Co-committed-by: crapStone --- cmd/flags.go | 7 ------- cmd/main.go | 2 -- server/handler/handler.go | 3 +-- server/handler/handler_raw_domain.go | 10 +++++++--- server/handler/handler_test.go | 1 - 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index a71dd35..7ac94e6 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -72,13 +72,6 @@ var ( EnvVars: []string{"RAW_DOMAIN"}, Value: "raw.codeberg.page", }, - // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). - &cli.StringFlag{ - Name: "raw-info-page", - Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", - EnvVars: []string{"RAW_INFO_PAGE"}, - Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", - }, // ######################### // ### Page Server Setup ### diff --git a/cmd/main.go b/cmd/main.go index 84915c9..683e859 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -47,7 +47,6 @@ func Serve(ctx *cli.Context) error { rawDomain := ctx.String("raw-domain") defaultBranches := ctx.StringSlice("pages-branch") mainDomainSuffix := ctx.String("pages-domain") - rawInfoPage := ctx.String("raw-info-page") listeningHost := ctx.String("host") listeningSSLPort := ctx.Uint("port") listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) @@ -137,7 +136,6 @@ func Serve(ctx *cli.Context) error { // Create ssl handler based on settings sslHandler := handler.Handler(mainDomainSuffix, rawDomain, giteaClient, - rawInfoPage, BlacklistedPaths, allowedCorsDomains, defaultBranches, dnsLookupCache, canonicalDomainCache, redirectsCache) diff --git a/server/handler/handler.go b/server/handler/handler.go index a3011f3..7da5d39 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -21,7 +21,6 @@ const ( // Handler handles a single HTTP request to the web server. func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, - rawInfoPage string, blacklistedPaths, allowedCorsDomains []string, defaultPagesBranches []string, dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, @@ -89,7 +88,7 @@ func Handler(mainDomainSuffix, rawDomain string, if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, - mainDomainSuffix, rawInfoPage, + mainDomainSuffix, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index b87991e..caa8209 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -16,7 +16,7 @@ import ( ) func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client, - mainDomainSuffix, rawInfoPage string, + mainDomainSuffix string, trimmedHost string, pathElements []string, canonicalDomainCache, redirectsCache cache.SetGetKey, @@ -25,8 +25,12 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie log.Debug().Msg("raw domain") if len(pathElements) < 2 { - // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) + html.ReturnErrorPage( + ctx, + "a url in the form of https://{domain}/{owner}/{repo}[/@{branch}]/{path} is required", + http.StatusBadRequest, + ) + return } diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index 6521633..d04ebda 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -16,7 +16,6 @@ func TestHandlerPerformance(t *testing.T) { testHandler := Handler( "codeberg.page", "raw.codeberg.org", giteaClient, - "https://docs.codeberg.org/pages/raw-content/", []string{"/.well-known/acme-challenge/"}, []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, []string{"pages"}, From dd5124912ecde8246bdcaad223ecde7737928429 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 17 Nov 2023 21:46:52 +0000 Subject: [PATCH 60/62] CI: run on pull only once (#264) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/264 Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- .woodpecker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 759e2f1..de8341c 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,3 +1,6 @@ +when: + branch: main + steps: # use vendor to cache dependencies vendor: From 15916444e18c71c066ffe8a7cd37cf272ec115eb Mon Sep 17 00:00:00 2001 From: Gusted Date: Thu, 18 Jan 2024 14:31:46 +0000 Subject: [PATCH 61/62] Fix panic in formatting not found message (#276) Fix panic that was found in the logs, `targetOpt` is `nil`: http2: panic serving 10.0.3.1:[...]: runtime error: invalid memory address or nil pointer dereference net/http.(*http2serverConn).runHandler.func1() /usr/local/go/src/net/http/h2_bundle.go:6104 +0x145 panic({0x19c6820, 0x2d66db0}) /usr/local/go/src/runtime/panic.go:884 +0x213 codeberg.org/codeberg/pages/server/handler.handleSubDomain({{0x2008c68, 0xc00047df90}, 0x2, {0x0, 0x0}, {0xc0fe3ef800, 0x55, 0x1f4}, {0xc00047dfa0, 0x1, ...}, ...}, ...) /woodpecker/src/codeberg.org/Codeberg/pages-server/server/handler/handler_sub_domain.go:59 +0x5e0 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/276 Reviewed-by: crapStone Co-authored-by: Gusted Co-committed-by: Gusted --- server/handler/handler_sub_domain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index e7cb3a6..6c14393 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -57,7 +57,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite } else { html.ReturnErrorPage( ctx, - formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + formatSetBranchNotFoundMessage(pathElements[1][1:], targetOwner, pathElements[0]), http.StatusFailedDependency, ) } @@ -89,7 +89,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite } else { html.ReturnErrorPage( ctx, - formatSetBranchNotFoundMessage(targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo), + formatSetBranchNotFoundMessage(targetBranch, targetOwner, defaultPagesRepo), http.StatusFailedDependency, ) } From a09bee68adf43c2ade9eb3264708f29b68d599ea Mon Sep 17 00:00:00 2001 From: "Panagiotis \"Ivory\" Vasilopoulos" Date: Thu, 18 Jan 2024 20:35:32 +0000 Subject: [PATCH 62/62] Meta: Redirect user support to Codeberg/Community (#277) Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/277 Reviewed-by: crapStone Co-authored-by: Panagiotis "Ivory" Vasilopoulos Co-committed-by: Panagiotis "Ivory" Vasilopoulos --- .gitea/ISSUE_TEMPLATE/config.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitea/ISSUE_TEMPLATE/config.yml diff --git a/.gitea/ISSUE_TEMPLATE/config.yml b/.gitea/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5a9cce6 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Codeberg Pages Usage Support + url: https://codeberg.org/Codeberg/Community/issues/ + about: If you need help with configuring Codeberg Pages on codeberg.org, please go here.