From b9966487f6f7a9fc88ae285c7350917e455f27a0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 20:37:20 +0100 Subject: [PATCH] switch to std http implementation instead of fasthttp (#106) close #100 close #109 close #113 close #28 close #63 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/106 --- cmd/main.go | 41 ++-- go.mod | 17 +- go.sum | 33 +-- html/404.html | 8 +- html/error.go | 47 +++-- html/error.html | 38 ++++ html/html.go | 5 +- integration/get_test.go | 33 ++- server/certificates/certificates.go | 36 ++-- server/context/context.go | 62 ++++++ server/database/mock.go | 2 +- server/dns/const.go | 6 - server/dns/dns.go | 4 + server/gitea/cache.go | 112 +++++++++- server/gitea/client.go | 308 ++++++++++++++++++++-------- server/gitea/client_test.go | 23 --- server/gitea/fasthttp.go | 15 -- server/handler.go | 186 +++++++++-------- server/handler_test.go | 36 ++-- server/helpers.go | 8 +- server/setup.go | 54 ++--- server/try.go | 21 +- server/upstream/const.go | 24 --- server/upstream/domains.go | 10 + server/upstream/helper.go | 74 ++----- server/upstream/upstream.go | 194 +++++++++--------- server/utils/utils.go | 8 +- server/utils/utils_test.go | 6 +- 28 files changed, 827 insertions(+), 584 deletions(-) create mode 100644 html/error.html create mode 100644 server/context/context.go delete mode 100644 server/dns/const.go delete mode 100644 server/gitea/client_test.go delete mode 100644 server/gitea/fasthttp.go delete mode 100644 server/upstream/const.go diff --git a/cmd/main.go b/cmd/main.go index 41809cb..a3a61e1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,12 +1,12 @@ package cmd import ( - "bytes" "context" "crypto/tls" "errors" "fmt" "net" + "net/http" "os" "strings" "time" @@ -24,15 +24,15 @@ import ( // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. // TODO: make it a flag -var AllowedCorsDomains = [][]byte{ - []byte("fonts.codeberg.org"), - []byte("design.codeberg.org"), +var AllowedCorsDomains = []string{ + "fonts.codeberg.org", + "design.codeberg.org", } // BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. // TODO: Make it a flag too -var BlacklistedPaths = [][]byte{ - []byte("/.well-known/acme-challenge/"), +var BlacklistedPaths = []string{ + "/.well-known/acme-challenge/", } // Serve sets up and starts the web server. @@ -47,7 +47,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") - mainDomainSuffix := []byte(ctx.String("pages-domain")) + mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") @@ -65,12 +65,12 @@ func Serve(ctx *cli.Context) error { allowedCorsDomains := AllowedCorsDomains if len(rawDomain) != 0 { - allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain)) + allowedCorsDomains = append(allowedCorsDomains, rawDomain) } // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash - if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) { - mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) + if !strings.HasPrefix(mainDomainSuffix, ".") { + mainDomainSuffix = "." + mainDomainSuffix } keyCache := cache.NewKeyValueCache() @@ -79,26 +79,22 @@ func Serve(ctx *cli.Context) error { canonicalDomainCache := cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains dnsLookupCache := cache.NewKeyValueCache() - // branchTimestampCache stores branch timestamps for faster cache checking - branchTimestampCache := cache.NewKeyValueCache() - // fileResponseCache stores responses from the Gitea server - // TODO: make this an MRU cache with a size limit - fileResponseCache := cache.NewKeyValueCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewKeyValueCache() - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) + giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) if err != nil { return fmt.Errorf("could not create new gitea client: %v", err) } // Create handler based on settings - handler := server.Handler(mainDomainSuffix, []byte(rawDomain), + httpsHandler := server.Handler(mainDomainSuffix, rawDomain, giteaClient, giteaRoot, rawInfoPage, BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) + dnsLookupCache, canonicalDomainCache) - fastServer := server.SetupServer(handler) - httpServer := server.SetupHTTPACMEChallengeServer(challengeCache) + httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) @@ -138,7 +134,7 @@ func Serve(ctx *cli.Context) error { if enableHTTPServer { go func() { log.Info().Msg("Start HTTP server listening on :80") - err := httpServer.ListenAndServe("[::]:80") + err := http.ListenAndServe("[::]:80", httpHandler) if err != nil { log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") } @@ -147,8 +143,7 @@ func Serve(ctx *cli.Context) error { // Start the web fastServer log.Info().Msgf("Start listening on %s", listener.Addr()) - err = fastServer.Serve(listener) - if err != nil { + if err := http.Serve(listener, httpsHandler); err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") } diff --git a/go.mod b/go.mod index 479c328..77ed762 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module codeberg.org/codeberg/pages -go 1.18 +go 1.19 require ( + code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 @@ -11,8 +12,6 @@ require ( github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 - github.com/valyala/fasthttp v1.31.0 - github.com/valyala/fastjson v1.6.3 ) require ( @@ -31,7 +30,6 @@ require ( github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect - github.com/andybalholm/brotli v1.0.2 // indirect github.com/aws/aws-sdk-go v1.39.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect @@ -39,6 +37,7 @@ require ( github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dnsimple/dnsimple-go v0.70.1 // indirect @@ -46,6 +45,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/go-errors/errors v1.0.1 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect @@ -57,13 +57,13 @@ require ( github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect github.com/jarcoal/httpmock v1.0.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.7 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect - github.com/klauspost/compress v1.13.4 // indirect github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect @@ -104,15 +104,14 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect github.com/vultr/govultr/v2 v2.7.1 // indirect go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect diff --git a/go.sum b/go.sum index 23a58bc..a44001c 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -66,8 +68,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbxuNdz3FeyvHFnRjmo= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= -github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -106,6 +106,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/deepmap/oapi-codegen v1.6.1 h1:2BvsmRb6pogGNtr8Ann+esAbSKFXx2CZN18VpAMecnw= github.com/deepmap/oapi-codegen v1.6.1/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -136,6 +138,8 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -182,7 +186,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -243,6 +246,9 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -282,8 +288,6 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcM github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -494,15 +498,9 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE= -github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= -github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 h1:TFXGGMHmml4rs29PdPisC/aaCzOxUu1Vsh9on/IpUfE= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14/go.mod h1:RWc47jtnVuQv6+lY3c768WtXCas/Xi+U5UFc5xULmYg= github.com/vultr/govultr/v2 v2.7.1 h1:uF9ERet++Gb+7Cqs3p1P6b6yebeaZqVd7t5P2uZCaJU= @@ -539,8 +537,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -604,8 +604,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -667,12 +667,13 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/html/404.html b/html/404.html index 21d968e..7c721b5 100644 --- a/html/404.html +++ b/html/404.html @@ -3,7 +3,7 @@ - %status + %status% @@ -23,11 +23,11 @@

- Page not found! + Page not found!

- Sorry, but this page couldn't be found or is inaccessible (%status).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! + Sorry, but this page couldn't be found or is inaccessible (%status%).
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
diff --git a/html/error.go b/html/error.go index 325dada..826c42b 100644 --- a/html/error.go +++ b/html/error.go @@ -1,24 +1,45 @@ package html import ( - "bytes" + "net/http" "strconv" + "strings" - "github.com/valyala/fasthttp" + "codeberg.org/codeberg/pages/server/context" ) -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) { - ctx.Response.SetStatusCode(code) - ctx.Response.Header.SetContentType("text/html; charset=utf-8") - message := fasthttp.StatusMessage(code) - if code == fasthttp.StatusMisdirectedRequest { - message += " - domain not specified in .domains file" +// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, +// with "%status%" and %message% replaced with the provided statusCode and msg +func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) { + ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") + ctx.RespWriter.WriteHeader(statusCode) + + if msg == "" { + msg = errorBody(statusCode) + } else { + // TODO: use template engine + msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode)) } - if code == fasthttp.StatusFailedDependency { + + _, _ = ctx.RespWriter.Write([]byte(msg)) +} + +func errorMessage(statusCode int) string { + message := http.StatusText(statusCode) + + switch statusCode { + case http.StatusMisdirectedRequest: + message += " - domain not specified in .domains file" + case http.StatusFailedDependency: message += " - target repo/branch doesn't exist or is private" } - // TODO: use template engine? - ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) + + return message +} + +// TODO: use template engine +func errorBody(statusCode int) string { + return strings.ReplaceAll(NotFoundPage, + "%status%", + strconv.Itoa(statusCode)+" "+errorMessage(statusCode)) } diff --git a/html/error.html b/html/error.html new file mode 100644 index 0000000..f1975f7 --- /dev/null +++ b/html/error.html @@ -0,0 +1,38 @@ + + + + + + %status% + + + + + + + + + +

+ %status%! +

+
+ Sorry, but this page couldn't be served.
+ We got an "%message%"
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs! +
+ + + Static pages made easy - Codeberg Pages + + + diff --git a/html/html.go b/html/html.go index d223e15..a76ce59 100644 --- a/html/html.go +++ b/html/html.go @@ -3,4 +3,7 @@ package html import _ "embed" //go:embed 404.html -var NotFoundPage []byte +var NotFoundPage string + +//go:embed error.html +var ErrorPage string diff --git a/integration/get_test.go b/integration/get_test.go index 6054e17..8794651 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -25,7 +25,7 @@ func TestGetRedirect(t *testing.T) { t.FailNow() } assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location")) - assert.EqualValues(t, 0, getSize(resp.Body)) + assert.EqualValues(t, `Temporary Redirect.`, strings.TrimSpace(string(getBytes(resp.Body)))) } func TestGetContent(t *testing.T) { @@ -44,12 +44,13 @@ func TestGetContent(t *testing.T) { // specify branch resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.True(t, getSize(resp.Body) > 1000) - assert.Len(t, resp.Header.Get("ETag"), 42) + assert.Len(t, resp.Header.Get("ETag"), 44) // access branch name contains '/' resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/") @@ -59,7 +60,7 @@ func TestGetContent(t *testing.T) { } assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.True(t, getSize(resp.Body) > 100) - assert.Len(t, resp.Header.Get("ETag"), 42) + assert.Len(t, resp.Header.Get("ETag"), 44) // TODO: test get of non cachable content (content size > fileCacheSizeLimit) } @@ -68,9 +69,10 @@ func TestCustomDomain(t *testing.T) { log.Println("=== TestCustomDomain ===") resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type")) assert.EqualValues(t, "106", resp.Header.Get("Content-Length")) assert.EqualValues(t, 106, getSize(resp.Body)) @@ -81,9 +83,10 @@ func TestGetNotFound(t *testing.T) { // test custom not found pages resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type")) assert.EqualValues(t, "37", resp.Header.Get("Content-Length")) assert.EqualValues(t, 37, getSize(resp.Body)) @@ -94,9 +97,10 @@ func TestFollowSymlink(t *testing.T) { resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type")) assert.EqualValues(t, "4", resp.Header.Get("Content-Length")) body := getBytes(resp.Body) @@ -109,14 +113,27 @@ func TestLFSSupport(t *testing.T) { resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") assert.NoError(t, err) - if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + if !assert.NotNil(t, resp) { t.FailNow() } + assert.EqualValues(t, http.StatusOK, resp.StatusCode) body := strings.TrimSpace(string(getBytes(resp.Body))) assert.EqualValues(t, 12, len(body)) assert.EqualValues(t, "actual value", body) } +func TestGetOptions(t *testing.T) { + log.Println("=== TestGetOptions ===") + req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", nil) + resp, err := getTestHTTPSClient().Do(req) + assert.NoError(t, err) + if !assert.NotNil(t, resp) { + t.FailNow() + } + assert.EqualValues(t, http.StatusNoContent, resp.StatusCode) + assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow")) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 2f59fb4..429ab23 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -36,7 +36,7 @@ import ( ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TLSConfig(mainDomainSuffix []byte, +func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, dnsProvider string, acmeUseRateLimits bool, @@ -47,7 +47,6 @@ func TLSConfig(mainDomainSuffix []byte, // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { sni := strings.ToLower(strings.TrimSpace(info.ServerName)) - sniBytes := []byte(sni) if len(sni) < 1 { return nil, errors.New("missing sni") } @@ -69,23 +68,20 @@ func TLSConfig(mainDomainSuffix []byte, } targetOwner := "" - if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) { + if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) { // deliver default certificate for the main domain (*.codeberg.page) - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } else { _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, mainDomainSuffix, canonicalDomainCache) if !valid { - sniBytes = mainDomainSuffix - sni = string(sniBytes) + sni = mainDomainSuffix } } } @@ -98,9 +94,9 @@ func TLSConfig(mainDomainSuffix []byte, var tlsCertificate tls.Certificate var err error var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { + if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { // request a new certificate - if bytes.Equal(sniBytes, mainDomainSuffix) { + if strings.EqualFold(sni, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } @@ -192,7 +188,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { // parse certificate from database res, err := certDB.Get(string(sni)) if err != nil { @@ -208,7 +204,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs } // TODO: document & put into own function - if !bytes.Equal(sni, mainDomainSuffix) { + if !strings.EqualFold(sni, mainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { panic(err) @@ -239,7 +235,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -252,7 +248,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } @@ -405,7 +401,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce return myAcmeConfig, nil } -func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { +func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { // getting main cert before ACME account so that we can fail here without hitting rate limits mainCertBytes, err := certDB.Get(string(mainDomainSuffix)) if err != nil { @@ -460,7 +456,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * return nil } -func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { +func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { for { // clean up expired certs now := time.Now() @@ -468,7 +464,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi keyDatabaseIterator := certDB.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { - if !bytes.Equal(key, mainDomainSuffix) { + if !strings.EqualFold(string(key), mainDomainSuffix) { resGob := bytes.NewBuffer(resBytes) resDec := gob.NewDecoder(resGob) res := &certificate.Resource{} diff --git a/server/context/context.go b/server/context/context.go new file mode 100644 index 0000000..be01df0 --- /dev/null +++ b/server/context/context.go @@ -0,0 +1,62 @@ +package context + +import ( + stdContext "context" + "net/http" +) + +type Context struct { + RespWriter http.ResponseWriter + Req *http.Request + StatusCode int +} + +func New(w http.ResponseWriter, r *http.Request) *Context { + return &Context{ + RespWriter: w, + Req: r, + StatusCode: http.StatusOK, + } +} + +func (c *Context) Context() stdContext.Context { + if c.Req != nil { + return c.Req.Context() + } + return stdContext.Background() +} + +func (c *Context) Response() *http.Response { + if c.Req != nil && c.Req.Response != nil { + return c.Req.Response + } + return nil +} + +func (c *Context) String(raw string, status ...int) { + code := http.StatusOK + if len(status) != 0 { + code = status[0] + } + c.RespWriter.WriteHeader(code) + _, _ = c.RespWriter.Write([]byte(raw)) +} + +func (c *Context) IsMethod(m string) bool { + return c.Req.Method == m +} + +func (c *Context) Redirect(uri string, statusCode int) { + http.Redirect(c.RespWriter, c.Req, uri, statusCode) +} + +// Path returns requested path. +// +// The returned bytes are valid until your request handler returns. +func (c *Context) Path() string { + return c.Req.URL.Path +} + +func (c *Context) Host() string { + return c.Req.URL.Host +} diff --git a/server/database/mock.go b/server/database/mock.go index e6c1b5a..dfe2316 100644 --- a/server/database/mock.go +++ b/server/database/mock.go @@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error { func (p tmpDB) Get(name string) (*certificate.Resource, error) { cert, has := p.intern.Get(name) if !has { - return nil, fmt.Errorf("cert for '%s' not found", name) + return nil, fmt.Errorf("cert for %q not found", name) } return cert.(*certificate.Resource), nil } diff --git a/server/dns/const.go b/server/dns/const.go deleted file mode 100644 index bb2413b..0000000 --- a/server/dns/const.go +++ /dev/null @@ -1,6 +0,0 @@ -package dns - -import "time" - -// lookupCacheTimeout specifies the timeout for the DNS lookup cache. -var lookupCacheTimeout = 15 * time.Minute diff --git a/server/dns/dns.go b/server/dns/dns.go index dc759b0..818e29a 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -3,10 +3,14 @@ package dns import ( "net" "strings" + "time" "codeberg.org/codeberg/pages/server/cache" ) +// lookupCacheTimeout specifies the timeout for the DNS lookup cache. +var lookupCacheTimeout = 15 * time.Minute + // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 932ff3c..b11a370 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -1,12 +1,116 @@ package gitea +import ( + "bytes" + "fmt" + "io" + "net/http" + "time" + + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/server/cache" +) + +const ( + // defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. + defaultBranchCacheTimeout = 15 * time.Minute + + // branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter + // than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be + // picked up faster, while still allowing the content to be cached longer if nothing changes. + branchExistenceCacheTimeout = 5 * time.Minute + + // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending + // on your available memory. + // TODO: move as option into cache interface + fileCacheTimeout = 5 * time.Minute + + // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. + fileCacheSizeLimit = int64(1000 * 1000) +) + type FileResponse struct { - Exists bool - ETag []byte - MimeType string - Body []byte + Exists bool + IsSymlink bool + ETag string + MimeType string + Body []byte } func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } + +func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) { + header := make(http.Header) + var statusCode int + + if f.Exists { + statusCode = http.StatusOK + } else { + statusCode = http.StatusNotFound + } + + if f.IsSymlink { + header.Set(giteaObjectTypeHeader, objTypeSymlink) + } + header.Set(ETagHeader, f.ETag) + header.Set(ContentTypeHeader, f.MimeType) + header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body))) + header.Set(PagesCacheIndicatorHeader, "true") + + log.Trace().Msgf("fileCache for %q used", cacheKey) + return header, statusCode +} + +type BranchTimestamp struct { + Branch string + Timestamp time.Time + notFound bool +} + +type writeCacheReader struct { + originalReader io.ReadCloser + buffer *bytes.Buffer + rileResponse *FileResponse + cacheKey string + cache cache.SetGetKey + hasError bool +} + +func (t *writeCacheReader) Read(p []byte) (n int, err error) { + n, err = t.originalReader.Read(p) + if err != nil { + log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) + t.hasError = true + } else if n > 0 { + _, _ = t.buffer.Write(p[:n]) + } + return +} + +func (t *writeCacheReader) Close() error { + if !t.hasError { + fc := *t.rileResponse + fc.Body = t.buffer.Bytes() + _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + } + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError) + return t.originalReader.Close() +} + +func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { + if r == nil || cache == nil || cacheKey == "" { + log.Error().Msg("could not create CacheReader") + return nil + } + + return &writeCacheReader{ + originalReader: r, + buffer: bytes.NewBuffer(make([]byte, 0)), + rileResponse: &f, + cache: cache, + cacheKey: cacheKey, + } +} diff --git a/server/gitea/client.go b/server/gitea/client.go index 16cba84..c63ee21 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -1,142 +1,276 @@ package gitea import ( + "bytes" "errors" "fmt" + "io" + "mime" + "net/http" "net/url" + "path" + "strconv" "strings" "time" + "code.gitea.io/sdk/gitea" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" -) -const ( - giteaAPIRepos = "/api/v1/repos/" - giteaObjectTypeHeader = "X-Gitea-Object-Type" + "codeberg.org/codeberg/pages/server/cache" ) var ErrorNotFound = errors.New("not found") +const ( + // cache key prefixe + branchTimestampCacheKeyPrefix = "branchTime" + defaultBranchCacheKeyPrefix = "defaultBranch" + rawContentCacheKeyPrefix = "rawContent" + + // pages server + PagesCacheIndicatorHeader = "X-Pages-Cache" + symlinkReadLimit = 10000 + + // gitea + giteaObjectTypeHeader = "X-Gitea-Object-Type" + objTypeSymlink = "symlink" + + // std + ETagHeader = "ETag" + ContentTypeHeader = "Content-Type" + ContentLengthHeader = "Content-Length" +) + type Client struct { - giteaRoot string - giteaAPIToken string - fastClient *fasthttp.Client - infoTimeout time.Duration - contentTimeout time.Duration + sdkClient *gitea.Client + responseCache cache.SetGetKey followSymlinks bool supportLFS bool + + forbiddenMimeTypes map[string]bool + defaultMimeType string } -// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package -func joinURL(baseURL string, paths ...string) string { - p := make([]string, 0, len(paths)) - for i := range paths { - path := strings.TrimSpace(paths[i]) - path = strings.Trim(path, "/") - if len(path) != 0 { - p = append(p, path) - } - } - - return baseURL + "/" + strings.Join(p, "/") -} - -func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) { +func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { rootURL, err := url.Parse(giteaRoot) + if err != nil { + return nil, err + } giteaRoot = strings.Trim(rootURL.String(), "/") + stdClient := http.Client{Timeout: 10 * time.Second} + + // TODO: pass down + var ( + forbiddenMimeTypes map[string]bool + defaultMimeType string + ) + + if forbiddenMimeTypes == nil { + forbiddenMimeTypes = make(map[string]bool) + } + if defaultMimeType == "" { + defaultMimeType = "application/octet-stream" + } + + sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) return &Client{ - giteaRoot: giteaRoot, - giteaAPIToken: giteaAPIToken, - infoTimeout: 5 * time.Second, - contentTimeout: 10 * time.Second, - fastClient: getFastHTTPClient(), + sdkClient: sdk, + responseCache: respCache, followSymlinks: followSymlinks, supportLFS: supportLFS, + + forbiddenMimeTypes: forbiddenMimeTypes, + defaultMimeType: defaultMimeType, }, err } func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { - resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) + reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) if err != nil { return nil, err } - return resp.Body(), nil + defer reader.Close() + return io.ReadAll(reader) } -func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { - var apiURL string - if client.supportLFS { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref)) - } else { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - } - resp, err := client.do(client.contentTimeout, apiURL) - if err != nil { - return nil, err - } +func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) { + cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource) + log := log.With().Str("cache_key", cacheKey).Logger() - if err != nil { - return nil, err - } - - switch resp.StatusCode() { - case fasthttp.StatusOK: - objType := string(resp.Header.Peek(giteaObjectTypeHeader)) - log.Trace().Msgf("server raw content object: %s", objType) - if client.followSymlinks && objType == "symlink" { - // TODO: limit to 1000 chars if we switched to std - linkDest := strings.TrimSpace(string(resp.Body())) - log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) - return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + // handle if cache entry exist + if cache, ok := client.responseCache.Get(cacheKey); ok { + cache := cache.(FileResponse) + cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) + // TODO: check against some timestamp missmatch?!? + if cache.Exists { + if cache.IsSymlink { + linkDest := string(cache.Body) + log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } else { + log.Debug().Msg("[cache] return bytes") + return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil + } + } else { + return nil, cachedHeader, cachedStatusCode, ErrorNotFound } - - return resp, nil - - case fasthttp.StatusNotFound: - return nil, ErrorNotFound - - default: - return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode()) } + + // not in cache, open reader via gitea api + reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) + if resp != nil { + switch resp.StatusCode { + case http.StatusOK: + // first handle symlinks + { + objType := resp.Header.Get(giteaObjectTypeHeader) + log.Trace().Msgf("server raw content object %q", objType) + if client.followSymlinks && objType == objTypeSymlink { + defer reader.Close() + // read limited chars for symlink + linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit)) + if err != nil { + return nil, nil, http.StatusInternalServerError, err + } + linkDest := strings.TrimSpace(string(linkDestBytes)) + + // we store symlink not content to reduce duplicates in cache + if err := client.responseCache.Set(cacheKey, FileResponse{ + Exists: true, + IsSymlink: true, + Body: []byte(linkDest), + ETag: resp.Header.Get(ETagHeader), + }, fileCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + + log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } + } + + // now we are sure it's content so set the MIME type + mimeType := client.getMimeTypeByExtension(resource) + resp.Response.Header.Set(ContentTypeHeader, mimeType) + + if !shouldRespBeSavedToCache(resp.Response) { + return reader, resp.Response.Header, resp.StatusCode, err + } + + // now we write to cache and respond at the sime time + fileResp := FileResponse{ + Exists: true, + ETag: resp.Header.Get(ETagHeader), + MimeType: mimeType, + } + return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil + + case http.StatusNotFound: + if err := client.responseCache.Set(cacheKey, FileResponse{ + Exists: false, + ETag: resp.Header.Get(ETagHeader), + }, fileCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + + return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound + default: + return nil, resp.Response.Header, resp.StatusCode, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) + } + } + return nil, nil, http.StatusInternalServerError, err } -func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName) - res, err := client.do(client.infoTimeout, url) +func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) { + cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) + + if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { + branchTimeStamp := stamp.(*BranchTimestamp) + if branchTimeStamp.notFound { + log.Trace().Msgf("[cache] use branch %q not found", branchName) + return &BranchTimestamp{}, ErrorNotFound + } + log.Trace().Msgf("[cache] use branch %q exist", branchName) + return branchTimeStamp, nil + } + + branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) if err != nil { - return time.Time{}, err + if resp != nil && resp.StatusCode == http.StatusNotFound { + log.Trace().Msgf("[cache] set cache branch %q not found", branchName) + if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return &BranchTimestamp{}, ErrorNotFound + } + return &BranchTimestamp{}, err } - if res.StatusCode() != fasthttp.StatusOK { - return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + if resp.StatusCode != http.StatusOK { + return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) } - return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) + + stamp := &BranchTimestamp{ + Branch: branch.Name, + Timestamp: branch.Commit.Timestamp, + } + + log.Trace().Msgf("set cache branch [%s] exist", branchName) + if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return stamp, nil } func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName) - res, err := client.do(client.infoTimeout, url) + cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) + + if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil { + return branch.(string), nil + } + + repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName) if err != nil { return "", err } - if res.StatusCode() != fasthttp.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode) } - return fastjson.GetString(res.Body(), "default_branch"), nil + + branch := repo.DefaultBranch + if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return branch, nil } -func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(url) - req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken) - res := fasthttp.AcquireResponse() - - err := client.fastClient.DoTimeout(req, res, timeout) - - return res, err +func (client *Client) getMimeTypeByExtension(resource string) string { + mimeType := mime.TypeByExtension(path.Ext(resource)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { + mimeType = client.defaultMimeType + } + log.Trace().Msgf("probe mime of %q is %q", resource, mimeType) + return mimeType +} + +func shouldRespBeSavedToCache(resp *http.Response) bool { + if resp == nil { + return false + } + + contentLengthRaw := resp.Header.Get(ContentLengthHeader) + if contentLengthRaw == "" { + return false + } + + contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64) + if err != nil { + log.Error().Err(err).Msg("could not parse content length") + } + + // if content to big or could not be determined we not cache it + return contentLeng > 0 && contentLeng < fileCacheSizeLimit } diff --git a/server/gitea/client_test.go b/server/gitea/client_test.go deleted file mode 100644 index 7dbad68..0000000 --- a/server/gitea/client_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package gitea - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestJoinURL(t *testing.T) { - baseURL := "" - assert.EqualValues(t, "/", joinURL(baseURL)) - assert.EqualValues(t, "/", joinURL(baseURL, "", "")) - - baseURL = "http://wwow.url.com" - assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d")) - - baseURL = "http://wow.url.com/subpath/2" - assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main"))) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main")) -} diff --git a/server/gitea/fasthttp.go b/server/gitea/fasthttp.go deleted file mode 100644 index 4ff0f4a..0000000 --- a/server/gitea/fasthttp.go +++ /dev/null @@ -1,15 +0,0 @@ -package gitea - -import ( - "time" - - "github.com/valyala/fasthttp" -) - -func getFastHTTPClient() *fasthttp.Client { - return &fasthttp.Client{ - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! - } -} diff --git a/server/handler.go b/server/handler.go index fb8b419..894cd25 100644 --- a/server/handler.go +++ b/server/handler.go @@ -1,15 +1,17 @@ package server import ( - "bytes" + "fmt" + "net/http" + "path" "strings" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/dns" "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" @@ -17,42 +19,48 @@ import ( "codeberg.org/codeberg/pages/server/version" ) +const ( + headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" + headerAccessControlAllowMethods = "Access-Control-Allow-Methods" +) + // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain []byte, +func Handler(mainDomainSuffix, rawDomain string, giteaClient *gitea.Client, giteaRoot, rawInfoPage string, - blacklistedPaths, allowedCorsDomains [][]byte, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, -) func(ctx *fasthttp.RequestCtx) { - return func(ctx *fasthttp.RequestCtx) { - log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger() + blacklistedPaths, allowedCorsDomains []string, + dnsLookupCache, canonicalDomainCache cache.SetGetKey, +) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger() + ctx := context.New(w, req) - ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version) + ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version) // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin - ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin") + ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") // Enable browser caching for up to 10 minutes - ctx.Response.Header.Set("Cache-Control", "public, max-age=600") + ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600") - trimmedHost := utils.TrimHostPort(ctx.Request.Host()) + trimmedHost := utils.TrimHostPort(req.Host) // Add HSTS for RawDomain and MainDomainSuffix - if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { - ctx.Response.Header.Set("Strict-Transport-Security", hsts) + if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) } // Block all methods not required for static pages - if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() { - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed) + if !ctx.IsMethod(http.MethodGet) && !ctx.IsMethod(http.MethodHead) && !ctx.IsMethod(http.MethodOptions) { + ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 + ctx.String("Method not allowed", http.StatusMethodNotAllowed) return } // Block blacklisted paths (like ACME challenges) for _, blacklistedPath := range blacklistedPaths { - if bytes.HasPrefix(ctx.Path(), blacklistedPath) { - html.ReturnErrorPage(ctx, fasthttp.StatusForbidden) + if strings.HasPrefix(ctx.Path(), blacklistedPath) { + html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden) return } } @@ -60,18 +68,19 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Allow CORS for specified domains allowCors := false for _, allowedCorsDomain := range allowedCorsDomains { - if bytes.Equal(trimmedHost, allowedCorsDomain) { + if strings.EqualFold(trimmedHost, allowedCorsDomain) { allowCors = true break } } if allowCors { - ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") - ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD") + ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*") + ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead) } - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - if ctx.IsOptions() { - ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent) + + ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1 + if ctx.IsMethod(http.MethodOptions) { + ctx.RespWriter.WriteHeader(http.StatusNoContent) return } @@ -83,9 +92,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will // also disallow search indexing and add a Link header to the canonical URL. - tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool { + // TODO: move into external func to not alert vars indirectly + tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool { if repo == "" { - log.Warn().Msg("tryBranch: repo is empty") + log.Debug().Msg("tryBranch: repo is empty") return false } @@ -94,23 +104,23 @@ func Handler(mainDomainSuffix, rawDomain []byte, branch = strings.ReplaceAll(branch, "~", "/") // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache) + branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch) if branchTimestampResult == nil { - log.Warn().Msg("tryBranch: branch doesn't exist") + log.Debug().Msg("tryBranch: branch doesn't exist") return false } // Branch exists, use it targetRepo = repo - targetPath = strings.Trim(strings.Join(path, "/"), "/") + targetPath = path.Join(_path...) targetBranch = branchTimestampResult.Branch targetOptions.BranchTimestamp = branchTimestampResult.Timestamp if canonicalLink != "" { // Hide from search machines & add canonical link - ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex") - ctx.Response.Header.Set("Link", + ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex") + ctx.RespWriter.Header().Set("Link", strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ "; rel=\"canonical\"", ) @@ -120,22 +130,18 @@ func Handler(mainDomainSuffix, rawDomain []byte, return true } - log.Debug().Msg("Preparing") - if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) { + log.Debug().Msg("preparations") + if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { // Serve raw content from RawDomain - log.Debug().Msg("Serving raw domain") + log.Debug().Msg("raw domain") targetOptions.TryIndexPages = false - if targetOptions.ForbiddenMimeTypes == nil { - targetOptions.ForbiddenMimeTypes = make(map[string]bool) - } - targetOptions.ForbiddenMimeTypes["text/html"] = true - targetOptions.DefaultMimeType = "text/plain; charset=utf-8" + targetOptions.ServeRaw = true - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") if len(pathElements) < 2 { // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect) + ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect) return } targetOwner = pathElements[0] @@ -143,45 +149,45 @@ func Handler(mainDomainSuffix, rawDomain []byte, // raw.codeberg.org/example/myrepo/@main/index.html if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - log.Debug().Msg("Preparing raw domain, now trying with specified branch") + log.Debug().Msg("raw domain preparations, now trying with specified branch") if tryBranch(log, targetRepo, pathElements[2][1:], pathElements[3:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { - log.Info().Msg("tryBranch, now trying upstream 1") + log.Debug().Msg("tryBranch, now trying upstream 1") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } - log.Warn().Msg("Path missed a branch") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + log.Debug().Msg("missing branch info") + html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency) return } - log.Debug().Msg("Preparing raw domain, now trying with default branch") + log.Debug().Msg("raw domain preparations, now trying with default branch") tryBranch(log, targetRepo, "", pathElements[2:], giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) - log.Info().Msg("tryBranch, now trying upstream 2") + log.Debug().Msg("tryBranch, now trying upstream 2") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return - } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { // Serve pages from subdomains of MainDomainSuffix - log.Info().Msg("Serve pages from main domain suffix") + log.Debug().Msg("main domain suffix") - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix)) + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") + targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix) targetRepo = pathElements[0] targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") if targetOwner == "www" { // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? - ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) + ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect) return } @@ -190,22 +196,24 @@ func Handler(mainDomainSuffix, rawDomain []byte, if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { if targetRepo == "pages" { // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... - ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect) + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect) return } - log.Debug().Msg("Preparing main domain, now trying with specified repo & branch") + log.Debug().Msg("main domain preparations, now trying with specified repo & branch") + branch := pathElements[1][1:] if tryBranch(log, - pathElements[0], pathElements[1][1:], pathElements[2:], + pathElements[0], branch, pathElements[2:], "/"+pathElements[0]+"/%p", ) { - log.Info().Msg("tryBranch, now trying upstream 3") + log.Debug().Msg("tryBranch, now trying upstream 3") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) } else { - log.Warn().Msg("tryBranch: upstream 3 failed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo), + http.StatusFailedDependency) } return } @@ -213,16 +221,18 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Check if the first directory is a branch for the "pages" repo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { - log.Debug().Msg("Preparing main domain, now trying with specified branch") + log.Debug().Msg("main domain preparations, now trying with specified branch") + branch := pathElements[0][1:] if tryBranch(log, - "pages", pathElements[0][1:], pathElements[1:], "/%p") { - log.Info().Msg("tryBranch, now trying upstream 4") + "pages", branch, pathElements[1:], "/%p") { + log.Debug().Msg("tryBranch, now trying upstream 4") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + targetOptions, targetOwner, "pages", targetBranch, targetPath, + canonicalDomainCache) } else { - log.Warn().Msg("tryBranch: upstream 4 failed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"), + http.StatusFailedDependency) } return } @@ -233,10 +243,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified repo") if pathElements[0] != "pages" && tryBranch(log, pathElements[0], "pages", pathElements[1:], "") { - log.Info().Msg("tryBranch, now trying upstream 5") + log.Debug().Msg("tryBranch, now trying upstream 5") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } @@ -245,28 +255,31 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with default repo/branch") if tryBranch(log, "pages", "", pathElements, "") { - log.Info().Msg("tryBranch, now trying upstream 6") + log.Debug().Msg("tryBranch, now trying upstream 6") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } // Couldn't find a valid repo/branch - - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + fmt.Sprintf("couldn't find a valid repo[%s]/branch[%s]", targetRepo, targetBranch), + http.StatusFailedDependency) return } else { trimmedHostStr := string(trimmedHost) - // Serve pages from external domains + // Serve pages from custom domains targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, + "could not obtain repo owner from custom domain", + http.StatusFailedDependency) return } - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") canonicalLink := "" if strings.HasPrefix(pathElements[0], "@") { targetBranch = pathElements[0][1:] @@ -275,36 +288,33 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Try to use the given repo on the given branch or the default branch - log.Debug().Msg("Preparing custom domain, now trying with details from DNS") + log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(log, targetRepo, targetBranch, pathElements, canonicalLink) { canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) if !valid { - log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical") - html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHostStr { // only redirect if the target is also a codeberg page! targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect) return } - log.Warn().Msg("Custom domains, targetOwner from DNS is empty") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency) return } - log.Info().Msg("tryBranch, now trying upstream 7") + log.Debug().Msg("tryBranch, now trying upstream 7") tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache, branchTimestampCache, fileResponseCache) + canonicalDomainCache) return } - log.Warn().Msg("Couldn't handle request, none of the options succeed") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency) return } } diff --git a/server/handler_test.go b/server/handler_test.go index f9a721a..c0aca14 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1,44 +1,42 @@ package server import ( - "fmt" + "net/http/httptest" "testing" "time" - "github.com/valyala/fasthttp" - "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" + "github.com/rs/zerolog/log" ) func TestHandlerPerformance(t *testing.T) { giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false) + giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false) testHandler := Handler( - []byte("codeberg.page"), []byte("raw.codeberg.org"), + "codeberg.page", "raw.codeberg.org", giteaClient, giteaRoot, "https://docs.codeberg.org/pages/raw-content/", - [][]byte{[]byte("/.well-known/acme-challenge/")}, - [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), + []string{"/.well-known/acme-challenge/"}, + []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), ) testCase := func(uri string, status int) { - ctx := &fasthttp.RequestCtx{ - Request: *fasthttp.AcquireRequest(), - Response: *fasthttp.AcquireResponse(), - } - ctx.Request.SetRequestURI(uri) - fmt.Printf("Start: %v\n", time.Now()) + req := httptest.NewRequest("GET", uri, nil) + w := httptest.NewRecorder() + + log.Printf("Start: %v\n", time.Now()) start := time.Now() - testHandler(ctx) + testHandler(w, req) end := time.Now() - fmt.Printf("Done: %v\n", time.Now()) - if ctx.Response.StatusCode() != status { - t.Errorf("request failed with status code %d", ctx.Response.StatusCode()) + log.Printf("Done: %v\n", time.Now()) + + resp := w.Result() + + if resp.StatusCode != status { + t.Errorf("request failed with status code %d", resp.StatusCode) } else { t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds()) } diff --git a/server/helpers.go b/server/helpers.go index 6d55ddf..7c898cd 100644 --- a/server/helpers.go +++ b/server/helpers.go @@ -1,13 +1,13 @@ package server import ( - "bytes" + "strings" ) -// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty +// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty // string for custom domains. -func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string { - if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) { +func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string { + if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) { return "max-age=63072000; includeSubdomains; preload" } else { return "" diff --git a/server/setup.go b/server/setup.go index 176bb42..e7194ed 100644 --- a/server/setup.go +++ b/server/setup.go @@ -1,53 +1,27 @@ package server import ( - "bytes" - "fmt" "net/http" - "time" - - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" + "strings" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/utils" ) -type fasthttpLogger struct{} +func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc { + challengePath := "/.well-known/acme-challenge/" -func (fasthttpLogger) Printf(format string, args ...interface{}) { - log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...)) -} - -func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server { - // Enable compression by wrapping the handler with the compression function provided by FastHTTP - compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - - return &fasthttp.Server{ - Handler: compressedHandler, - DisablePreParseMultipartForm: true, - NoDefaultServerHeader: true, - NoDefaultDate: true, - ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge - Logger: fasthttpLogger{}, - } -} - -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { - challengePath := []byte("/.well-known/acme-challenge/") - - return &fasthttp.Server{ - Handler: func(ctx *fasthttp.RequestCtx) { - if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) - if !ok || challenge == nil { - ctx.SetStatusCode(http.StatusNotFound) - ctx.SetBodyString("no challenge for this token") - } - ctx.SetBodyString(challenge.(string)) - } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) + return func(w http.ResponseWriter, req *http.Request) { + ctx := context.New(w, req) + if strings.HasPrefix(ctx.Path(), challengePath) { + challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + string(strings.TrimPrefix(ctx.Path(), challengePath))) + if !ok || challenge == nil { + ctx.String("no challenge for this token", http.StatusNotFound) } - }, + ctx.String(challenge.(string)) + } else { + ctx.Redirect("https://"+string(ctx.Host())+string(ctx.Path()), http.StatusMovedPermanently) + } } } diff --git a/server/try.go b/server/try.go index 24831c4..135c1e0 100644 --- a/server/try.go +++ b/server/try.go @@ -1,38 +1,37 @@ package server import ( - "bytes" + "net/http" "strings" - "github.com/valyala/fasthttp" - "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" "codeberg.org/codeberg/pages/server/upstream" ) // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, - mainDomainSuffix, trimmedHost []byte, +func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, + mainDomainSuffix, trimmedHost string, targetOptions *upstream.Options, targetOwner, targetRepo, targetBranch, targetPath string, - canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey, + canonicalDomainCache cache.SetGetKey, ) { // check if a canonical domain exists on a request on MainDomain - if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + if strings.HasSuffix(trimmedHost, mainDomainSuffix) { canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { - canonicalPath := string(ctx.RequestURI()) + canonicalPath := ctx.Req.RequestURI if targetRepo != "pages" { path := strings.SplitN(canonicalPath, "/", 3) if len(path) >= 3 { canonicalPath = "/" + path[2] } } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) + ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect) return } } @@ -44,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, targetOptions.Host = string(trimmedHost) // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) + if !targetOptions.Upstream(ctx, giteaClient) { + html.ReturnErrorPage(ctx, "", ctx.StatusCode) } } diff --git a/server/upstream/const.go b/server/upstream/const.go deleted file mode 100644 index 247e1d1..0000000 --- a/server/upstream/const.go +++ /dev/null @@ -1,24 +0,0 @@ -package upstream - -import "time" - -// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. -var defaultBranchCacheTimeout = 15 * time.Minute - -// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter -// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be -// picked up faster, while still allowing the content to be cached longer if nothing changes. -var branchExistenceCacheTimeout = 5 * time.Minute - -// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending -// on your available memory. -// TODO: move as option into cache interface -var fileCacheTimeout = 5 * time.Minute - -// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. -var fileCacheSizeLimit = 1024 * 1024 - -// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. -var canonicalDomainCacheTimeout = 15 * time.Minute - -const canonicalDomainConfig = ".domains" diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 553c148..6ad6506 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -2,11 +2,19 @@ package upstream import ( "strings" + "time" + + "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" ) +// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. +var canonicalDomainCacheTimeout = 15 * time.Minute + +const canonicalDomainConfig = ".domains" + // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( @@ -36,6 +44,8 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta valid = true } } + } else { + log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo) } domains = append(domains, targetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 28f4474..6bc23c8 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -1,84 +1,36 @@ package upstream import ( - "mime" - "path" - "strconv" - "strings" - "time" + "errors" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/gitea" "github.com/rs/zerolog/log" -) -type branchTimestamp struct { - Branch string - Timestamp time.Time -} + "codeberg.org/codeberg/pages/server/gitea" +) // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp { +func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp { log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger() - if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { - if result == nil { - log.Debug().Msg("branchTimestampCache found item, but result is empty") - return nil - } - log.Debug().Msg("branchTimestampCache found item, returning result") - return result.(*branchTimestamp) - } - result := &branchTimestamp{ - Branch: branch, - } + if len(branch) == 0 { // Get default branch defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo) if err != nil { log.Err(err).Msg("Could't fetch default branch from repository") - _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil } - log.Debug().Msg("Succesfully fetched default branch from Gitea") - result.Branch = defaultBranch + log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch) + branch = defaultBranch } - timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch) + timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch) if err != nil { - log.Err(err).Msg("Could not get latest commit's timestamp from branch") + if !errors.Is(err, gitea.ErrorNotFound) { + log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch") + } return nil } - log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache") - result.Timestamp = timestamp - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout) - return result -} - -func (o *Options) getMimeTypeByExtension() string { - if o.ForbiddenMimeTypes == nil { - o.ForbiddenMimeTypes = make(map[string]bool) - } - mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" { - if o.DefaultMimeType != "" { - mimeType = o.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } - return mimeType -} - -func (o *Options) generateUri() string { - return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) -} - -func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) { - return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath -} - -func (o *Options) timestamp() string { - return strconv.FormatInt(o.BranchTimestamp.Unix(), 10) + log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp) + return timestamp } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 61c90de..d37c35e 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -1,20 +1,27 @@ package upstream import ( - "bytes" "errors" + "fmt" "io" + "net/http" "strings" "time" "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/context" "codeberg.org/codeberg/pages/server/gitea" ) +const ( + headerLastModified = "Last-Modified" + headerIfModifiedSince = "If-Modified-Since" + + rawMime = "text/plain; charset=utf-8" +) + // upstreamIndexPages lists pages that may be considered as index pages for directories. var upstreamIndexPages = []string{ "index.html", @@ -35,61 +42,61 @@ type Options struct { // Used for debugging purposes. Host string - DefaultMimeType string - ForbiddenMimeTypes map[string]bool - TryIndexPages bool - BranchTimestamp time.Time + TryIndexPages bool + BranchTimestamp time.Time // internal appendTrailingSlash bool redirectIfExists string + + ServeRaw bool } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { - log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger() +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) { + log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() + + if o.TargetOwner == "" || o.TargetRepo == "" { + html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest) + return true + } // Check if the branch exists and when it was modified if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache) + branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) - if branch == nil { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + if branch == nil || branch.Branch == "" { + html.ReturnErrorPage(ctx, + fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch), + http.StatusFailedDependency) return true } o.TargetBranch = branch.Branch o.BranchTimestamp = branch.Timestamp } - if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest) - return true - } - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(o.BranchTimestamp) { - ctx.Response.SetStatusCode(fasthttp.StatusNotModified) - return true + if ctx.Response() != nil { + if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil { + if !ifModifiedSince.Before(o.BranchTimestamp) { + ctx.RespWriter.WriteHeader(http.StatusNotModified) + log.Trace().Msg("check response against last modified: valid") + return true + } } + log.Trace().Msg("check response against last modified: outdated") } log.Debug().Msg("Preparing") - // Make a GET request to the upstream URL - uri := o.generateUri() - var res *fasthttp.Response - var cachedResponse gitea.FileResponse - var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { - cachedResponse = cachedValue.(gitea.FileResponse) - } else { - res, err = giteaClient.ServeRawContent(o.generateUriClientArgs()) + reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath) + if reader != nil { + defer reader.Close() } log.Debug().Msg("Aquisting") - // Handle errors - if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) { + // Handle not found error + if err != nil && errors.Is(err, gitea.ErrorNotFound) { if o.TryIndexPages { // copy the o struct & try if an index page exists optionsForIndexPages := *o @@ -97,25 +104,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForIndexPages.Upstream(ctx, giteaClient) { return true } } // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.appendTrailingSlash = false - optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" + optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForIndexPages.Upstream(ctx, giteaClient) { return true } } - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) + + ctx.StatusCode = http.StatusNotFound if o.TryIndexPages { // copy the o struct & try if a not found page exists optionsForNotFoundPages := *o @@ -123,94 +125,84 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, optionsForNotFoundPages.appendTrailingSlash = false for _, notFoundPage := range upstreamNotFoundPages { optionsForNotFoundPages.TargetPath = "/" + notFoundPage - if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) + if optionsForNotFoundPages.Upstream(ctx, giteaClient) { return true } } } - if res != nil { - // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - Exists: false, - }, fileCacheTimeout) - } return false } - if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode()) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) + + // handle unexpected client errors + if err != nil || reader == nil || statusCode != http.StatusOK { + log.Debug().Msg("Handling error") + var msg string + + if err != nil { + msg = "gitea client returned unexpected error" + log.Error().Err(err).Msg(msg) + msg = fmt.Sprintf("%s: %v", msg, err) + } + if reader == nil { + msg = "gitea client returned no reader" + log.Error().Msg(msg) + } + if statusCode != http.StatusOK { + msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode) + log.Error().Msg(msg) + } + + html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError) return true } // Append trailing slash if missing (for index files), and redirect to fix filenames in general // o.appendTrailingSlash is only true when looking for index pages - if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { - ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) + if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { + ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) return true } - if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { - ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) + if strings.HasSuffix(ctx.Path(), "/index.html") { + ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) return true } if o.redirectIfExists != "" { - ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect) + ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect) return true } - log.Debug().Msg("Handling error") - - // Set the MIME type - mimeType := o.getMimeTypeByExtension() - ctx.Response.Header.SetContentType(mimeType) - - // Set ETag - if cachedResponse.Exists { - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) - } else if res != nil { - cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag) - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) + // Set ETag & MIME + if eTag := header.Get(gitea.ETagHeader); eTag != "" { + ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag) } - - if ctx.Response.StatusCode() != fasthttp.StatusNotFound { - // Everything's okay so far - ctx.Response.SetStatusCode(fasthttp.StatusOK) + if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" { + ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator) } - ctx.Response.Header.SetLastModified(o.BranchTimestamp) + if length := header.Get(gitea.ContentLengthHeader); length != "" { + ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length) + } + if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime) + } else { + ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime) + } + ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) log.Debug().Msg("Prepare response") - // Write the response body to the original request - var cacheBodyWriter bytes.Buffer - if res != nil { - if res.Header.ContentLength() > fileCacheSizeLimit { - // fasthttp else will set "Content-Length: 0" - ctx.Response.SetBodyStream(&strings.Reader{}, -1) + ctx.RespWriter.WriteHeader(ctx.StatusCode) - err = res.BodyWriteTo(ctx.Response.BodyWriter()) - } else { - // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? - err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) + // Write the response body to the original request + if reader != nil { + _, err := io.Copy(ctx.RespWriter, reader) + if err != nil { + log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath) + html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) + return true } - } else { - _, err = ctx.Write(cachedResponse.Body) - } - if err != nil { - log.Error().Err(err).Msgf("Couldn't write body for %q", uri) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true } log.Debug().Msg("Sending response") - if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil { - cachedResponse.Exists = true - cachedResponse.MimeType = mimeType - cachedResponse.Body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) - } - return true } diff --git a/server/utils/utils.go b/server/utils/utils.go index 7be330f..30f948d 100644 --- a/server/utils/utils.go +++ b/server/utils/utils.go @@ -1,9 +1,11 @@ package utils -import "bytes" +import ( + "strings" +) -func TrimHostPort(host []byte) []byte { - i := bytes.IndexByte(host, ':') +func TrimHostPort(host string) string { + i := strings.IndexByte(host, ':') if i >= 0 { return host[:i] } diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go index 3dc0632..2532392 100644 --- a/server/utils/utils_test.go +++ b/server/utils/utils_test.go @@ -7,7 +7,7 @@ import ( ) func TestTrimHostPort(t *testing.T) { - assert.EqualValues(t, "aa", TrimHostPort([]byte("aa"))) - assert.EqualValues(t, "", TrimHostPort([]byte(":"))) - assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80"))) + assert.EqualValues(t, "aa", TrimHostPort("aa")) + assert.EqualValues(t, "", TrimHostPort(":")) + assert.EqualValues(t, "example.com", TrimHostPort("example.com:80")) }