diff --git a/backend_configuration.go b/backend_configuration.go index 7f21f01..85d4537 100644 --- a/backend_configuration.go +++ b/backend_configuration.go @@ -90,6 +90,7 @@ func (b *Backend) AddSession(session Session) error { if b.sessions == nil { b.sessions = make(map[string]bool) } else if uint64(len(b.sessions)) >= b.sessionLimit { + statsBackendLimitExceededTotal.WithLabelValues(b.id).Inc() return SessionLimitExceeded } @@ -123,6 +124,7 @@ func NewBackendConfiguration(config *goconf.ConfigFile) (*BackendConfiguration, } backends := make(map[string][]*Backend) var compatBackend *Backend + numBackends := 0 if allowAll { log.Println("WARNING: All backend hostnames are allowed, only use for development!") compatBackend = &Backend{ @@ -137,12 +139,14 @@ func NewBackendConfiguration(config *goconf.ConfigFile) (*BackendConfiguration, if sessionLimit > 0 { log.Printf("Allow a maximum of %d sessions", sessionLimit) } + numBackends += 1 } else if backendIds, _ := config.GetString("backend", "backends"); backendIds != "" { for host, configuredBackends := range getConfiguredHosts(backendIds, config) { backends[host] = append(backends[host], configuredBackends...) for _, be := range configuredBackends { log.Printf("Backend %s added for %s", be.id, be.url) } + numBackends += len(configuredBackends) } } else if allowedUrls, _ := config.GetString("backend", "allowed"); allowedUrls != "" { // Old-style configuration, only hosts are configured and are using a common secret. @@ -182,9 +186,14 @@ func NewBackendConfiguration(config *goconf.ConfigFile) (*BackendConfiguration, if sessionLimit > 0 { log.Printf("Allow a maximum of %d sessions", sessionLimit) } + numBackends += 1 } } + RegisterBackendConfigurationStats() + log.Printf("Initial: %d", numBackends) + statsBackendsCurrent.Add(float64(numBackends)) + return &BackendConfiguration{ backends: backends, @@ -199,6 +208,7 @@ func (b *BackendConfiguration) RemoveBackendsForHost(host string) { for _, backend := range oldBackends { log.Printf("Backend %s removed for %s", backend.id, backend.url) } + statsBackendsCurrent.Sub(float64(len(oldBackends))) } delete(b.backends, host) } @@ -225,6 +235,7 @@ func (b *BackendConfiguration) UpsertHost(host string, backends []*Backend) { removed := b.backends[host][existingIndex] log.Printf("Backend %s removed for %s", removed.id, removed.url) b.backends[host] = append(b.backends[host][:existingIndex], b.backends[host][existingIndex+1:]...) + statsBackendsCurrent.Dec() } } @@ -232,6 +243,7 @@ func (b *BackendConfiguration) UpsertHost(host string, backends []*Backend) { for _, added := range backends { log.Printf("Backend %s added for %s", added.id, added.url) } + statsBackendsCurrent.Add(float64(len(backends))) } func getConfiguredBackendIDs(backendIds string) (ids []string) { diff --git a/backend_configuration_stats_prometheus.go b/backend_configuration_stats_prometheus.go new file mode 100644 index 0000000..13b7c7d --- /dev/null +++ b/backend_configuration_stats_prometheus.go @@ -0,0 +1,50 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsBackendLimitExceededTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "backend", + Name: "session_limit_exceeded_total", + Help: "The number of times the session limit exceeded", + }, []string{"backend"}) + statsBackendsCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "backend", + Name: "current", + Help: "The current number of configured backends", + }) + + backendConfigurationStats = []prometheus.Collector{ + statsBackendLimitExceededTotal, + statsBackendsCurrent, + } +) + +func RegisterBackendConfigurationStats() { + registerAll(backendConfigurationStats...) +} diff --git a/backend_configuration_test.go b/backend_configuration_test.go index 94ad7a5..17e7508 100644 --- a/backend_configuration_test.go +++ b/backend_configuration_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/dlintw/goconf" + "github.com/prometheus/client_golang/prometheus/testutil" ) func testUrls(t *testing.T, config *BackendConfiguration, valid_urls []string, invalid_urls []string) { @@ -238,6 +239,7 @@ func TestParseBackendIds(t *testing.T) { } func TestBackendReloadNoChange(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend", "allowall", "false") @@ -249,6 +251,7 @@ func TestBackendReloadNoChange(t *testing.T) { if err != nil { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+2) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1, backend2") @@ -262,13 +265,16 @@ func TestBackendReloadNoChange(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+4) o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+4) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } } func TestBackendReloadChangeExistingURL(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend", "allowall", "false") @@ -281,6 +287,7 @@ func TestBackendReloadChangeExistingURL(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+2) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1, backend2") new_config.AddOption("backend", "allowall", "false") @@ -294,17 +301,20 @@ func TestBackendReloadChangeExistingURL(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+4) original_config.RemoveOption("backend1", "url") original_config.AddOption("backend1", "url", "http://domain3.invalid") original_config.AddOption("backend1", "sessionlimit", "10") o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+4) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } } func TestBackendReloadChangeSecret(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend", "allowall", "false") @@ -317,6 +327,7 @@ func TestBackendReloadChangeSecret(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+2) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1, backend2") new_config.AddOption("backend", "allowall", "false") @@ -329,16 +340,19 @@ func TestBackendReloadChangeSecret(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+4) original_config.RemoveOption("backend1", "secret") original_config.AddOption("backend1", "secret", string(testBackendSecret)+"-backend3") o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+4) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } } func TestBackendReloadAddBackend(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1") original_config.AddOption("backend", "allowall", "false") @@ -349,6 +363,7 @@ func TestBackendReloadAddBackend(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+1) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1, backend2") new_config.AddOption("backend", "allowall", "false") @@ -362,6 +377,7 @@ func TestBackendReloadAddBackend(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+3) original_config.RemoveOption("backend", "backends") original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend2", "url", "http://domain2.invalid") @@ -369,12 +385,14 @@ func TestBackendReloadAddBackend(t *testing.T) { original_config.AddOption("backend2", "sessionlimit", "10") o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+4) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } } func TestBackendReloadRemoveHost(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend", "allowall", "false") @@ -387,6 +405,7 @@ func TestBackendReloadRemoveHost(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+2) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1") new_config.AddOption("backend", "allowall", "false") @@ -397,17 +416,20 @@ func TestBackendReloadRemoveHost(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+3) original_config.RemoveOption("backend", "backends") original_config.AddOption("backend", "backends", "backend1") original_config.RemoveSection("backend2") o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+2) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } } func TestBackendReloadRemoveBackendFromSharedHost(t *testing.T) { + current := testutil.ToFloat64(statsBackendsCurrent) original_config := goconf.NewConfigFile() original_config.AddOption("backend", "backends", "backend1, backend2") original_config.AddOption("backend", "allowall", "false") @@ -420,6 +442,7 @@ func TestBackendReloadRemoveBackendFromSharedHost(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+2) new_config := goconf.NewConfigFile() new_config.AddOption("backend", "backends", "backend1") new_config.AddOption("backend", "allowall", "false") @@ -430,11 +453,13 @@ func TestBackendReloadRemoveBackendFromSharedHost(t *testing.T) { t.Fatal(err) } + checkStatsValue(t, statsBackendsCurrent, current+3) original_config.RemoveOption("backend", "backends") original_config.AddOption("backend", "backends", "backend1") original_config.RemoveSection("backend2") o_cfg.Reload(original_config) + checkStatsValue(t, statsBackendsCurrent, current+2) if !reflect.DeepEqual(n_cfg, o_cfg) { t.Error("BackendConfiguration should be equal after Reload") } diff --git a/backend_server.go b/backend_server.go index 85804f3..65e99f4 100644 --- a/backend_server.go +++ b/backend_server.go @@ -41,6 +41,7 @@ import ( "github.com/dlintw/goconf" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( @@ -153,6 +154,9 @@ func (b *BackendServer) Start(r *mux.Router) error { s.HandleFunc("/room/{roomid}", b.setComonHeaders(b.parseRequestBody(b.roomHandler))).Methods("POST") s.HandleFunc("/stats", b.setComonHeaders(b.validateStatsRequest(b.statsHandler))).Methods("GET") + // Expose prometheus metrics at "/metrics". + r.HandleFunc("/metrics", b.setComonHeaders(b.validateStatsRequest(b.metricsHandler))).Methods("GET") + // Provide a REST service to get TURN credentials. // See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00 r.HandleFunc("/turn/credentials", b.setComonHeaders(b.getTurnCredentials)).Methods("GET") @@ -632,3 +636,7 @@ func (b *BackendServer) statsHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(statsData) // nolint } + +func (b *BackendServer) metricsHandler(w http.ResponseWriter, r *http.Request) { + promhttp.Handler().ServeHTTP(w, r) +} diff --git a/client.go b/client.go index 943b465..7ae7156 100644 --- a/client.go +++ b/client.go @@ -58,6 +58,10 @@ var ( unknownCountry string = "unknown-country" ) +func init() { + RegisterClientStats() +} + func IsValidCountry(country string) bool { switch country { case "": diff --git a/client_stats_prometheus.go b/client_stats_prometheus.go new file mode 100644 index 0000000..e20447e --- /dev/null +++ b/client_stats_prometheus.go @@ -0,0 +1,43 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsClientCountries = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "client", + Name: "countries_total", + Help: "The total number of connections by country", + }, []string{"country"}) + + clientStats = []prometheus.Collector{ + statsClientCountries, + } +) + +func RegisterClientStats() { + registerAll(clientStats...) +} diff --git a/go.mod b/go.mod index 17a46e5..fa07872 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/nats-io/nats.go v1.11.0 github.com/notedit/janus-go v0.0.0-20200517101215-10eb8b95d1a0 github.com/oschwald/maxminddb-golang v1.8.0 + github.com/prometheus/client_golang v1.10.0 go.etcd.io/etcd v0.5.0-alpha.5.0.20210226220824-aa7126864d82 google.golang.org/protobuf v1.26.0 // indirect gopkg.in/dgrijalva/jwt-go.v3 v3.2.0 diff --git a/go.sum b/go.sum index 7412e67..cefe3d2 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,49 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -26,20 +54,37 @@ github.com/dlintw/goconf v0.0.0-20120228082610-dcc070983490 h1:I8/Qu5NTaiXi1TsEY github.com/dlintw/goconf v0.0.0-20120228082610-dcc070983490/go.mod h1:jWlUIP63OLr0cV2FGN2IEzSFsMAe58if8rk/SAE0JRE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -50,24 +95,34 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= @@ -76,36 +131,81 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -113,14 +213,21 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/jwt v1.2.2 h1:w3GMTO969dFg+UOKTmmyuu7IGdusK+7Ytlt//OYH/uU= github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= github.com/nats-io/jwt/v2 v2.0.2 h1:ejVCLO8gu6/4bOKIHQpmB5UhhUJfAQw55yvLWpfmKjI= github.com/nats-io/jwt/v2 v2.0.2/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats-server/v2 v2.2.6 h1:FPK9wWx9pagxcw14s8W9rlfzfyHm61uNLnJyybZbn48= github.com/nats-io/nats-server/v2 v2.2.6/go.mod h1:sEnFaxqe09cDmfMgACxZbziXnhQFhwk+aKkZjBBRYrI= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nats.go v1.11.0 h1:L263PZkrmkRJRJT2YHU8GwWWvEvmr9/LUKuJTXsF32k= github.com/nats-io/nats.go v1.11.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= @@ -128,113 +235,248 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/notedit/janus-go v0.0.0-20200517101215-10eb8b95d1a0 h1:EFU9iv8BMPyBo8iFMHvQleYlF5M3PY6zpAbxsngImjE= github.com/notedit/janus-go v0.0.0-20200517101215-10eb8b95d1a0/go.mod h1:BN/Txse3qz8tZOmCm2OfajB2wHVujWmX3o9nVdsI6gE= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk= github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20210226220824-aa7126864d82 h1:RCaUKN0yRYKT2JzV9kH4u+D6l9VWcJMQ449QKRriFc8= go.etcd.io/etcd v0.5.0-alpha.5.0.20210226220824-aa7126864d82/go.mod h1:WWRiAtnzDdtuCMxtFwME/Knea11a6fJgJkwtC1QSc/k= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b h1:wSOdpTq0/eI46Ez/LkDwIsAKA71YP2SRKBODiRWM0as= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 h1:DnSr2mCsxyCE6ZgIkmcWUQY2R5cH/6wL7eIxEmQOMSE= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -250,17 +492,32 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/dgrijalva/jwt-go.v3 v3.2.0 h1:N46iQqOtHry7Hxzb9PGrP68oovQmj7EhudNoKHvbOvI= gopkg.in/dgrijalva/jwt-go.v3 v3.2.0/go.mod h1:hdNXC2Z9yC029rvsQ/on2ZNQ44Z2XToVhpXXbR+J05A= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/hub.go b/hub.go index 1eae587..47f154c 100644 --- a/hub.go +++ b/hub.go @@ -95,6 +95,10 @@ const ( publicSessionName = "public-session" ) +func init() { + RegisterHubStats() +} + type Hub struct { // 64-bit members that are accessed atomically must be 64-bit aligned. sid uint64 @@ -607,6 +611,7 @@ func (h *Hub) removeSession(session Session) (removed bool) { delete(h.clients, data.Sid) if _, found := h.sessions[data.Sid]; found { delete(h.sessions, data.Sid) + statsHubSessionsCurrent.WithLabelValues(session.Backend().Id(), session.ClientType()).Dec() removed = true } } @@ -742,6 +747,12 @@ func (h *Hub) processRegister(client *Client, message *ClientMessage, backend *B } h.mu.Unlock() + if country := client.Country(); IsValidCountry(country) { + statsClientCountries.WithLabelValues(country).Inc() + } + statsHubSessionsCurrent.WithLabelValues(backend.Id(), session.ClientType()).Inc() + statsHubSessionsTotal.WithLabelValues(backend.Id(), session.ClientType()).Inc() + h.setDecodedSessionId(privateSessionId, privateSessionName, sessionIdData) h.setDecodedSessionId(publicSessionId, publicSessionName, sessionIdData) h.sendHelloResponse(session, message) @@ -840,6 +851,7 @@ func (h *Hub) processHello(client *Client, message *ClientMessage) { if resumeId != "" { data := h.decodeSessionId(resumeId, privateSessionName) if data == nil { + statsHubSessionResumeFailed.Inc() client.SendMessage(message.NewErrorServerMessage(NoSuchSession)) return } @@ -848,6 +860,7 @@ func (h *Hub) processHello(client *Client, message *ClientMessage) { session, found := h.sessions[data.Sid] if !found || resumeId != session.PrivateId() { h.mu.Unlock() + statsHubSessionResumeFailed.Inc() client.SendMessage(message.NewErrorServerMessage(NoSuchSession)) return } @@ -857,6 +870,7 @@ func (h *Hub) processHello(client *Client, message *ClientMessage) { // Should never happen as clients only can resume their own sessions. h.mu.Unlock() log.Printf("Client resumed non-client session %s (private=%s)", session.PublicId(), session.PrivateId()) + statsHubSessionResumeFailed.Inc() client.SendMessage(message.NewErrorServerMessage(NoSuchSession)) return } @@ -879,6 +893,7 @@ func (h *Hub) processHello(client *Client, message *ClientMessage) { log.Printf("Resume session from %s in %s (%s) %s (private=%s)", client.RemoteAddr(), client.Country(), client.UserAgent(), session.PublicId(), session.PrivateId()) + statsHubSessionsResumedTotal.WithLabelValues(clientSession.Backend().Id(), clientSession.ClientType()).Inc() h.sendHelloResponse(clientSession, message) clientSession.NotifySessionResumed(client) return @@ -1087,7 +1102,10 @@ func (h *Hub) getRoomForBackend(id string, backend *Backend) *Room { func (h *Hub) removeRoom(room *Room) { internalRoomId := getRoomIdForBackend(room.Id(), room.Backend()) h.ru.Lock() - delete(h.rooms, internalRoomId) + if _, found := h.rooms[internalRoomId]; found { + delete(h.rooms, internalRoomId) + statsHubRoomsCurrent.WithLabelValues(room.Backend().Id()).Dec() + } h.ru.Unlock() } @@ -1100,6 +1118,7 @@ func (h *Hub) createRoom(id string, properties *json.RawMessage, backend *Backen internalRoomId := getRoomIdForBackend(id, backend) h.rooms[internalRoomId] = room + statsHubRoomsCurrent.WithLabelValues(backend.Id()).Inc() return room, nil } @@ -1550,6 +1569,8 @@ func (h *Hub) processInternalMsg(client *Client, message *ClientMessage) { h.sessions[sessionIdData.Sid] = sess h.virtualSessions[virtualSessionId] = sessionIdData.Sid h.mu.Unlock() + statsHubSessionsCurrent.WithLabelValues(session.Backend().Id(), sess.ClientType()).Inc() + statsHubSessionsTotal.WithLabelValues(session.Backend().Id(), sess.ClientType()).Inc() log.Printf("Session %s added virtual session %s with initial flags %d", session.PublicId(), sess.PublicId(), sess.Flags()) session.AddVirtualSession(sess) sess.SetRoom(room) diff --git a/hub_stats_prometheus.go b/hub_stats_prometheus.go new file mode 100644 index 0000000..f3d5c1a --- /dev/null +++ b/hub_stats_prometheus.go @@ -0,0 +1,70 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsHubRoomsCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "hub", + Name: "rooms", + Help: "The current number of rooms per backend", + }, []string{"backend"}) + statsHubSessionsCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "hub", + Name: "sessions", + Help: "The current number of sessions per backend", + }, []string{"backend", "clienttype"}) + statsHubSessionsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "hub", + Name: "sessions_total", + Help: "The total number of sessions per backend", + }, []string{"backend", "clienttype"}) + statsHubSessionsResumedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "hub", + Name: "sessions_resume_total", + Help: "The total number of resumed sessions per backend", + }, []string{"backend", "clienttype"}) + statsHubSessionResumeFailed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "hub", + Name: "sessions_resume_failed_total", + Help: "The total number of failed session resume requests", + }) + + hubStats = []prometheus.Collector{ + statsHubRoomsCurrent, + statsHubSessionsCurrent, + statsHubSessionsTotal, + statsHubSessionResumeFailed, + } +) + +func RegisterHubStats() { + registerAll(hubStats...) +} diff --git a/mcu_common_test.go b/mcu_common_test.go new file mode 100644 index 0000000..0609ef3 --- /dev/null +++ b/mcu_common_test.go @@ -0,0 +1,30 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "testing" +) + +func TestCommonMcuStats(t *testing.T) { + collectAndLint(t, commonMcuStats...) +} diff --git a/mcu_janus.go b/mcu_janus.go index 1f2c1a9..81ae35f 100644 --- a/mcu_janus.go +++ b/mcu_janus.go @@ -449,6 +449,7 @@ type mcuJanusClient struct { handleDetached func(event *janus.DetachedMsg) handleConnected func(event *janus.WebRTCUpMsg) handleSlowLink func(event *janus.SlowLinkMsg) + handleMedia func(event *janus.MediaMsg) } func (c *mcuJanusClient) Id() string { @@ -465,7 +466,7 @@ func (c *mcuJanusClient) Close(ctx context.Context) { func (c *mcuJanusClient) SendMessage(ctx context.Context, message *MessageClientMessage, data *MessageClientMessageData, callback func(error, map[string]interface{})) { } -func (c *mcuJanusClient) closeClient(ctx context.Context) { +func (c *mcuJanusClient) closeClient(ctx context.Context) bool { if handle := c.handle; handle != nil { c.handle = nil c.closeChan <- true @@ -474,7 +475,10 @@ func (c *mcuJanusClient) closeClient(ctx context.Context) { log.Println("Could not detach client", handle.Id, err) } } + return true } + + return false } func (c *mcuJanusClient) run(handle *JanusHandle, closeChan chan bool) { @@ -490,7 +494,7 @@ loop: case *janus.DetachedMsg: c.handleDetached(t) case *janus.MediaMsg: - // Ignore + c.handleMedia(t) case *janus.WebRTCUpMsg: c.handleConnected(t) case *janus.SlowLinkMsg: @@ -602,11 +606,103 @@ func (c *mcuJanusClient) selectStream(ctx context.Context, substream int, tempor callback(nil, nil) } +type publisherStatsCounter struct { + mu sync.Mutex + + streamTypes map[string]bool + subscribers map[string]bool +} + +func (c *publisherStatsCounter) Reset() { + c.mu.Lock() + defer c.mu.Unlock() + + count := len(c.subscribers) + for streamType := range c.streamTypes { + statsMcuPublisherStreamTypesCurrent.WithLabelValues(streamType).Dec() + statsMcuSubscriberStreamTypesCurrent.WithLabelValues(streamType).Sub(float64(count)) + } + c.streamTypes = nil + c.subscribers = nil +} + +func (c *publisherStatsCounter) EnableStream(streamType string, enable bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if enable == c.streamTypes[streamType] { + return + } + + if enable { + if c.streamTypes == nil { + c.streamTypes = make(map[string]bool) + } + c.streamTypes[streamType] = true + statsMcuPublisherStreamTypesCurrent.WithLabelValues(streamType).Inc() + statsMcuSubscriberStreamTypesCurrent.WithLabelValues(streamType).Add(float64(len(c.subscribers))) + } else { + delete(c.streamTypes, streamType) + statsMcuPublisherStreamTypesCurrent.WithLabelValues(streamType).Dec() + statsMcuSubscriberStreamTypesCurrent.WithLabelValues(streamType).Sub(float64(len(c.subscribers))) + } +} + +func (c *publisherStatsCounter) AddSubscriber(id string) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.subscribers[id] { + return + } + + if c.subscribers == nil { + c.subscribers = make(map[string]bool) + } + c.subscribers[id] = true + for streamType := range c.streamTypes { + statsMcuSubscriberStreamTypesCurrent.WithLabelValues(streamType).Inc() + } +} + +func (c *publisherStatsCounter) RemoveSubscriber(id string) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.subscribers[id] { + return + } + + delete(c.subscribers, id) + for streamType := range c.streamTypes { + statsMcuSubscriberStreamTypesCurrent.WithLabelValues(streamType).Dec() + } +} + type mcuJanusPublisher struct { mcuJanusClient id string bitrate int + stats publisherStatsCounter +} + +func (m *mcuJanus) SubscriberConnected(id string, publisher string, streamType string) { + m.mu.Lock() + defer m.mu.Unlock() + + if p, found := m.publishers[publisher+"|"+streamType]; found { + p.stats.AddSubscriber(id) + } +} + +func (m *mcuJanus) SubscriberDisconnected(id string, publisher string, streamType string) { + m.mu.Lock() + defer m.mu.Unlock() + + if p, found := m.publishers[publisher+"|"+streamType]; found { + p.stats.RemoveSubscriber(id) + } } func min(a, b int) int { @@ -718,6 +814,7 @@ func (m *mcuJanus) NewPublisher(ctx context.Context, listener McuListener, id st client.mcuJanusClient.handleDetached = client.handleDetached client.mcuJanusClient.handleConnected = client.handleConnected client.mcuJanusClient.handleSlowLink = client.handleSlowLink + client.mcuJanusClient.handleMedia = client.handleMedia m.registerClient(client) log.Printf("Publisher %s is using handle %d", client.id, client.handleId) @@ -726,6 +823,8 @@ func (m *mcuJanus) NewPublisher(ctx context.Context, listener McuListener, id st m.publishers[id+"|"+streamType] = client m.publisherCreated.Notify(id + "|" + streamType) m.mu.Unlock() + statsPublishersCurrent.WithLabelValues(streamType).Inc() + statsPublishersTotal.WithLabelValues(streamType).Inc() return client, nil } @@ -769,6 +868,16 @@ func (p *mcuJanusPublisher) handleSlowLink(event *janus.SlowLinkMsg) { } } +func (p *mcuJanusPublisher) handleMedia(event *janus.MediaMsg) { + mediaType := event.Type + if mediaType == "video" && p.streamType == "screen" { + // We want to differentiate between audio, video and screensharing + mediaType = p.streamType + } + + p.stats.EnableStream(mediaType, event.Receiving) +} + func (p *mcuJanusPublisher) NotifyReconnected() { ctx := context.TODO() handle, session, roomId, err := p.mcu.getOrCreatePublisherHandle(ctx, p.id, p.streamType, p.bitrate) @@ -808,13 +917,18 @@ func (p *mcuJanusPublisher) Close(ctx context.Context) { p.closeClient(ctx) p.mu.Unlock() + p.stats.Reset() + if notify { + statsPublishersCurrent.WithLabelValues(p.streamType).Dec() p.mcu.unregisterClient(p) p.listener.PublisherClosed(p) } + p.mcuJanusClient.Close(ctx) } func (p *mcuJanusPublisher) SendMessage(ctx context.Context, message *MessageClientMessage, data *MessageClientMessageData, callback func(error, map[string]interface{})) { + statsMcuMessagesTotal.WithLabelValues(data.Type).Inc() jsep_msg := data.Payload switch data.Type { case "offer": @@ -923,8 +1037,11 @@ func (m *mcuJanus) NewSubscriber(ctx context.Context, listener McuListener, publ client.mcuJanusClient.handleDetached = client.handleDetached client.mcuJanusClient.handleConnected = client.handleConnected client.mcuJanusClient.handleSlowLink = client.handleSlowLink + client.mcuJanusClient.handleMedia = client.handleMedia m.registerClient(client) go client.run(handle, client.closeChan) + statsSubscribersCurrent.WithLabelValues(streamType).Inc() + statsSubscribersTotal.WithLabelValues(streamType).Inc() return client, nil } @@ -963,6 +1080,7 @@ func (p *mcuJanusSubscriber) handleDetached(event *janus.DetachedMsg) { func (p *mcuJanusSubscriber) handleConnected(event *janus.WebRTCUpMsg) { log.Printf("Subscriber %d received connected", p.handleId) + p.mcu.SubscriberConnected(p.Id(), p.publisher, p.streamType) } func (p *mcuJanusSubscriber) handleSlowLink(event *janus.SlowLinkMsg) { @@ -973,6 +1091,10 @@ func (p *mcuJanusSubscriber) handleSlowLink(event *janus.SlowLinkMsg) { } } +func (p *mcuJanusSubscriber) handleMedia(event *janus.MediaMsg) { + // Only triggered for publishers +} + func (p *mcuJanusSubscriber) NotifyReconnected() { ctx, cancel := context.WithTimeout(context.Background(), p.mcu.mcuTimeout) defer cancel() @@ -992,11 +1114,16 @@ func (p *mcuJanusSubscriber) NotifyReconnected() { func (p *mcuJanusSubscriber) Close(ctx context.Context) { p.mu.Lock() - p.closeClient(ctx) + closed := p.closeClient(ctx) p.mu.Unlock() + if closed { + p.mcu.SubscriberDisconnected(p.Id(), p.publisher, p.streamType) + statsSubscribersCurrent.WithLabelValues(p.streamType).Dec() + } p.mcu.unregisterClient(p) p.listener.SubscriberClosed(p) + p.mcuJanusClient.Close(ctx) } func (p *mcuJanusSubscriber) joinRoom(ctx context.Context, callback func(error, map[string]interface{})) { @@ -1009,6 +1136,7 @@ func (p *mcuJanusSubscriber) joinRoom(ctx context.Context, callback func(error, waiter := p.mcu.publisherConnected.NewWaiter(p.publisher + "|" + p.streamType) defer p.mcu.publisherConnected.Release(waiter) + loggedNotPublishingYet := false retry: join_msg := map[string]interface{}{ "request": "join", @@ -1063,6 +1191,11 @@ retry: log.Printf("Publisher %s not sending yet for %s, wait and retry to join room %d as subscriber", p.publisher, p.streamType, p.roomId) } + if !loggedNotPublishingYet { + loggedNotPublishingYet = true + statsWaitingForPublisherTotal.WithLabelValues(p.streamType).Inc() + } + if err := waiter.Wait(ctx); err != nil { callback(err, nil) return @@ -1082,6 +1215,7 @@ retry: } func (p *mcuJanusSubscriber) SendMessage(ctx context.Context, message *MessageClientMessage, data *MessageClientMessageData, callback func(error, map[string]interface{})) { + statsMcuMessagesTotal.WithLabelValues(data.Type).Inc() jsep_msg := data.Payload switch data.Type { case "requestoffer": diff --git a/mcu_janus_test.go b/mcu_janus_test.go new file mode 100644 index 0000000..975089b --- /dev/null +++ b/mcu_janus_test.go @@ -0,0 +1,111 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "testing" +) + +func TestPublisherStatsCounter(t *testing.T) { + RegisterJanusMcuStats() + + var c publisherStatsCounter + + c.Reset() + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + c.EnableStream("audio", false) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + c.EnableStream("audio", true) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + c.EnableStream("audio", true) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + c.EnableStream("video", true) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + c.EnableStream("audio", false) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + c.EnableStream("audio", false) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + + c.AddSubscriber("1") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 1) + c.EnableStream("audio", true) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 1) + c.AddSubscriber("1") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 1) + + c.AddSubscriber("2") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 2) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 2) + + c.RemoveSubscriber("3") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 2) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 2) + + c.RemoveSubscriber("1") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 1) + + c.AddSubscriber("1") + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 2) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 2) + + c.EnableStream("audio", false) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 2) + + c.EnableStream("audio", true) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 1) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 1) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 2) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 2) + + c.EnableStream("audio", false) + c.EnableStream("video", false) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuPublisherStreamTypesCurrent.WithLabelValues("video"), 0) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("audio"), 0) + checkStatsValue(t, statsMcuSubscriberStreamTypesCurrent.WithLabelValues("video"), 0) + + collectAndLint(t, commonMcuStats...) +} diff --git a/mcu_proxy.go b/mcu_proxy.go index 4aa09cf..beb91a7 100644 --- a/mcu_proxy.go +++ b/mcu_proxy.go @@ -278,6 +278,7 @@ type mcuProxyConnection struct { reconnectTimer *time.Timer shutdownScheduled uint32 closeScheduled uint32 + trackClose uint32 helloMsgId string sessionId string @@ -500,6 +501,9 @@ func (c *mcuProxyConnection) close() { if c.conn != nil { c.conn.Close() c.conn = nil + if atomic.CompareAndSwapUint32(&c.trackClose, 1, 0) { + statsConnectedProxyBackendsCurrent.WithLabelValues(c.Country()).Dec() + } } } @@ -538,8 +542,8 @@ func (c *mcuProxyConnection) closeIfEmpty() bool { func (c *mcuProxyConnection) scheduleReconnect() { if err := c.sendClose(); err != nil && err != ErrNotConnected { log.Printf("Could not send close message to %s: %s", c.url, err) - c.close() } + c.close() interval := atomic.LoadInt64(&c.reconnectInterval) c.reconnectTimer.Reset(time.Duration(interval)) @@ -600,7 +604,10 @@ func (c *mcuProxyConnection) removePublisher(publisher *mcuProxyPublisher) { c.publishersLock.Lock() defer c.publishersLock.Unlock() - delete(c.publishers, publisher.proxyId) + if _, found := c.publishers[publisher.proxyId]; found { + delete(c.publishers, publisher.proxyId) + statsPublishersCurrent.WithLabelValues(publisher.StreamType()).Dec() + } delete(c.publisherIds, publisher.id+"|"+publisher.StreamType()) if len(c.publishers) == 0 && atomic.LoadUint32(&c.closeScheduled) != 0 { @@ -629,7 +636,10 @@ func (c *mcuProxyConnection) removeSubscriber(subscriber *mcuProxySubscriber) { c.subscribersLock.Lock() defer c.subscribersLock.Unlock() - delete(c.subscribers, subscriber.proxyId) + if _, found := c.subscribers[subscriber.proxyId]; found { + delete(c.subscribers, subscriber.proxyId) + statsSubscribersCurrent.WithLabelValues(subscriber.StreamType()).Dec() + } if len(c.subscribers) == 0 && atomic.LoadUint32(&c.closeScheduled) != 0 { go c.closeIfEmpty() @@ -708,6 +718,9 @@ func (c *mcuProxyConnection) processMessage(msg *ProxyServerMessage) { } else { log.Printf("Received session %s from %s", c.sessionId, c.url) } + if atomic.CompareAndSwapUint32(&c.trackClose, 0, 1) { + statsConnectedProxyBackendsCurrent.WithLabelValues(c.Country()).Inc() + } default: log.Printf("Received unsupported hello response %+v from %s, reconnecting", msg, c.url) c.scheduleReconnect() @@ -775,6 +788,7 @@ func (c *mcuProxyConnection) processEvent(msg *ProxyServerMessage) { log.Printf("Load of %s now at %d", c.url, event.Load) } atomic.StoreInt64(&c.load, event.Load) + statsProxyBackendLoadCurrent.WithLabelValues(c.url.String()).Set(float64(event.Load)) return case "shutdown-scheduled": log.Printf("Proxy %s is scheduled to shutdown", c.url) @@ -926,6 +940,8 @@ func (c *mcuProxyConnection) newPublisher(ctx context.Context, listener McuListe c.publishers[proxyId] = publisher c.publisherIds[id+"|"+streamType] = proxyId c.publishersLock.Unlock() + statsPublishersCurrent.WithLabelValues(streamType).Inc() + statsPublishersTotal.WithLabelValues(streamType).Inc() return publisher, nil } @@ -958,6 +974,8 @@ func (c *mcuProxyConnection) newSubscriber(ctx context.Context, listener McuList c.subscribersLock.Lock() c.subscribers[proxyId] = subscriber c.subscribersLock.Unlock() + statsSubscribersCurrent.WithLabelValues(streamType).Inc() + statsSubscribersTotal.WithLabelValues(streamType).Inc() return subscriber, nil } @@ -1692,6 +1710,7 @@ func (m *mcuProxy) NewPublisher(ctx context.Context, listener McuListener, id st return publisher, nil } + statsProxyNobackendAvailableTotal.WithLabelValues(streamType).Inc() return nil, fmt.Errorf("No MCU connection available") } @@ -1709,6 +1728,7 @@ func (m *mcuProxy) getPublisherConnection(ctx context.Context, publisher string, conn = m.publishers[publisher+"|"+streamType] if conn != nil { + // Publisher was created while waiting for lock. return conn } @@ -1716,6 +1736,7 @@ func (m *mcuProxy) getPublisherConnection(ctx context.Context, publisher string, id := m.addWaiter(ch) defer m.removeWaiter(id) + statsWaitingForPublisherTotal.WithLabelValues(streamType).Inc() for { m.mu.Unlock() select { diff --git a/mcu_proxy_test.go b/mcu_proxy_test.go index 8fc3422..1232602 100644 --- a/mcu_proxy_test.go +++ b/mcu_proxy_test.go @@ -25,6 +25,10 @@ import ( "testing" ) +func TestMcuProxyStats(t *testing.T) { + collectAndLint(t, proxyMcuStats...) +} + func newProxyConnectionWithCountry(country string) *mcuProxyConnection { conn := &mcuProxyConnection{} conn.country.Store(country) diff --git a/mcu_stats_prometheus.go b/mcu_stats_prometheus.go new file mode 100644 index 0000000..0d0e9ca --- /dev/null +++ b/mcu_stats_prometheus.go @@ -0,0 +1,131 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsPublishersCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "publishers", + Help: "The current number of publishers", + }, []string{"type"}) + statsPublishersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "publishers_total", + Help: "The total number of created publishers", + }, []string{"type"}) + statsSubscribersCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "subscribers", + Help: "The current number of subscribers", + }, []string{"type"}) + statsSubscribersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "subscribers_total", + Help: "The total number of created subscribers", + }, []string{"type"}) + statsWaitingForPublisherTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "nopublisher_total", + Help: "The total number of subscribe requests where no publisher exists", + }, []string{"type"}) + statsMcuMessagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "messages_total", + Help: "The total number of MCU messages", + }, []string{"type"}) + statsMcuSubscriberStreamTypesCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "subscriber_streams", + Help: "The current number of subscribed media streams", + }, []string{"type"}) + statsMcuPublisherStreamTypesCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "publisher_streams", + Help: "The current number of published media streams", + }, []string{"type"}) + + commonMcuStats = []prometheus.Collector{ + statsPublishersCurrent, + statsPublishersTotal, + statsSubscribersCurrent, + statsSubscribersTotal, + statsWaitingForPublisherTotal, + statsMcuMessagesTotal, + statsMcuSubscriberStreamTypesCurrent, + statsMcuPublisherStreamTypesCurrent, + } + + statsConnectedProxyBackendsCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "backend_connections", + Help: "Current number of connections to signaling proxy backends", + }, []string{"country"}) + statsProxyBackendLoadCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "backend_load", + Help: "Current load of signaling proxy backends", + }, []string{"url"}) + statsProxyNobackendAvailableTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "mcu", + Name: "no_backend_available_total", + Help: "Total number of publishing requests where no backend was available", + }, []string{"type"}) + + proxyMcuStats = []prometheus.Collector{ + statsConnectedProxyBackendsCurrent, + statsProxyBackendLoadCurrent, + statsProxyNobackendAvailableTotal, + } +) + +func RegisterJanusMcuStats() { + registerAll(commonMcuStats...) +} + +func UnregisterJanusMcuStats() { + unregisterAll(commonMcuStats...) +} + +func RegisterProxyMcuStats() { + registerAll(commonMcuStats...) + registerAll(proxyMcuStats...) +} + +func UnregisterProxyMcuStats() { + unregisterAll(commonMcuStats...) + unregisterAll(proxyMcuStats...) +} diff --git a/natsclient.go b/natsclient.go index a9e07ae..c82c00b 100644 --- a/natsclient.go +++ b/natsclient.go @@ -47,6 +47,8 @@ type NatsMessage struct { Room *BackendServerRoomRequest `json:"room,omitempty"` Permissions []Permission `json:"permissions,omitempty"` + + Id string `json:"id"` } type NatsSubscription interface { diff --git a/proxy/proxy_server.go b/proxy/proxy_server.go index 58e2030..a56a002 100644 --- a/proxy/proxy_server.go +++ b/proxy/proxy_server.go @@ -43,6 +43,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/securecookie" "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/promhttp" "gopkg.in/dgrijalva/jwt-go.v3" @@ -205,6 +206,7 @@ func NewProxyServer(r *mux.Router, version string, config *goconf.ConfigFile) (* r.HandleFunc("/proxy", result.setCommonHeaders(result.proxyHandler)).Methods("GET") r.HandleFunc("/stats", result.setCommonHeaders(result.validateStatsRequest(result.statsHandler))).Methods("GET") + r.HandleFunc("/metrics", result.setCommonHeaders(result.validateStatsRequest(result.metricsHandler))).Methods("GET") return result, nil } @@ -236,6 +238,9 @@ func (s *ProxyServer) Start(config *goconf.ConfigFile) error { switch mcuType { case signaling.McuTypeJanus: mcu, err = signaling.NewMcuJanus(s.url, config) + if err == nil { + signaling.RegisterJanusMcuStats() + } default: return fmt.Errorf("Unsupported MCU type: %s", mcuType) } @@ -555,6 +560,7 @@ func (s *ProxyServer) processMessage(client *ProxyClient, data []byte) { } else { s.sendCurrentLoad(session) } + statsSessionsResumedTotal.Inc() } else { var err error if session, err = s.NewSession(message.Hello); err != nil { @@ -619,6 +625,9 @@ func (i *emptyInitiator) Country() string { func (s *ProxyServer) processCommand(ctx context.Context, client *ProxyClient, session *ProxySession, message *signaling.ProxyClientMessage) { cmd := message.Command + + statsCommandMessagesTotal.WithLabelValues(cmd.Type).Inc() + switch cmd.Type { case "create-publisher": if atomic.LoadUint32(&s.shutdownScheduled) != 0 { @@ -650,6 +659,8 @@ func (s *ProxyServer) processCommand(ctx context.Context, client *ProxyClient, s }, } session.sendMessage(response) + statsPublishersCurrent.WithLabelValues(cmd.StreamType).Inc() + statsPublishersTotal.WithLabelValues(cmd.StreamType).Inc() case "create-subscriber": id := uuid.New().String() publisherId := cmd.PublisherId @@ -676,6 +687,8 @@ func (s *ProxyServer) processCommand(ctx context.Context, client *ProxyClient, s }, } session.sendMessage(response) + statsSubscribersCurrent.WithLabelValues(cmd.StreamType).Inc() + statsSubscribersTotal.WithLabelValues(cmd.StreamType).Inc() case "delete-publisher": client := s.GetClient(cmd.ClientId) if client == nil { @@ -688,7 +701,9 @@ func (s *ProxyServer) processCommand(ctx context.Context, client *ProxyClient, s return } - s.DeleteClient(cmd.ClientId, client) + if s.DeleteClient(cmd.ClientId, client) { + statsPublishersCurrent.WithLabelValues(client.StreamType()).Dec() + } go func() { log.Printf("Closing %s publisher %s as %s", client.StreamType(), client.Id(), cmd.ClientId) @@ -721,7 +736,9 @@ func (s *ProxyServer) processCommand(ctx context.Context, client *ProxyClient, s return } - s.DeleteClient(cmd.ClientId, client) + if s.DeleteClient(cmd.ClientId, client) { + statsSubscribersCurrent.WithLabelValues(client.StreamType()).Dec() + } go func() { log.Printf("Closing %s subscriber %s as %s", client.StreamType(), client.Id(), cmd.ClientId) @@ -750,6 +767,8 @@ func (s *ProxyServer) processPayload(ctx context.Context, client *ProxyClient, s return } + statsPayloadMessagesTotal.WithLabelValues(payload.Type).Inc() + var mcuData *signaling.MessageClientMessageData switch payload.Type { case "offer": @@ -811,42 +830,50 @@ func (s *ProxyServer) NewSession(hello *signaling.HelloProxyClientMessage) (*Pro log.Printf("Hello: %+v", hello) } + reason := "auth-failed" token, err := jwt.ParseWithClaims(hello.Token, &signaling.TokenClaims{}, func(token *jwt.Token) (interface{}, error) { // Don't forget to validate the alg is what you expect: if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { log.Printf("Unexpected signing method: %v", token.Header["alg"]) + reason = "unsupported-signing-method" return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) } claims, ok := token.Claims.(*signaling.TokenClaims) if !ok { log.Printf("Unsupported claims type: %+v", token.Claims) + reason = "unsupported-claims" return nil, fmt.Errorf("Unsupported claims type") } tokenKey, err := s.tokens.Get(claims.Issuer) if err != nil { log.Printf("Could not get token for %s: %s", claims.Issuer, err) + reason = "missing-issuer" return nil, err } if tokenKey == nil || tokenKey.key == nil { log.Printf("Issuer %s is not supported", claims.Issuer) + reason = "unsupported-issuer" return nil, fmt.Errorf("No key found for issuer") } return tokenKey.key, nil }) if err != nil { + statsTokenErrorsTotal.WithLabelValues(reason).Inc() return nil, TokenAuthFailed } claims, ok := token.Claims.(*signaling.TokenClaims) if !ok || !token.Valid { + statsTokenErrorsTotal.WithLabelValues("auth-failed").Inc() return nil, TokenAuthFailed } minIssuedAt := time.Now().Add(-maxTokenAge) if issuedAt := time.Unix(claims.IssuedAt, 0); issuedAt.Before(minIssuedAt) { + statsTokenErrorsTotal.WithLabelValues("expired").Inc() return nil, TokenExpired } @@ -868,6 +895,8 @@ func (s *ProxyServer) NewSession(hello *signaling.HelloProxyClientMessage) (*Pro log.Printf("Created session %s for %+v", encoded, claims) session := NewProxySession(s, sid, encoded) s.StoreSession(sid, session) + statsSessionsCurrent.Inc() + statsSessionsTotal.Inc() return session, nil } @@ -905,7 +934,11 @@ func (s *ProxyServer) DeleteSession(id uint64) { } func (s *ProxyServer) deleteSessionLocked(id uint64) { - delete(s.sessions, id) + if session, found := s.sessions[id]; found { + delete(s.sessions, id) + session.Close() + statsSessionsCurrent.Dec() + } } func (s *ProxyServer) StoreClient(id string, client signaling.McuClient) { @@ -915,9 +948,13 @@ func (s *ProxyServer) StoreClient(id string, client signaling.McuClient) { s.clientIds[client.Id()] = id } -func (s *ProxyServer) DeleteClient(id string, client signaling.McuClient) { +func (s *ProxyServer) DeleteClient(id string, client signaling.McuClient) bool { s.clientsLock.Lock() defer s.clientsLock.Unlock() + if _, found := s.clients[id]; !found { + return false + } + delete(s.clients, id) delete(s.clientIds, client.Id()) @@ -926,6 +963,7 @@ func (s *ProxyServer) DeleteClient(id string, client signaling.McuClient) { s.shutdownChannel <- true }() } + return true } func (s *ProxyServer) GetClientCount() int64 { @@ -986,3 +1024,8 @@ func (s *ProxyServer) statsHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(statsData) // nolint } + +func (s *ProxyServer) metricsHandler(w http.ResponseWriter, r *http.Request) { + // Expose prometheus metrics at "/metrics". + promhttp.Handler().ServeHTTP(w, r) +} diff --git a/proxy/proxy_session.go b/proxy/proxy_session.go index 2a6b0c6..7f3f84c 100644 --- a/proxy/proxy_session.go +++ b/proxy/proxy_session.go @@ -28,7 +28,7 @@ import ( "sync/atomic" "time" - "github.com/strukturag/nextcloud-spreed-signaling" + signaling "github.com/strukturag/nextcloud-spreed-signaling" ) const ( @@ -95,6 +95,11 @@ func (s *ProxySession) MarkUsed() { atomic.StoreInt64(&s.lastUsed, now.UnixNano()) } +func (s *ProxySession) Close() { + s.clearPublishers() + s.clearSubscribers() +} + func (s *ProxySession) SetClient(client *ProxyClient) *ProxyClient { s.clientLock.Lock() prev := s.client @@ -169,7 +174,9 @@ func (s *ProxySession) OnIceCompleted(client signaling.McuClient) { func (s *ProxySession) PublisherClosed(publisher signaling.McuPublisher) { if id := s.DeletePublisher(publisher); id != "" { - s.proxy.DeleteClient(id, publisher) + if s.proxy.DeleteClient(id, publisher) { + statsPublishersCurrent.WithLabelValues(publisher.StreamType()).Dec() + } msg := &signaling.ProxyServerMessage{ Type: "event", @@ -184,7 +191,9 @@ func (s *ProxySession) PublisherClosed(publisher signaling.McuPublisher) { func (s *ProxySession) SubscriberClosed(subscriber signaling.McuSubscriber) { if id := s.DeleteSubscriber(subscriber); id != "" { - s.proxy.DeleteClient(id, subscriber) + if s.proxy.DeleteClient(id, subscriber) { + statsSubscribersCurrent.WithLabelValues(subscriber.StreamType()).Dec() + } msg := &signaling.ProxyServerMessage{ Type: "event", @@ -246,7 +255,10 @@ func (s *ProxySession) clearPublishers() { defer s.publishersLock.Unlock() go func(publishers map[string]signaling.McuPublisher) { - for _, publisher := range publishers { + for id, publisher := range publishers { + if s.proxy.DeleteClient(id, publisher) { + statsPublishersCurrent.WithLabelValues(publisher.StreamType()).Dec() + } publisher.Close(context.Background()) } }(s.publishers) @@ -259,7 +271,10 @@ func (s *ProxySession) clearSubscribers() { defer s.publishersLock.Unlock() go func(subscribers map[string]signaling.McuSubscriber) { - for _, subscriber := range subscribers { + for id, subscriber := range subscribers { + if s.proxy.DeleteClient(id, subscriber) { + statsSubscribersCurrent.WithLabelValues(subscriber.StreamType()).Dec() + } subscriber.Close(context.Background()) } }(s.subscribers) diff --git a/proxy/proxy_stats_prometheus.go b/proxy/proxy_stats_prometheus.go new file mode 100644 index 0000000..054ec2b --- /dev/null +++ b/proxy/proxy_stats_prometheus.go @@ -0,0 +1,102 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package main + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsSessionsCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "sessions", + Help: "The current number of sessions", + }) + statsSessionsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "sessions_total", + Help: "The total number of created sessions", + }) + statsSessionsResumedTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "sessions_resumed_total", + Help: "The total number of resumed sessions", + }) + statsPublishersCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "publishers", + Help: "The current number of publishers", + }, []string{"type"}) + statsPublishersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "publishers_total", + Help: "The total number of created publishers", + }, []string{"type"}) + statsSubscribersCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "subscribers", + Help: "The current number of subscribers", + }, []string{"type"}) + statsSubscribersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "subscribers_total", + Help: "The total number of created subscribers", + }, []string{"type"}) + statsCommandMessagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "command_messages_total", + Help: "The total number of command messages", + }, []string{"type"}) + statsPayloadMessagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "payload_messages_total", + Help: "The total number of payload messages", + }, []string{"type"}) + statsTokenErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "proxy", + Name: "token_errors_total", + Help: "The total number of token errors", + }, []string{"reason"}) +) + +func init() { + prometheus.MustRegister(statsSessionsCurrent) + prometheus.MustRegister(statsSessionsTotal) + prometheus.MustRegister(statsSessionsResumedTotal) + prometheus.MustRegister(statsPublishersCurrent) + prometheus.MustRegister(statsPublishersTotal) + prometheus.MustRegister(statsSubscribersCurrent) + prometheus.MustRegister(statsSubscribersTotal) + prometheus.MustRegister(statsCommandMessagesTotal) + prometheus.MustRegister(statsPayloadMessagesTotal) + prometheus.MustRegister(statsTokenErrorsTotal) +} diff --git a/room.go b/room.go index a2d3a4c..857e7ed 100644 --- a/room.go +++ b/room.go @@ -32,6 +32,7 @@ import ( "time" "github.com/nats-io/nats.go" + "github.com/prometheus/client_golang/prometheus" ) const ( @@ -47,6 +48,10 @@ var ( updateActiveSessionsInterval = 10 * time.Second ) +func init() { + RegisterRoomStats() +} + type Room struct { id string hub *Hub @@ -64,6 +69,8 @@ type Room struct { inCallSessions map[Session]bool roomSessionData map[string]*RoomSessionData + statsRoomSessionsCurrent *prometheus.GaugeVec + natsReceiver chan *nats.Msg backendSubscription NatsSubscription @@ -123,6 +130,11 @@ func NewRoom(roomId string, properties *json.RawMessage, hub *Hub, n NatsClient, inCallSessions: make(map[Session]bool), roomSessionData: make(map[string]*RoomSessionData), + statsRoomSessionsCurrent: statsRoomSessionsCurrent.MustCurryWith(prometheus.Labels{ + "backend": backend.Id(), + "room": roomId, + }), + natsReceiver: natsReceiver, backendSubscription: backendSubscription, @@ -194,6 +206,9 @@ func (r *Room) Close() []Session { result = append(result, s) } r.sessions = nil + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeClient}) + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeInternal}) + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeVirtual}) r.mu.Unlock() return result } @@ -264,6 +279,9 @@ func (r *Room) AddSession(session Session, sessionData *json.RawMessage) []Sessi } } r.sessions[sid] = session + if !found { + r.statsRoomSessionsCurrent.With(prometheus.Labels{"clienttype": session.ClientType()}).Inc() + } var publishUsersChanged bool switch session.ClientType() { case HelloClientTypeInternal: @@ -311,6 +329,7 @@ func (r *Room) RemoveSession(session Session) bool { } sid := session.PublicId() + r.statsRoomSessionsCurrent.With(prometheus.Labels{"clienttype": session.ClientType()}).Dec() delete(r.sessions, sid) delete(r.internalSessions, session) if virtualSession, ok := session.(*VirtualSession); ok { @@ -325,6 +344,9 @@ func (r *Room) RemoveSession(session Session) bool { } r.hub.removeRoom(r) + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeClient}) + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeInternal}) + r.statsRoomSessionsCurrent.Delete(prometheus.Labels{"clienttype": HelloClientTypeVirtual}) r.unsubscribeBackend() r.doClose() r.mu.Unlock() diff --git a/room_stats_prometheus.go b/room_stats_prometheus.go new file mode 100644 index 0000000..eec8a96 --- /dev/null +++ b/room_stats_prometheus.go @@ -0,0 +1,43 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsRoomSessionsCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "signaling", + Subsystem: "room", + Name: "sessions", + Help: "The current number of sessions in a room", + }, []string{"backend", "room", "clienttype"}) + + roomStats = []prometheus.Collector{ + statsRoomSessionsCurrent, + } +) + +func RegisterRoomStats() { + registerAll(roomStats...) +} diff --git a/server/main.go b/server/main.go index 5568048..1b1b52c 100644 --- a/server/main.go +++ b/server/main.go @@ -141,6 +141,8 @@ func main() { runtime.GOMAXPROCS(cpus) log.Printf("Using a maximum of %d CPUs", cpus) + signaling.RegisterStats() + natsUrl, _ := config.GetString("nats", "url") if natsUrl == "" { natsUrl = nats.DefaultURL @@ -176,8 +178,12 @@ func main() { switch mcuType { case signaling.McuTypeJanus: mcu, err = signaling.NewMcuJanus(mcuUrl, config) + signaling.UnregisterProxyMcuStats() + signaling.RegisterJanusMcuStats() case signaling.McuTypeProxy: mcu, err = signaling.NewMcuProxy(config) + signaling.UnregisterJanusMcuStats() + signaling.RegisterProxyMcuStats() default: log.Fatal("Unsupported MCU type: ", mcuType) } diff --git a/stats_prometheus.go b/stats_prometheus.go new file mode 100644 index 0000000..93af6af --- /dev/null +++ b/stats_prometheus.go @@ -0,0 +1,59 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + statsMessagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "signaling", + Subsystem: "server", + Name: "messages_total", + Help: "The total number of signaling messages", + }, []string{"type"}) + + signalingStats = []prometheus.Collector{ + statsMessagesTotal, + } +) + +func registerAll(cs ...prometheus.Collector) { + for _, c := range cs { + if err := prometheus.DefaultRegisterer.Register(c); err != nil { + if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { + panic(err) + } + } + } +} + +func unregisterAll(cs ...prometheus.Collector) { + for _, c := range cs { + prometheus.Unregister(c) + } +} + +func RegisterStats() { + registerAll(signalingStats...) +} diff --git a/stats_prometheus_test.go b/stats_prometheus_test.go new file mode 100644 index 0000000..08d6a7a --- /dev/null +++ b/stats_prometheus_test.go @@ -0,0 +1,76 @@ +/** + * Standalone signaling server for the Nextcloud Spreed app. + * Copyright (C) 2021 struktur AG + * + * @author Joachim Bauch + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package signaling + +import ( + "fmt" + "runtime" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func checkStatsValue(t *testing.T, collector prometheus.Collector, value float64) { + ch := make(chan *prometheus.Desc, 1) + collector.Describe(ch) + desc := <-ch + v := testutil.ToFloat64(collector) + if v != value { + pc := make([]uintptr, 10) + n := runtime.Callers(2, pc) + if n == 0 { + t.Errorf("Expected value %f for %s, got %f", value, desc, v) + return + } + + pc = pc[:n] + frames := runtime.CallersFrames(pc) + stack := "" + for { + frame, more := frames.Next() + if !strings.Contains(frame.File, "nextcloud-spreed-signaling") { + break + } + stack += fmt.Sprintf("%s:%d\n", frame.File, frame.Line) + if !more { + break + } + } + t.Errorf("Expected value %f for %s, got %f at\n%s", value, desc, v, stack) + } +} + +func collectAndLint(t *testing.T, collectors ...prometheus.Collector) { + for _, collector := range collectors { + problems, err := testutil.CollectAndLint(collector) + if err != nil { + t.Errorf("Error linting %+v: %s", collector, err) + continue + } + + for _, problem := range problems { + t.Errorf("Problem with %s: %s", problem.Metric, problem.Text) + } + } +}