deblan-mirror/nextcloud-spreed-signaling
deblan-mirror/nextcloud-spreed-signaling
1
0
Fork 0
mirror of https://github.com/strukturag/nextcloud-spreed-signaling synced 2024-05-05 23:33:10 +02:00
Code Issues Releases Wiki Activity

Merge pull request #716 from strukturag/leak-grpc-goroutines

Browse source 
Prevent goroutine leaks in GRPC tests.
This commit is contained in:
Joachim Bauch 2024-04-23 10:59:23 +02:00 committed by GitHub
parent bc7aea68f3 b77525603c
commit 6960912681
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 204 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
certificate_reloader.go
View file

@ -66,6 +66,11 @@ func NewCertificateReloader(certFile string, keyFile string) (*CertificateReload
return reloader, nil
}
func (r *CertificateReloader) Close() {
r.keyWatcher.Close()
r.certWatcher.Close()
}
func (r *CertificateReloader) reload(filename string) {
log.Printf("reloading certificate from %s with %s", r.certFile, r.keyFile)
pair, err := tls.LoadX509KeyPair(r.certFile, r.keyFile)
@ -135,6 +140,10 @@ func NewCertPoolReloader(certFile string) (*CertPoolReloader, error) {
return reloader, nil
}
func (r *CertPoolReloader) Close() {
r.certWatcher.Close()
}
func (r *CertPoolReloader) reload(filename string) {
log.Printf("reloading certificate pool from %s", r.certFile)
pool, err := loadCertPool(r.certFile)

13
file_watcher.go
View file

@ -22,6 +22,7 @@
package signaling
import (
"context"
"errors"
"log"
"os"
@ -54,7 +55,9 @@ type FileWatcher struct {
target string
callback FileWatcherCallback
watcher *fsnotify.Watcher
watcher *fsnotify.Watcher
closeCtx context.Context
closeFunc context.CancelFunc
}
func NewFileWatcher(filename string, callback FileWatcherCallback) (*FileWatcher, error) {
@ -78,17 +81,23 @@ func NewFileWatcher(filename string, callback FileWatcherCallback) (*FileWatcher
return nil, err
}
closeCtx, closeFunc := context.WithCancel(context.Background())
w := &FileWatcher{
filename: filename,
target: realFilename,
callback: callback,
watcher: watcher,
closeCtx: closeCtx,
closeFunc: closeFunc,
}
go w.run()
return w, nil
}
func (f *FileWatcher) Close() error {
f.closeFunc()
return f.watcher.Close()
}
@ -152,6 +161,8 @@ func (f *FileWatcher) run() {
}
log.Printf("Error watching %s: %s", f.filename, err)
case <-f.closeCtx.Done():
return
}
}
}

82
file_watcher_test.go
View file

@ -47,48 +47,50 @@ func TestFileWatcher_NotExist(t *testing.T) {
}
func TestFileWatcher_File(t *testing.T) {
tmpdir := t.TempDir()
filename := path.Join(tmpdir, "test.txt")
if err := os.WriteFile(filename, []byte("Hello world!"), 0644); err != nil {
t.Fatal(err)
}
ensureNoGoroutinesLeak(t, func(t *testing.T) {
tmpdir := t.TempDir()
filename := path.Join(tmpdir, "test.txt")
if err := os.WriteFile(filename, []byte("Hello world!"), 0644); err != nil {
t.Fatal(err)
}
modified := make(chan struct{})
w, err := NewFileWatcher(filename, func(filename string) {
modified <- struct{}{}
modified := make(chan struct{})
w, err := NewFileWatcher(filename, func(filename string) {
modified <- struct{}{}
})
if err != nil {
t.Fatal(err)
}
defer w.Close()
if err := os.WriteFile(filename, []byte("Updated"), 0644); err != nil {
t.Fatal(err)
}
<-modified
ctxTimeout, cancel := context.WithTimeout(context.Background(), testWatcherNoEventTimeout)
defer cancel()
select {
case <-modified:
t.Error("should not have received another event")
case <-ctxTimeout.Done():
}
if err := os.WriteFile(filename, []byte("Updated"), 0644); err != nil {
t.Fatal(err)
}
<-modified
ctxTimeout, cancel = context.WithTimeout(context.Background(), testWatcherNoEventTimeout)
defer cancel()
select {
case <-modified:
t.Error("should not have received another event")
case <-ctxTimeout.Done():
}
})
if err != nil {
t.Fatal(err)
}
defer w.Close()
if err := os.WriteFile(filename, []byte("Updated"), 0644); err != nil {
t.Fatal(err)
}
<-modified
ctxTimeout, cancel := context.WithTimeout(context.Background(), testWatcherNoEventTimeout)
defer cancel()
select {
case <-modified:
t.Error("should not have received another event")
case <-ctxTimeout.Done():
}
if err := os.WriteFile(filename, []byte("Updated"), 0644); err != nil {
t.Fatal(err)
}
<-modified
ctxTimeout, cancel = context.WithTimeout(context.Background(), testWatcherNoEventTimeout)
defer cancel()
select {
case <-modified:
t.Error("should not have received another event")
case <-ctxTimeout.Done():
}
}
func TestFileWatcher_Rename(t *testing.T) {

15
grpc_client.go
View file

@ -38,6 +38,7 @@ import (
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/resolver"
status "google.golang.org/grpc/status"
)
@ -275,6 +276,7 @@ type GrpcClients struct {
targetPrefix string
targetInformation map[string]*GrpcTargetInformationEtcd
dialOptions atomic.Value // []grpc.DialOption
creds credentials.TransportCredentials
initializedCtx context.Context
initializedFunc context.CancelFunc
@ -308,6 +310,13 @@ func (c *GrpcClients) load(config *goconf.ConfigFile, fromReload bool) error {
return err
}
if c.creds != nil {
if cr, ok := c.creds.(*reloadableCredentials); ok {
cr.Close()
}
}
c.creds = creds
opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)}
c.dialOptions.Store(opts)
@ -490,6 +499,7 @@ func (c *GrpcClients) loadTargetsStatic(config *goconf.ConfigFile, fromReload bo
entry, found := clientsMap[target]
if !found {
entry = &grpcClientsList{}
clientsMap[target] = entry
}
entry.clients = append(entry.clients, client)
clients = append(clients, client)
@ -794,6 +804,11 @@ func (c *GrpcClients) Close() {
if c.etcdClient != nil {
c.etcdClient.RemoveListener(c)
}
if c.creds != nil {
if cr, ok := c.creds.(*reloadableCredentials); ok {
cr.Close()
}
}
c.closeFunc()
}

222
grpc_client_test.go
View file

@ -112,24 +112,26 @@ func waitForEvent(ctx context.Context, t *testing.T, ch <-chan struct{}) {
}
func Test_GrpcClients_EtcdInitial(t *testing.T) {
_, addr1 := NewGrpcServerForTest(t)
_, addr2 := NewGrpcServerForTest(t)
ensureNoGoroutinesLeak(t, func(t *testing.T) {
_, addr1 := NewGrpcServerForTest(t)
_, addr2 := NewGrpcServerForTest(t)
etcd := NewEtcdForTest(t)
etcd := NewEtcdForTest(t)
SetEtcdValue(etcd, "/grpctargets/one", []byte("{\"address\":\""+addr1+"\"}"))
SetEtcdValue(etcd, "/grpctargets/two", []byte("{\"address\":\""+addr2+"\"}"))
SetEtcdValue(etcd, "/grpctargets/one", []byte("{\"address\":\""+addr1+"\"}"))
SetEtcdValue(etcd, "/grpctargets/two", []byte("{\"address\":\""+addr2+"\"}"))
client, _ := NewGrpcClientsWithEtcdForTest(t, etcd)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := client.WaitForInitialized(ctx); err != nil {
t.Fatal(err)
}
client, _ := NewGrpcClientsWithEtcdForTest(t, etcd)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := client.WaitForInitialized(ctx); err != nil {
t.Fatal(err)
}
if clients := client.GetClients(); len(clients) != 2 {
t.Errorf("Expected two clients, got %+v", clients)
}
if clients := client.GetClients(); len(clients) != 2 {
t.Errorf("Expected two clients, got %+v", clients)
}
})
}
func Test_GrpcClients_EtcdUpdate(t *testing.T) {
@ -231,57 +233,59 @@ func Test_GrpcClients_EtcdIgnoreSelf(t *testing.T) {
}
func Test_GrpcClients_DnsDiscovery(t *testing.T) {
lookup := newMockDnsLookupForTest(t)
target := "testgrpc:12345"
ip1 := net.ParseIP("192.168.0.1")
ip2 := net.ParseIP("192.168.0.2")
targetWithIp1 := fmt.Sprintf("%s (%s)", target, ip1)
targetWithIp2 := fmt.Sprintf("%s (%s)", target, ip2)
lookup.Set("testgrpc", []net.IP{ip1})
client, dnsMonitor := NewGrpcClientsForTest(t, target)
ch := client.getWakeupChannelForTesting()
ensureNoGoroutinesLeak(t, func(t *testing.T) {
lookup := newMockDnsLookupForTest(t)
target := "testgrpc:12345"
ip1 := net.ParseIP("192.168.0.1")
ip2 := net.ParseIP("192.168.0.2")
targetWithIp1 := fmt.Sprintf("%s (%s)", target, ip1)
targetWithIp2 := fmt.Sprintf("%s (%s)", target, ip2)
lookup.Set("testgrpc", []net.IP{ip1})
client, dnsMonitor := NewGrpcClientsForTest(t, target)
ch := client.getWakeupChannelForTesting()
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
dnsMonitor.checkHostnames()
if clients := client.GetClients(); len(clients) != 1 {
t.Errorf("Expected one client, got %+v", clients)
} else if clients[0].Target() != targetWithIp1 {
t.Errorf("Expected target %s, got %s", targetWithIp1, clients[0].Target())
} else if !clients[0].ip.Equal(ip1) {
t.Errorf("Expected IP %s, got %s", ip1, clients[0].ip)
}
dnsMonitor.checkHostnames()
if clients := client.GetClients(); len(clients) != 1 {
t.Errorf("Expected one client, got %+v", clients)
} else if clients[0].Target() != targetWithIp1 {
t.Errorf("Expected target %s, got %s", targetWithIp1, clients[0].Target())
} else if !clients[0].ip.Equal(ip1) {
t.Errorf("Expected IP %s, got %s", ip1, clients[0].ip)
}
lookup.Set("testgrpc", []net.IP{ip1, ip2})
drainWakeupChannel(ch)
dnsMonitor.checkHostnames()
waitForEvent(ctx, t, ch)
lookup.Set("testgrpc", []net.IP{ip1, ip2})
drainWakeupChannel(ch)
dnsMonitor.checkHostnames()
waitForEvent(ctx, t, ch)
if clients := client.GetClients(); len(clients) != 2 {
t.Errorf("Expected two client, got %+v", clients)
} else if clients[0].Target() != targetWithIp1 {
t.Errorf("Expected target %s, got %s", targetWithIp1, clients[0].Target())
} else if !clients[0].ip.Equal(ip1) {
t.Errorf("Expected IP %s, got %s", ip1, clients[0].ip)
} else if clients[1].Target() != targetWithIp2 {
t.Errorf("Expected target %s, got %s", targetWithIp2, clients[1].Target())
} else if !clients[1].ip.Equal(ip2) {
t.Errorf("Expected IP %s, got %s", ip2, clients[1].ip)
}
if clients := client.GetClients(); len(clients) != 2 {
t.Errorf("Expected two client, got %+v", clients)
} else if clients[0].Target() != targetWithIp1 {
t.Errorf("Expected target %s, got %s", targetWithIp1, clients[0].Target())
} else if !clients[0].ip.Equal(ip1) {
t.Errorf("Expected IP %s, got %s", ip1, clients[0].ip)
} else if clients[1].Target() != targetWithIp2 {
t.Errorf("Expected target %s, got %s", targetWithIp2, clients[1].Target())
} else if !clients[1].ip.Equal(ip2) {
t.Errorf("Expected IP %s, got %s", ip2, clients[1].ip)
}
lookup.Set("testgrpc", []net.IP{ip2})
drainWakeupChannel(ch)
dnsMonitor.checkHostnames()
waitForEvent(ctx, t, ch)
lookup.Set("testgrpc", []net.IP{ip2})
drainWakeupChannel(ch)
dnsMonitor.checkHostnames()
waitForEvent(ctx, t, ch)
if clients := client.GetClients(); len(clients) != 1 {
t.Errorf("Expected one client, got %+v", clients)
} else if clients[0].Target() != targetWithIp2 {
t.Errorf("Expected target %s, got %s", targetWithIp2, clients[0].Target())
} else if !clients[0].ip.Equal(ip2) {
t.Errorf("Expected IP %s, got %s", ip2, clients[0].ip)
}
if clients := client.GetClients(); len(clients) != 1 {
t.Errorf("Expected one client, got %+v", clients)
} else if clients[0].Target() != targetWithIp2 {
t.Errorf("Expected target %s, got %s", targetWithIp2, clients[0].Target())
} else if !clients[0].ip.Equal(ip2) {
t.Errorf("Expected IP %s, got %s", ip2, clients[0].ip)
}
})
}
func Test_GrpcClients_DnsDiscoveryInitialFailed(t *testing.T) {
@ -320,55 +324,57 @@ func Test_GrpcClients_DnsDiscoveryInitialFailed(t *testing.T) {
}
func Test_GrpcClients_Encryption(t *testing.T) {
serverKey, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
t.Fatal(err)
}
clientKey, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
t.Fatal(err)
}
serverCert := GenerateSelfSignedCertificateForTesting(t, 1024, "Server cert", serverKey)
clientCert := GenerateSelfSignedCertificateForTesting(t, 1024, "Testing client", clientKey)
dir := t.TempDir()
serverPrivkeyFile := path.Join(dir, "server-privkey.pem")
serverPubkeyFile := path.Join(dir, "server-pubkey.pem")
serverCertFile := path.Join(dir, "server-cert.pem")
WritePrivateKey(serverKey, serverPrivkeyFile) // nolint
WritePublicKey(&serverKey.PublicKey, serverPubkeyFile) // nolint
os.WriteFile(serverCertFile, serverCert, 0755) // nolint
clientPrivkeyFile := path.Join(dir, "client-privkey.pem")
clientPubkeyFile := path.Join(dir, "client-pubkey.pem")
clientCertFile := path.Join(dir, "client-cert.pem")
WritePrivateKey(clientKey, clientPrivkeyFile) // nolint
WritePublicKey(&clientKey.PublicKey, clientPubkeyFile) // nolint
os.WriteFile(clientCertFile, clientCert, 0755) // nolint
serverConfig := goconf.NewConfigFile()
serverConfig.AddOption("grpc", "servercertificate", serverCertFile)
serverConfig.AddOption("grpc", "serverkey", serverPrivkeyFile)
serverConfig.AddOption("grpc", "clientca", clientCertFile)
_, addr := NewGrpcServerForTestWithConfig(t, serverConfig)
clientConfig := goconf.NewConfigFile()
clientConfig.AddOption("grpc", "targets", addr)
clientConfig.AddOption("grpc", "clientcertificate", clientCertFile)
clientConfig.AddOption("grpc", "clientkey", clientPrivkeyFile)
clientConfig.AddOption("grpc", "serverca", serverCertFile)
clients, _ := NewGrpcClientsForTestWithConfig(t, clientConfig, nil)
ctx, cancel1 := context.WithTimeout(context.Background(), time.Second)
defer cancel1()
if err := clients.WaitForInitialized(ctx); err != nil {
t.Fatal(err)
}
for _, client := range clients.GetClients() {
if _, err := client.GetServerId(ctx); err != nil {
ensureNoGoroutinesLeak(t, func(t *testing.T) {
serverKey, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
t.Fatal(err)
}
}
clientKey, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
t.Fatal(err)
}
serverCert := GenerateSelfSignedCertificateForTesting(t, 1024, "Server cert", serverKey)
clientCert := GenerateSelfSignedCertificateForTesting(t, 1024, "Testing client", clientKey)
dir := t.TempDir()
serverPrivkeyFile := path.Join(dir, "server-privkey.pem")
serverPubkeyFile := path.Join(dir, "server-pubkey.pem")
serverCertFile := path.Join(dir, "server-cert.pem")
WritePrivateKey(serverKey, serverPrivkeyFile) // nolint
WritePublicKey(&serverKey.PublicKey, serverPubkeyFile) // nolint
os.WriteFile(serverCertFile, serverCert, 0755) // nolint
clientPrivkeyFile := path.Join(dir, "client-privkey.pem")
clientPubkeyFile := path.Join(dir, "client-pubkey.pem")
clientCertFile := path.Join(dir, "client-cert.pem")
WritePrivateKey(clientKey, clientPrivkeyFile) // nolint
WritePublicKey(&clientKey.PublicKey, clientPubkeyFile) // nolint
os.WriteFile(clientCertFile, clientCert, 0755) // nolint
serverConfig := goconf.NewConfigFile()
serverConfig.AddOption("grpc", "servercertificate", serverCertFile)
serverConfig.AddOption("grpc", "serverkey", serverPrivkeyFile)
serverConfig.AddOption("grpc", "clientca", clientCertFile)
_, addr := NewGrpcServerForTestWithConfig(t, serverConfig)
clientConfig := goconf.NewConfigFile()
clientConfig.AddOption("grpc", "targets", addr)
clientConfig.AddOption("grpc", "clientcertificate", clientCertFile)
clientConfig.AddOption("grpc", "clientkey", clientPrivkeyFile)
clientConfig.AddOption("grpc", "serverca", serverCertFile)
clients, _ := NewGrpcClientsForTestWithConfig(t, clientConfig, nil)
ctx, cancel1 := context.WithTimeout(context.Background(), time.Second)
defer cancel1()
if err := clients.WaitForInitialized(ctx); err != nil {
t.Fatal(err)
}
for _, client := range clients.GetClients() {
if _, err := client.GetServerId(ctx); err != nil {
t.Fatal(err)
}
}
})
}

9
grpc_common.go
View file

@ -125,6 +125,15 @@ func (c *reloadableCredentials) OverrideServerName(serverName string) error {
return nil
}
func (c *reloadableCredentials) Close() {
if c.loader != nil {
c.loader.Close()
}
if c.pool != nil {
c.pool.Close()
}
}
func NewReloadableCredentials(config *goconf.ConfigFile, server bool) (credentials.TransportCredentials, error) {
var prefix string
var caPrefix string

3
grpc_server.go
View file

@ -108,6 +108,9 @@ func (s *GrpcServer) Run() error {
func (s *GrpcServer) Close() {
s.conn.GracefulStop()
if cr, ok := s.creds.(*reloadableCredentials); ok {
cr.Close()
}
}
func (s *GrpcServer) LookupSessionId(ctx context.Context, request *LookupSessionIdRequest) (*LookupSessionIdReply, error) {