diff --git a/client.go b/client.go
index 62843218..ec527dd0 100644
--- a/client.go
+++ b/client.go
@@ -323,6 +323,7 @@ const (
 	LogBodyContextKey contextKey = iota
 	LogRequestIDContextKey
 	MaxAttemptsContextKey
+	SyncTokenContextKey
 )
 
 func (cli *Client) RequestStart(req *http.Request) {
diff --git a/crypto/cryptohelper/cryptohelper.go b/crypto/cryptohelper/cryptohelper.go
index 74710678..1939ea79 100644
--- a/crypto/cryptohelper/cryptohelper.go
+++ b/crypto/cryptohelper/cryptohelper.go
@@ -297,24 +297,14 @@ func (helper *CryptoHelper) HandleEncrypted(ctx context.Context, evt *event.Even
 	ctx = log.WithContext(ctx)
 
 	decrypted, err := helper.Decrypt(ctx, evt)
-	if errors.Is(err, NoSessionFound) {
-		log.Debug().
-			Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())).
-			Msg("Couldn't find session, waiting for keys to arrive...")
-		if helper.mach.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
-			log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
-			decrypted, err = helper.Decrypt(ctx, evt)
-		} else {
-			go helper.waitLongerForSession(ctx, log, evt)
-			return
-		}
-	}
-	if err != nil {
+	if errors.Is(err, NoSessionFound) && ctx.Value(mautrix.SyncTokenContextKey) != "" {
+		go helper.waitForSession(ctx, evt)
+	} else if err != nil {
 		log.Warn().Err(err).Msg("Failed to decrypt event")
 		helper.DecryptErrorCallback(evt, err)
-		return
+	} else {
+		helper.postDecrypt(ctx, decrypted)
 	}
-	helper.postDecrypt(ctx, decrypted)
 }
 
 func (helper *CryptoHelper) postDecrypt(ctx context.Context, decrypted *event.Event) {
@@ -355,7 +345,29 @@ func (helper *CryptoHelper) RequestSession(ctx context.Context, roomID id.RoomID
 	}
 }
 
-func (helper *CryptoHelper) waitLongerForSession(ctx context.Context, log zerolog.Logger, evt *event.Event) {
+func (helper *CryptoHelper) waitForSession(ctx context.Context, evt *event.Event) {
+	log := zerolog.Ctx(ctx)
+	content := evt.Content.AsEncrypted()
+
+	log.Debug().
+		Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())).
+		Msg("Couldn't find session, waiting for keys to arrive...")
+	if helper.mach.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
+		log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
+		decrypted, err := helper.Decrypt(ctx, evt)
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to decrypt event")
+			helper.DecryptErrorCallback(evt, err)
+		} else {
+			helper.postDecrypt(ctx, decrypted)
+		}
+	} else {
+		go helper.waitLongerForSession(ctx, evt)
+	}
+}
+
+func (helper *CryptoHelper) waitLongerForSession(ctx context.Context, evt *event.Event) {
+	log := zerolog.Ctx(ctx)
 	content := evt.Content.AsEncrypted()
 	log.Debug().Int("wait_seconds", int(extendedSessionWaitTimeout.Seconds())).Msg("Couldn't find session, requesting keys and waiting longer...")
 
diff --git a/sync.go b/sync.go
index c52bd2f9..598df8e0 100644
--- a/sync.go
+++ b/sync.go
@@ -90,6 +90,7 @@ func (s *DefaultSyncer) ProcessResponse(ctx context.Context, res *RespSync, sinc
 			err = fmt.Errorf("ProcessResponse panicked! since=%s panic=%s\n%s", since, r, debug.Stack())
 		}
 	}()
+	ctx = context.WithValue(ctx, SyncTokenContextKey, since)
 
 	for _, listener := range s.syncListeners {
 		if !listener(ctx, res, since) {