diff --git a/bridgev2/ghost.go b/bridgev2/ghost.go
index 6cef6f06..f7072a9c 100644
--- a/bridgev2/ghost.go
+++ b/bridgev2/ghost.go
@@ -234,7 +234,7 @@ func (br *Bridge) allowAggressiveUpdateForType(evtType RemoteEventType) bool {
 }
 
 func (ghost *Ghost) UpdateInfoIfNecessary(ctx context.Context, source *UserLogin, evtType RemoteEventType) {
-	if ghost.Name != "" && ghost.NameSet && !ghost.Bridge.allowAggressiveUpdateForType(evtType) {
+	if ghost.Name != "" && ghost.NameSet && ghost.AvatarSet && !ghost.Bridge.allowAggressiveUpdateForType(evtType) {
 		return
 	}
 	info, err := source.Client.GetUserInfo(ctx, ghost)
@@ -244,12 +244,16 @@ func (ghost *Ghost) UpdateInfoIfNecessary(ctx context.Context, source *UserLogin
 		zerolog.Ctx(ctx).Debug().
 			Bool("has_name", ghost.Name != "").
 			Bool("name_set", ghost.NameSet).
+			Bool("has_avatar", ghost.AvatarMXC != "").
+			Bool("avatar_set", ghost.AvatarSet).
 			Msg("Updating ghost info in IfNecessary call")
 		ghost.UpdateInfo(ctx, info)
 	} else {
 		zerolog.Ctx(ctx).Trace().
 			Bool("has_name", ghost.Name != "").
 			Bool("name_set", ghost.NameSet).
+			Bool("has_avatar", ghost.AvatarMXC != "").
+			Bool("avatar_set", ghost.AvatarSet).
 			Msg("No ghost info received in IfNecessary call")
 	}
 }
@@ -277,6 +281,11 @@ func (ghost *Ghost) UpdateInfo(ctx context.Context, info *UserInfo) {
 	}
 	if info.Avatar != nil {
 		update = ghost.UpdateAvatar(ctx, info.Avatar) || update
+	} else if oldAvatar == "" && !ghost.AvatarSet {
+		// Special case: nil avatar means we're not expecting one ever, if we don't currently have
+		// one we flag it as set to avoid constantly refetching in UpdateInfoIfNecessary.
+		ghost.AvatarSet = true
+		update = true
 	}
 	if info.Identifiers != nil || info.IsBot != nil {
 		update = ghost.UpdateContactInfo(ctx, info.Identifiers, info.IsBot) || update