format/htmlparser: fix generating markdown for code blocks with backticks

Closes #468

.editorconfig

@@ -10,3 +10,6 @@ insert_final_newline = true
 
 [*.{yaml,yml}]
 indent_style = space
+
+[provisioning.yaml]
+indent_size = 2

.github/workflows/go.yml

@@ -2,17 +2,20 @@ name: Go
 
 on: [push, pull_request]
 
+env:
+  GOTOOLCHAIN: local
+
 jobs:
   lint:
     runs-on: ubuntu-latest
     name: Lint (latest)
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
-          go-version: "1.22"
+          go-version: "1.26"
           cache: true
 
       - name: Install libolm
@@ -21,27 +24,25 @@ jobs:
       - name: Install goimports
         run: |
           go install golang.org/x/tools/cmd/goimports@latest
+          go install honnef.co/go/tools/cmd/staticcheck@latest
           export PATH="$HOME/go/bin:$PATH"
 
-      - name: Install pre-commit
-        run: pip install pre-commit
-
-      - name: Lint
-        run: pre-commit run -a
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
 
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        go-version: ["1.21", "1.22"]
-    name: Build (${{ matrix.go-version == '1.22' && 'latest' || 'old' }}, libolm)
+        go-version: ["1.25", "1.26"]
+    name: Build (${{ matrix.go-version == '1.26' && 'latest' || 'old' }}, libolm)
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.go-version }}
           cache: true
@@ -60,30 +61,29 @@ jobs:
       - name: Test
         run: go test -json -v ./... 2>&1 | gotestfmt
 
+      - name: Test (jsonv2)
+        env:
+          GOEXPERIMENT: jsonv2
+        run: go test -json -v ./... 2>&1 | gotestfmt
+
   build-goolm:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        go-version: ["1.21", "1.22"]
-    name: Build (${{ matrix.go-version == '1.22' && 'latest' || 'old' }}, goolm)
+        go-version: ["1.25", "1.26"]
+    name: Build (${{ matrix.go-version == '1.26' && 'latest' || 'old' }}, goolm)
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.go-version }}
           cache: true
 
-      - name: Set up gotestfmt
-        uses: GoTestTools/gotestfmt-action@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Build
-        run: go build -tags=goolm -v ./...
-
-      - name: Test
-        run: go test -tags=goolm -json -v ./... 2>&1 | gotestfmt
+        run: |
+          rm -rf crypto/libolm
+          go build -tags=goolm -v ./...

.github/workflows/stale.yml

@@ -0,0 +1,29 @@
+name: 'Lock old issues'
+
+on:
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+#  pull-requests: write
+#  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  lock-stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v6
+        id: lock
+        with:
+          issue-inactive-days: 90
+          process-only: issues
+      - name: Log processed threads
+        run: |
+          if [ '${{ steps.lock.outputs.issues }}' ]; then
+            echo "Issues:" && echo '${{ steps.lock.outputs.issues }}' | jq -r '.[] | "https://github.com/\(.owner)/\(.repo)/issues/\(.issue_number)"'
+          fi

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
@@ -9,7 +9,7 @@ repos:
       - id: check-added-large-files
 
   - repo: https://github.com/tekwizely/pre-commit-golang
-    rev: v1.0.0-rc.1
+    rev: v1.0.0-rc.4
     hooks:
       - id: go-imports-repo
         args:
@@ -18,8 +18,12 @@ repos:
           - "-w"
       - id: go-vet-repo-mod
       - id: go-mod-tidy
+      - id: go-staticcheck-repo-mod
 
   - repo: https://github.com/beeper/pre-commit-go
-    rev: v0.3.1
+    rev: v0.4.2
     hooks:
       - id: prevent-literal-http-methods
+      - id: zerolog-ban-global-log
+      - id: zerolog-ban-msgf
+      - id: zerolog-use-stringer

CHANGELOG.md

@@ -1,3 +1,519 @@
+## v0.26.3 (2026-02-16)
+
+* Bumped minimum Go version to 1.25.
+* *(client)* Added fields for sending [MSC4354] sticky events.
+* *(bridgev2)* Added automatic message request accepting when sending message.
+* *(mediaproxy)* Added support for federation thumbnail endpoint.
+* *(crypto/ssss)* Improved support for recovery keys with slightly broken
+  metadata.
+* *(crypto)* Changed key import to call session received callback even for
+  sessions that already exist in the database.
+* *(appservice)* Fixed building websocket URL accidentally using file path
+  separators instead of always `/`.
+* *(crypto)* Fixed key exports not including the `sender_claimed_keys` field.
+* *(client)* Fixed incorrect context usage in async uploads.
+* *(crypto)* Fixed panic when passing invalid input to megolm message index
+  parser used for debugging.
+* *(bridgev2/provisioning)* Fixed completed or failed logins not being cleaned
+  up properly.
+
+[MSC4354]: https://github.com/matrix-org/matrix-spec-proposals/pull/4354
+
+## v0.26.2 (2026-01-16)
+
+* *(bridgev2)* Added chunked portal deletion to avoid database locks when
+  deleting large portals.
+* *(crypto,bridgev2)* Added option to encrypt reaction and reply metadata
+  as per [MSC4392].
+* *(bridgev2/login)* Added `default_value` for user input fields.
+* *(bridgev2)* Added interfaces to let the Matrix connector provide suggested
+  HTTP client settings and to reset active connections of the network connector.
+* *(bridgev2)* Added interface to let network connectors get the provisioning
+  API HTTP router and add new endpoints.
+* *(event)* Added blurhash field to Beeper link preview objects.
+* *(event)* Added [MSC4391] support for bot commands.
+* *(event)* Dropped [MSC4332] support for bot commands.
+* *(client)* Changed media download methods to return an error if the provided
+  MXC URI is empty.
+* *(client)* Stabilized support for [MSC4323].
+* *(bridgev2/matrix)* Fixed `GetEvent` panicking when trying to decrypt events.
+* *(bridgev2)* Fixed some deadlocks when room creation happens in parallel with
+  a portal re-ID call.
+
+[MSC4391]: https://github.com/matrix-org/matrix-spec-proposals/pull/4391
+[MSC4392]: https://github.com/matrix-org/matrix-spec-proposals/pull/4392
+
+## v0.26.1 (2025-12-16)
+
+* **Breaking change *(mediaproxy)*** Changed `GetMediaResponseFile` to return
+  the mime type from the callback rather than in the return get media return
+  value. The callback can now also redirect the caller to a different file.
+* *(federation)* Added join/knock/leave functions
+  (thanks to [@nexy7574] in [#422]).
+* *(federation/eventauth)* Fixed various incorrect checks.
+* *(client)* Added backoff for retrying media uploads to external URLs
+  (with MSC3870).
+* *(bridgev2/config)* Added support for overriding config fields using
+  environment variables.
+* *(bridgev2/commands)* Added command to mute chat on remote network.
+* *(bridgev2)* Added interface for network connectors to redirect to a different
+  user ID when handling an invite from Matrix.
+* *(bridgev2)* Added interface for signaling message request status of portals.
+* *(bridgev2)* Changed portal creation to not backfill unless `CanBackfill` flag
+  is set in chat info.
+* *(bridgev2)* Changed Matrix reaction handling to only delete old reaction if
+  bridging the new one is successful.
+* *(bridgev2/mxmain)* Improved error message when trying to run bridge with
+  pre-megabridge database when no database migration exists.
+* *(bridgev2)* Improved reliability of database migration when enabling split
+  portals.
+* *(bridgev2)* Improved detection of orphaned DM rooms when starting new chats.
+* *(bridgev2)* Stopped sending redundant invites when joining ghosts to public
+  portal rooms.
+* *(bridgev2)* Stopped hardcoding room versions in favor of checking
+  server capabilities to determine appropriate `/createRoom` parameters.
+
+[#422]: https://github.com/mautrix/go/pull/422
+
+## v0.26.0 (2025-11-16)
+
+* *(client,appservice)* Deprecated `SendMassagedStateEvent` as `SendStateEvent`
+  has been able to do the same for a while now.
+* *(client,federation)* Added size limits for responses to make it safer to send
+  requests to untrusted servers.
+* *(client)* Added wrapper for `/admin/whois` client API
+  (thanks to [@nexy7574] in [#411]).
+* *(synapseadmin)* Added `force_purge` option to DeleteRoom
+  (thanks to [@nexy7574] in [#420]).
+* *(statestore)* Added saving join rules for rooms.
+* *(bridgev2)* Added optional automatic rollback of room state if bridging the
+  change to the remote network fails.
+* *(bridgev2)* Added management room notices if transient disconnect state
+  doesn't resolve within 3 minutes.
+* *(bridgev2)* Added interface to signal that certain participants couldn't be
+  invited when creating a group.
+* *(bridgev2)* Added `select` type for user input fields in login.
+* *(bridgev2)* Added interface to let network connector customize personal
+  filtering space.
+* *(bridgev2/matrix)* Added checks to avoid sending error messages in reply to
+  other bots.
+* *(bridgev2/matrix)* Switched to using [MSC4169] to send redactions whenever
+  possible.
+* *(bridgev2/publicmedia)* Added support for custom path prefixes, file names,
+  and encrypted files.
+* *(bridgev2/commands)* Added command to resync a single portal.
+* *(bridgev2/commands)* Added create group command.
+* *(bridgev2/config)* Added option to limit maximum number of logins.
+* *(bridgev2)* Changed ghost joining to skip unnecessary invite if portal room
+  is public.
+* *(bridgev2/disappear)* Changed read receipt handling to only start
+  disappearing timers for messages up to the read message (note: may not work in
+  all cases if the read receipt points at an unknown event).
+* *(event/reply)* Changed plaintext reply fallback removal to only happen when
+  an HTML reply fallback is removed successfully.
+* *(bridgev2/matrix)* Fixed unnecessary sleep after registering bot on first run.
+* *(crypto/goolm)* Fixed panic when processing certain malformed Olm messages.
+* *(federation)* Fixed HTTP method for sending transactions
+  (thanks to [@nexy7574] in [#426]).
+* *(federation)* Fixed response body being closed even when using `DontReadBody`
+  parameter.
+* *(federation)* Fixed validating auth for requests with query params.
+* *(federation/eventauth)* Fixed typo causing restricted joins to not work.
+
+[MSC4169]: https://github.com/matrix-org/matrix-spec-proposals/pull/4169
+[#411]: github.com/mautrix/go/pull/411
+[#420]: github.com/mautrix/go/pull/420
+[#426]: github.com/mautrix/go/pull/426
+
+## v0.25.2 (2025-10-16)
+
+* **Breaking change *(id)*** Split `UserID.ParseAndValidate` into
+  `ParseAndValidateRelaxed` and `ParseAndValidateStrict`. Strict is the old
+  behavior, but most users likely want the relaxed version, as there are real
+  users whose user IDs aren't valid under the strict rules.
+* *(crypto)* Added helper methods for generating and verifying with recovery
+  keys.
+* *(bridgev2/matrix)* Added config option to automatically generate a recovery
+  key for the bridge bot and self-sign the bridge's device.
+* *(bridgev2/matrix)* Added initial support for using appservice/MSC3202 mode
+  for encryption with standard servers like Synapse.
+* *(bridgev2)* Added optional support for implicit read receipts.
+* *(bridgev2)* Added interface for deleting chats on remote network.
+* *(bridgev2)* Added local enforcement of media duration and size limits.
+* *(bridgev2)* Extended event duration logging to log any event taking too long.
+* *(bridgev2)* Improved validation in group creation provisioning API.
+* *(event)* Added event type constant for poll end events.
+* *(client)* Added wrapper for searching user directory.
+* *(client)* Improved support for managing [MSC4140] delayed events.
+* *(crypto/helper)* Changed default sync handling to not block on waiting for
+  decryption keys. On initial sync, keys won't be requested at all by default.
+* *(crypto)* Fixed olm unwedging not working (regressed in v0.25.1).
+* *(bridgev2)* Fixed various bugs with migrating to split portals.
+* *(event)* Fixed poll start events having incorrect null `m.relates_to`.
+* *(client)* Fixed `RespUserProfile` losing standard fields when re-marshaling.
+* *(federation)* Fixed various bugs in event auth.
+
+## v0.25.1 (2025-09-16)
+
+* *(client)* Fixed HTTP method of delete devices API call
+  (thanks to [@fmseals] in [#393]).
+* *(client)* Added wrappers for [MSC4323]: User suspension & locking endpoints
+  (thanks to [@nexy7574] in [#407]).
+* *(client)* Stabilized support for extensible profiles.
+* *(client)* Stabilized support for `state_after` in sync.
+* *(client)* Removed deprecated MSC2716 requests.
+* *(crypto)* Added fallback to ensure `m.relates_to` is always copied even if
+  the content struct doesn't implement `Relatable`.
+* *(crypto)* Changed olm unwedging to ignore newly created sessions if they
+  haven't been used successfully in either direction.
+* *(federation)* Added utilities for generating, parsing, validating and
+  authorizing PDUs.
+  * Note: the new PDU code depends on `GOEXPERIMENT=jsonv2`
+* *(event)* Added `is_animated` flag from [MSC4230] to file info.
+* *(event)* Added types for [MSC4332]: In-room bot commands.
+* *(event)* Added missing poll end event type for [MSC3381].
+* *(appservice)* Fixed URLs not being escaped properly when using unix socket
+  for homeserver connections.
+* *(format)* Added more helpers for forming markdown links.
+* *(event,bridgev2)* Added support for Beeper's disappearing message state event.
+* *(bridgev2)* Redesigned group creation interface and added support in commands
+  and provisioning API.
+* *(bridgev2)* Added GetEvent to Matrix interface to allow network connectors to
+  get an old event. The method is best effort only, as some configurations don't
+  allow fetching old events.
+* *(bridgev2)* Added shared logic for provisioning that can be reused by the
+  API, commands and other sources.
+* *(bridgev2)* Fixed mentions and URL previews not being copied over when
+  caption and media are merged.
+* *(bridgev2)* Removed config option to change provisioning API prefix, which
+  had already broken in the previous release.
+
+[@fmseals]: https://github.com/fmseals
+[#393]: https://github.com/mautrix/go/pull/393
+[#407]: https://github.com/mautrix/go/pull/407
+[MSC3381]: https://github.com/matrix-org/matrix-spec-proposals/pull/3381
+[MSC4230]: https://github.com/matrix-org/matrix-spec-proposals/pull/4230
+[MSC4323]: https://github.com/matrix-org/matrix-spec-proposals/pull/4323
+[MSC4332]: https://github.com/matrix-org/matrix-spec-proposals/pull/4332
+
+## v0.25.0 (2025-08-16)
+
+* Bumped minimum Go version to 1.24.
+* **Breaking change *(appservice,bridgev2,federation)*** Replaced gorilla/mux
+  with standard library ServeMux.
+* *(client,bridgev2)* Added support for creator power in room v12.
+* *(client)* Added option to not set `User-Agent` header for improved Wasm
+  compatibility.
+* *(bridgev2)* Added support for following tombstones.
+* *(bridgev2)* Added interface for getting arbitrary state event from Matrix.
+* *(bridgev2)* Added batching to disappearing message queue to ensure it doesn't
+  use too many resources even if there are a large number of messages.
+* *(bridgev2/commands)* Added support for canceling QR login with `cancel`
+  command.
+* *(client)* Added option to override HTTP client used for .well-known
+  resolution.
+* *(crypto/backup)* Added method for encrypting key backup session without
+  private keys.
+* *(event->id)* Moved room version type and constants to id package.
+* *(bridgev2)* Bots in DM portals will now be added to the functional members
+  state event to hide them from the room name calculation.
+* *(bridgev2)* Changed message delete handling to ignore "delete for me" events
+  if there are multiple Matrix users in the room.
+* *(format/htmlparser)* Changed text processing to collapse multiple spaces into
+  one when outside `pre`/`code` tags.
+* *(format/htmlparser)* Removed link suffix in plaintext output when link text
+  is only missing protocol part of href.
+  * e.g. `<a href="https://example.com">example.com</a>` will turn into
+    `example.com` rather than `example.com (https://example.com)`
+* *(appservice)* Switched appservice websockets from gorilla/websocket to
+  coder/websocket.
+* *(bridgev2/matrix)* Fixed encryption key sharing not ignoring ghosts properly.
+* *(crypto/attachments)* Fixed hash check when decrypting file streams.
+* *(crypto)* Removed unnecessary `AlreadyShared` error in `ShareGroupSession`.
+  The function will now act as if it was successful instead.
+
+## v0.24.2 (2025-07-16)
+
+* *(bridgev2)* Added support for return values from portal event handlers. Note
+  that the return value will always be "queued" unless the event buffer is
+  disabled.
+* *(bridgev2)* Added support for [MSC4144] per-message profile passthrough in
+  relay mode.
+* *(bridgev2)* Added option to auto-reconnect logins after a certain period if
+  they hit an `UNKNOWN_ERROR` state.
+* *(bridgev2)* Added analytics for event handler panics.
+* *(bridgev2)* Changed new room creation to hardcode room v11 to avoid v12 rooms
+  being created before proper support for them can be added.
+* *(bridgev2)* Changed queuing events to block instead of dropping events if the
+  buffer is full.
+* *(bridgev2)* Fixed assumption that replies to unknown messages are cross-room.
+* *(id)* Fixed server name validation not including ports correctly
+  (thanks to [@krombel] in [#392]).
+* *(federation)* Fixed base64 algorithm in signature generation.
+* *(event)* Fixed [MSC4144] fallbacks not being removed from edits.
+
+[@krombel]: https://github.com/krombel
+[#392]: https://github.com/mautrix/go/pull/392
+
+## v0.24.1 (2025-06-16)
+
+* *(commands)* Added framework for using reactions as buttons that execute
+  command handlers.
+* *(client)* Added wrapper for `/relations` endpoints.
+* *(client)* Added support for stable version of room summary endpoint.
+* *(client)* Fixed parsing URL preview responses where width/height are strings.
+* *(federation)* Fixed bugs in server auth.
+* *(id)* Added utilities for validating server names.
+* *(event)* Fixed incorrect empty `entity` field when sending hashed moderation
+  policy events.
+* *(event)* Added [MSC4293] redact events field to member events.
+* *(event)* Added support for fallbacks in [MSC4144] per-message profiles.
+* *(format)* Added `MarkdownLink` and `MarkdownMention` utility functions for
+  generating properly escaped markdown.
+* *(synapseadmin)* Added support for synchronous (v1) room delete endpoint.
+* *(synapseadmin)* Changed `Client` struct to not embed the `mautrix.Client`.
+  This is a breaking change if you were relying on accessing non-admin functions
+  from the admin client.
+* *(bridgev2/provisioning)* Fixed `/display_and_wait` not passing through errors
+  from the network connector properly.
+* *(bridgev2/crypto)* Fixed encryption not working if the user's ID had the same
+  prefix as the bridge ghosts (e.g. `@whatsappbridgeuser:example.com` with a
+  `@whatsapp_` prefix).
+* *(bridgev2)* Fixed portals not being saved after creating a DM portal from a
+  Matrix DM invite.
+* *(bridgev2)* Added config option to determine whether cross-room replies
+  should be bridged.
+* *(appservice)* Fixed `EnsureRegistered` not being called when sending a custom
+  member event for the controlled user.
+
+[MSC4293]: https://github.com/matrix-org/matrix-spec-proposals/pull/4293
+
+## v0.24.0 (2025-05-16)
+
+* *(commands)* Added generic framework for implementing bot commands.
+* *(client)* Added support for specifying maximum number of HTTP retries using
+  a context value instead of having to call `MakeFullRequest` manually.
+* *(client,federation)* Added methods for fetching room directories.
+* *(federation)* Added support for server side of request authentication.
+* *(synapseadmin)* Added wrapper for the account suspension endpoint.
+* *(format)* Added method for safely wrapping a string in markdown inline code.
+* *(crypto)* Added method to import key backup without persisting to database,
+  to allow the client more control over the process.
+* *(bridgev2)* Added viewing chat interface to signal when the user is viewing
+  a given chat.
+* *(bridgev2)* Added option to pass through transaction ID from client when
+  sending messages to remote network.
+* *(crypto)* Fixed unnecessary error log when decrypting dummy events used for
+  unwedging Olm sessions.
+* *(crypto)* Fixed `forwarding_curve25519_key_chain` not being set consistently
+  when backing up keys.
+* *(event)* Fixed marshaling legacy VoIP events with no version field.
+* *(bridgev2)* Fixed disappearing message references not being deleted when the
+  portal is deleted.
+* *(bridgev2)* Fixed read receipt bridging not ignoring fake message entries
+  and causing unnecessary error logs.
+
+## v0.23.3 (2025-04-16)
+
+* *(commands)* Added generic command processing framework for bots.
+* *(client)* Added `allowed_room_ids` field to room summary responses
+  (thanks to [@nexy7574] in [#367]).
+* *(bridgev2)* Added support for custom timeouts on outgoing messages which have
+  to wait for a remote echo.
+* *(bridgev2)* Added automatic typing stop event if the ghost user had sent a
+  typing event before a message.
+* *(bridgev2)* The saved management room is now cleared if the user leaves the
+  room, allowing the next DM to be automatically marked as a management room.
+* *(bridge)* Removed deprecated fallback package for bridge statuses.
+  The status package is now only available under bridgev2.
+
+[#367]: https://github.com/mautrix/go/pull/367
+
+## v0.23.2 (2025-03-16)
+
+* **Breaking change *(bridge)*** Removed legacy bridge module.
+* **Breaking change *(event)*** Changed `m.federate` field in room create event
+  content to a pointer to allow detecting omitted values.
+* *(bridgev2/commands)* Added `set-management-room` command to set a new
+  management room.
+* *(bridgev2/portal)* Changed edit bridging to ignore remote edits if the
+  original sender on Matrix can't be puppeted.
+* *(bridgv2)* Added config option to disable bridging `m.notice` messages.
+* *(appservice/http)* Switched access token validation to use constant time
+  comparisons.
+* *(event)* Added support for [MSC3765] rich text topics.
+* *(event)* Added fields to policy list event contents for [MSC4204] and
+  [MSC4205].
+* *(client)* Added method for getting the content of a redacted event using
+  [MSC2815].
+* *(client)* Added methods for sending and updating [MSC4140] delayed events.
+* *(client)* Added support for [MSC4222] in sync payloads.
+* *(crypto/cryptohelper)* Switched to using `sqlite3-fk-wal` instead of plain
+  `sqlite3` by default.
+* *(crypto/encryptolm)* Added generic method for encrypting to-device events.
+* *(crypto/ssss)* Fixed panic if server-side key metadata is corrupted.
+* *(crypto/sqlstore)* Fixed error when marking over 32 thousand device lists
+  as outdated on SQLite.
+
+[MSC2815]: https://github.com/matrix-org/matrix-spec-proposals/pull/2815
+[MSC3765]: https://github.com/matrix-org/matrix-spec-proposals/pull/3765
+[MSC4140]: https://github.com/matrix-org/matrix-spec-proposals/pull/4140
+[MSC4204]: https://github.com/matrix-org/matrix-spec-proposals/pull/4204
+[MSC4205]: https://github.com/matrix-org/matrix-spec-proposals/pull/4205
+[MSC4222]: https://github.com/matrix-org/matrix-spec-proposals/pull/4222
+
+## v0.23.1 (2025-02-16)
+
+* *(client)* Added `FullStateEvent` method to get a state event including
+  metadata (using the `?format=event` query parameter).
+* *(client)* Added wrapper method for [MSC4194]'s redact endpoint.
+* *(pushrules)* Fixed content rules not considering word boundaries and being
+  case-sensitive.
+* *(crypto)* Fixed bugs that would cause key exports to fail for no reason.
+* *(crypto)* Deprecated `ResolveTrust` in favor of `ResolveTrustContext`.
+* *(crypto)* Stopped accepting secret shares from unverified devices.
+* **Breaking change *(crypto)*** Changed `GetAndVerifyLatestKeyBackupVersion`
+  to take an optional private key parameter. The method will now trust the
+  public key if it matches the provided private key even if there are no valid
+  signatures.
+* **Breaking change *(crypto)*** Added context parameter to `IsDeviceTrusted`.
+
+[MSC4194]: https://github.com/matrix-org/matrix-spec-proposals/pull/4194
+
+## v0.23.0 (2025-01-16)
+
+* **Breaking change *(client)*** Changed `JoinRoom` parameters to allow multiple
+  `via`s.
+* **Breaking change *(bridgev2)*** Updated capability system.
+  * The return type of `NetworkAPI.GetCapabilities` is now different.
+  * Media type capabilities are enforced automatically by bridgev2.
+  * Capabilities are now sent to Matrix rooms using the
+    `com.beeper.room_features` state event.
+* *(client)* Added `GetRoomSummary` to implement [MSC3266].
+* *(client)* Added support for arbitrary profile fields to implement [MSC4133]
+  (thanks to [@nexy7574] in [#337]).
+* *(crypto)* Started storing olm message hashes to prevent decryption errors
+  if messages are repeated (e.g. if the app crashes right after decrypting).
+* *(crypto)* Improved olm session unwedging to check when the last session was
+  created instead of only relying on an in-memory map.
+* *(crypto/verificationhelper)* Fixed emoji verification not doing cross-signing
+  properly after a successful verification.
+* *(bridgev2/config)* Moved MSC4190 flag from `appservice` to `encryption`.
+* *(bridgev2/space)* Fixed failing to add rooms to spaces if the room create
+  call was made with a temporary context.
+* *(bridgev2/commands)* Changed `help` command to hide commands which require
+  interfaces that aren't implemented by the network connector.
+* *(bridgev2/matrixinterface)* Moved deterministic room ID generation to Matrix
+  connector.
+* *(bridgev2)* Fixed service member state event not being set correctly when
+  creating a DM by inviting a ghost user.
+* *(bridgev2)* Fixed `RemoteReactionSync` events replacing all reactions every
+  time instead of only changed ones.
+
+[MSC3266]: https://github.com/matrix-org/matrix-spec-proposals/pull/3266
+[MSC4133]: https://github.com/matrix-org/matrix-spec-proposals/pull/4133
+[@nexy7574]: https://github.com/nexy7574
+[#337]: https://github.com/mautrix/go/pull/337
+
+## v0.22.1 (2024-12-16)
+
+* *(crypto)* Added automatic cleanup when there are too many olm sessions with
+  a single device.
+* *(crypto)* Added helper for getting cached device list with cross-signing
+  status.
+* *(crypto/verificationhelper)* Added interface for persisting the state of
+  in-progress verifications.
+* *(client)* Added `GetMutualRooms` wrapper for [MSC2666].
+* *(client)* Switched `JoinRoom` to use the `via` query param instead of
+  `server_name` as per [MSC4156].
+* *(bridgev2/commands)* Fixed `pm` command not actually starting the chat.
+* *(bridgev2/interface)* Added separate network API interface for starting
+  chats with a Matrix ghost user. This allows treating internal user IDs
+  differently than arbitrary user-input strings.
+* *(bridgev2/crypto)* Added support for [MSC4190]
+  (thanks to [@onestacked] in [#288]).
+
+[MSC2666]: https://github.com/matrix-org/matrix-spec-proposals/pull/2666
+[MSC4156]: https://github.com/matrix-org/matrix-spec-proposals/pull/4156
+[MSC4190]: https://github.com/matrix-org/matrix-spec-proposals/pull/4190
+[#288]: https://github.com/mautrix/go/pull/288
+[@onestacked]: https://github.com/onestacked
+
+## v0.22.0 (2024-11-16)
+
+* *(hicli)* Moved package into gomuks repo.
+* *(bridgev2/commands)* Fixed cookie unescaping in login commands.
+* *(bridgev2/portal)* Added special `DefaultChatName` constant to explicitly
+  reset portal names to the default (based on members).
+* *(bridgev2/config)* Added options to disable room tag bridging.
+* *(bridgev2/database)* Fixed reaction queries not including portal receiver.
+* *(appservice)* Updated [MSC2409] stable registration field name from
+  `push_ephemeral` to `receive_ephemeral`. Homeserver admins must update
+  existing registrations manually.
+* *(format)* Added support for `img` tags.
+* *(format/mdext)* Added goldmark extensions for Matrix math and custom emojis.
+* *(event/reply)* Removed support for generating reply fallbacks ([MSC2781]).
+* *(pushrules)* Added support for `sender_notification_permission` condition
+  kind (used for `@room` mentions).
+* *(crypto)* Added support for `json.RawMessage` in `EncryptMegolmEvent`.
+* *(mediaproxy)* Added `GetMediaResponseCallback` and `GetMediaResponseFile`
+  to write proxied data directly to http response or temp file instead of
+  having to use an `io.Reader`.
+* *(mediaproxy)* Dropped support for legacy media download endpoints.
+* *(mediaproxy,bridgev2)* Made interface pass through query parameters.
+
+[MSC2781]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781
+
+## v0.21.1 (2024-10-16)
+
+* *(bridgev2)* Added more features and fixed bugs.
+* *(hicli)* Added more features and fixed bugs.
+* *(appservice)* Removed TLS support. A reverse proxy should be used if TLS
+  is needed.
+* *(format/mdext)* Added goldmark extension to fix indented paragraphs when
+  disabling indented code block parser.
+* *(event)* Added `Has` method for `Mentions`.
+* *(event)* Added basic support for the unstable version of polls.
+
+## v0.21.0 (2024-09-16)
+
+* **Breaking change *(client)*** Dropped support for unauthenticated media.
+  Matrix v1.11 support is now required from the homeserver, although it's not
+  enforced using `/versions` as some servers don't advertise it.
+* *(bridgev2)* Added more features and fixed bugs.
+* *(appservice,crypto)* Added support for using MSC3202 for appservice
+  encryption.
+* *(crypto/olm)* Made everything into an interface to allow side-by-side
+  testing of libolm and goolm, as well as potentially support vodozemac
+  in the future.
+* *(client)* Fixed requests being retried even after context is canceled.
+* *(client)* Added option to move `/sync` request logs to trace level.
+* *(error)* Added `Write` and `WithMessage` helpers to `RespError` to make it
+  easier to use on servers.
+* *(event)* Fixed `org.matrix.msc1767.audio` field allowing omitting the
+  duration and waveform.
+* *(id)* Changed `MatrixURI` methods to not panic if the receiver is nil.
+* *(federation)* Added limit to response size when fetching `.well-known` files.
+
+## v0.20.0 (2024-08-16)
+
+* Bumped minimum Go version to 1.22.
+* *(bridgev2)* Added more features and fixed bugs.
+* *(event)* Added types for [MSC4144]: Per-message profiles.
+* *(federation)* Added implementation of server name resolution and a basic
+  client for making federation requests.
+* *(crypto/ssss)* Changed recovery key/passphrase verify functions to take the
+  key ID as a parameter to ensure it's correctly set even if the key metadata
+  wasn't fetched via `GetKeyData`.
+* *(format/mdext)* Added goldmark extensions for single-character bold, italic
+  and strikethrough parsing (as in `*foo*` -> **foo**, `_foo_` -> _foo_ and
+  `~foo~` -> ~~foo~~)
+* *(format)* Changed `RenderMarkdown` et al to always include `m.mentions` in
+  returned content. The mention list is filled with matrix.to URLs from the
+  input by default.
+
+[MSC4144]: https://github.com/matrix-org/matrix-spec-proposals/pull/4144
+
 ## v0.19.0 (2024-07-16)
 
 * Renamed `master` branch to `main`.

README.md

@@ -1,8 +1,9 @@
 # mautrix-go
 [![GoDoc](https://pkg.go.dev/badge/maunium.net/go/mautrix)](https://pkg.go.dev/maunium.net/go/mautrix)
 
-A Golang Matrix framework. Used by [gomuks](https://matrix.org/docs/projects/client/gomuks),
-[go-neb](https://github.com/matrix-org/go-neb), [mautrix-whatsapp](https://github.com/mautrix/whatsapp)
+A Golang Matrix framework. Used by [gomuks](https://gomuks.app),
+[go-neb](https://github.com/matrix-org/go-neb),
+[mautrix-whatsapp](https://github.com/mautrix/whatsapp)
 and others.
 
 Matrix room: [`#go:maunium.net`](https://matrix.to/#/#go:maunium.net)
@@ -13,9 +14,10 @@ The original project is licensed under [Apache 2.0](https://github.com/matrix-or
 In addition to the basic client API features the original project has, this framework also has:
 
 * Appservice support (Intent API like mautrix-python, room state storage, etc)
-* End-to-end encryption support (incl. interactive SAS verification)
+* End-to-end encryption support (incl. key backup, cross-signing, interactive verification, etc)
 * High-level module for building puppeting bridges
-* High-level module for building chat clients
+* Partial federation module (making requests, PDU processing and event authorization)
+* A media proxy server which can be used to expose anything as a Matrix media repo
 * Wrapper functions for the Synapse admin API
 * Structs for parsing event content
 * Helpers for parsing and generating Matrix HTML

appservice/appservice.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tulir Asokan
+// Copyright (c) 2025 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -19,8 +19,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/gorilla/mux"
-	"github.com/gorilla/websocket"
+	"github.com/coder/websocket"
 	"github.com/rs/zerolog"
 	"golang.org/x/net/publicsuffix"
 	"gopkg.in/yaml.v3"
@@ -32,7 +31,7 @@ import (
 
 // EventChannelSize is the size for the Events channel in Appservice instances.
 var EventChannelSize = 64
-var OTKChannelSize = 4
+var OTKChannelSize = 64
 
 // Create creates a blank appservice instance.
 func Create() *AppService {
@@ -43,7 +42,7 @@ func Create() *AppService {
 		intents:    make(map[id.UserID]*IntentAPI),
 		HTTPClient: &http.Client{Timeout: 180 * time.Second, Jar: jar},
 		StateStore: mautrix.NewMemoryStateStore().(StateStore),
-		Router:     mux.NewRouter(),
+		Router:     http.NewServeMux(),
 		UserAgent:  mautrix.DefaultUserAgent,
 		txnIDC:     NewTransactionIDCache(128),
 		Live:       true,
@@ -61,12 +60,12 @@ func Create() *AppService {
 		DefaultHTTPRetries: 4,
 	}
 
-	as.Router.HandleFunc("/_matrix/app/v1/transactions/{txnID}", as.PutTransaction).Methods(http.MethodPut)
-	as.Router.HandleFunc("/_matrix/app/v1/rooms/{roomAlias}", as.GetRoom).Methods(http.MethodGet)
-	as.Router.HandleFunc("/_matrix/app/v1/users/{userID}", as.GetUser).Methods(http.MethodGet)
-	as.Router.HandleFunc("/_matrix/app/v1/ping", as.PostPing).Methods(http.MethodPost)
-	as.Router.HandleFunc("/_matrix/mau/live", as.GetLive).Methods(http.MethodGet)
-	as.Router.HandleFunc("/_matrix/mau/ready", as.GetReady).Methods(http.MethodGet)
+	as.Router.HandleFunc("PUT /_matrix/app/v1/transactions/{txnID}", as.PutTransaction)
+	as.Router.HandleFunc("GET /_matrix/app/v1/rooms/{roomAlias}", as.GetRoom)
+	as.Router.HandleFunc("GET /_matrix/app/v1/users/{userID}", as.GetUser)
+	as.Router.HandleFunc("POST /_matrix/app/v1/ping", as.PostPing)
+	as.Router.HandleFunc("GET /_matrix/mau/live", as.GetLive)
+	as.Router.HandleFunc("GET /_matrix/mau/ready", as.GetReady)
 
 	return as
 }
@@ -114,13 +113,13 @@ var _ StateStore = (*mautrix.MemoryStateStore)(nil)
 
 // QueryHandler handles room alias and user ID queries from the homeserver.
 type QueryHandler interface {
-	QueryAlias(alias string) bool
+	QueryAlias(alias id.RoomAlias) bool
 	QueryUser(userID id.UserID) bool
 }
 
 type QueryHandlerStub struct{}
 
-func (qh *QueryHandlerStub) QueryAlias(alias string) bool {
+func (qh *QueryHandlerStub) QueryAlias(alias id.RoomAlias) bool {
 	return false
 }
 
@@ -128,7 +127,7 @@ func (qh *QueryHandlerStub) QueryUser(userID id.UserID) bool {
 	return false
 }
 
-type WebsocketHandler func(WebsocketCommand) (ok bool, data interface{})
+type WebsocketHandler func(WebsocketCommand) (ok bool, data any)
 
 type StateStore interface {
 	mautrix.StateStore
@@ -160,7 +159,7 @@ type AppService struct {
 	QueryHandler   QueryHandler
 	StateStore     StateStore
 
-	Router       *mux.Router
+	Router       *http.ServeMux
 	UserAgent    string
 	server       *http.Server
 	HTTPClient   *http.Client
@@ -179,7 +178,6 @@ type AppService struct {
 	intentsLock sync.RWMutex
 
 	ws                    *websocket.Conn
-	wsWriteLock           sync.Mutex
 	StopWebsocket         func(error)
 	websocketHandlers     map[string]WebsocketHandler
 	websocketHandlersLock sync.RWMutex
@@ -224,9 +222,6 @@ type HostConfig struct {
 	Hostname string `yaml:"hostname"`
 	// Port is required when Hostname is an IP address, optional for unix sockets
 	Port uint16 `yaml:"port"`
-
-	TLSKey  string `yaml:"tls_key,omitempty"`
-	TLSCert string `yaml:"tls_cert,omitempty"`
 }
 
 // Address gets the whole address of the Appservice.
@@ -339,7 +334,7 @@ func (as *AppService) SetHomeserverURL(homeserverURL string) error {
 	} else if as.hsURLForClient.Scheme == "" {
 		as.hsURLForClient.Scheme = "https"
 	}
-	as.hsURLForClient.RawPath = parsedURL.EscapedPath()
+	as.hsURLForClient.RawPath = as.hsURLForClient.EscapedPath()
 
 	jar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 	as.HTTPClient = &http.Client{Timeout: 180 * time.Second, Jar: jar}
@@ -365,7 +360,7 @@ func (as *AppService) NewMautrixClient(userID id.UserID) *mautrix.Client {
 		AccessToken:         as.Registration.AppToken,
 		UserAgent:           as.UserAgent,
 		StateStore:          as.StateStore,
-		Log:                 as.Log.With().Str("as_user_id", userID.String()).Logger(),
+		Log:                 as.Log.With().Stringer("as_user_id", userID).Logger(),
 		Client:              as.HTTPClient,
 		DefaultHTTPRetries:  as.DefaultHTTPRetries,
 		SpecVersions:        as.SpecVersions,

appservice/http.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tulir Asokan
+// Copyright (c) 2025 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -17,8 +17,9 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/rs/zerolog"
+	"go.mau.fi/util/exhttp"
+	"go.mau.fi/util/exstrings"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
@@ -59,13 +60,8 @@ func (as *AppService) listenUnix() error {
 }
 
 func (as *AppService) listenTCP() error {
-	if len(as.Host.TLSCert) == 0 || len(as.Host.TLSKey) == 0 {
-		as.Log.Info().Str("address", as.server.Addr).Msg("Starting HTTP listener")
-		return as.server.ListenAndServe()
-	} else {
-		as.Log.Info().Str("address", as.server.Addr).Msg("Starting HTTP listener with TLS")
-		return as.server.ListenAndServeTLS(as.Host.TLSCert, as.Host.TLSKey)
-	}
+	as.Log.Info().Str("address", as.server.Addr).Msg("Starting HTTP listener")
+	return as.server.ListenAndServe()
 }
 
 func (as *AppService) Stop() {
@@ -83,17 +79,9 @@ func (as *AppService) Stop() {
 func (as *AppService) CheckServerToken(w http.ResponseWriter, r *http.Request) (isValid bool) {
 	authHeader := r.Header.Get("Authorization")
 	if !strings.HasPrefix(authHeader, "Bearer ") {
-		Error{
-			ErrorCode:  ErrUnknownToken,
-			HTTPStatus: http.StatusForbidden,
-			Message:    "Missing access token",
-		}.Write(w)
-	} else if authHeader[len("Bearer "):] != as.Registration.ServerToken {
-		Error{
-			ErrorCode:  ErrUnknownToken,
-			HTTPStatus: http.StatusForbidden,
-			Message:    "Incorrect access token",
-		}.Write(w)
+		mautrix.MMissingToken.WithMessage("Missing access token").Write(w)
+	} else if !exstrings.ConstantTimeEqual(authHeader[len("Bearer "):], as.Registration.ServerToken) {
+		mautrix.MUnknownToken.WithMessage("Invalid access token").Write(w)
 	} else {
 		isValid = true
 	}
@@ -106,24 +94,15 @@ func (as *AppService) PutTransaction(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	vars := mux.Vars(r)
-	txnID := vars["txnID"]
+	txnID := r.PathValue("txnID")
 	if len(txnID) == 0 {
-		Error{
-			ErrorCode:  ErrNoTransactionID,
-			HTTPStatus: http.StatusBadRequest,
-			Message:    "Missing transaction ID",
-		}.Write(w)
+		mautrix.MInvalidParam.WithMessage("Missing transaction ID").Write(w)
 		return
 	}
 	defer r.Body.Close()
 	body, err := io.ReadAll(r.Body)
 	if err != nil || len(body) == 0 {
-		Error{
-			ErrorCode:  ErrNotJSON,
-			HTTPStatus: http.StatusBadRequest,
-			Message:    "Missing request body",
-		}.Write(w)
+		mautrix.MNotJSON.WithMessage("Failed to read response body").Write(w)
 		return
 	}
 	log := as.Log.With().Str("transaction_id", txnID).Logger()
@@ -132,7 +111,7 @@ func (as *AppService) PutTransaction(w http.ResponseWriter, r *http.Request) {
 	ctx = log.WithContext(ctx)
 	if as.txnIDC.IsProcessed(txnID) {
 		// Duplicate transaction ID: no-op
-		WriteBlankOK(w)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 		log.Debug().Msg("Ignoring duplicate transaction")
 		return
 	}
@@ -141,14 +120,10 @@ func (as *AppService) PutTransaction(w http.ResponseWriter, r *http.Request) {
 	err = json.Unmarshal(body, &txn)
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to parse transaction content")
-		Error{
-			ErrorCode:  ErrBadJSON,
-			HTTPStatus: http.StatusBadRequest,
-			Message:    "Failed to parse body JSON",
-		}.Write(w)
+		mautrix.MBadJSON.WithMessage("Failed to parse transaction content").Write(w)
 	} else {
 		as.handleTransaction(ctx, txnID, &txn)
-		WriteBlankOK(w)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 	}
 }
 
@@ -226,7 +201,7 @@ func (as *AppService) handleEvents(ctx context.Context, evts []*event.Event, def
 		}
 		err := evt.Content.ParseRaw(evt.Type)
 		if errors.Is(err, event.ErrUnsupportedContentType) {
-			log.Debug().Str("event_id", evt.ID.String()).Msg("Not parsing content of unsupported event")
+			log.Debug().Stringer("event_id", evt.ID).Msg("Not parsing content of unsupported event")
 		} else if err != nil {
 			log.Warn().Err(err).
 				Str("event_id", evt.ID.String()).
@@ -263,16 +238,12 @@ func (as *AppService) GetRoom(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	vars := mux.Vars(r)
-	roomAlias := vars["roomAlias"]
+	roomAlias := id.RoomAlias(r.PathValue("roomAlias"))
 	ok := as.QueryHandler.QueryAlias(roomAlias)
 	if ok {
-		WriteBlankOK(w)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 	} else {
-		Error{
-			ErrorCode:  ErrUnknown,
-			HTTPStatus: http.StatusNotFound,
-		}.Write(w)
+		mautrix.MNotFound.WithMessage("Alias not found").Write(w)
 	}
 }
 
@@ -282,16 +253,12 @@ func (as *AppService) GetUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	vars := mux.Vars(r)
-	userID := id.UserID(vars["userID"])
+	userID := id.UserID(r.PathValue("userID"))
 	ok := as.QueryHandler.QueryUser(userID)
 	if ok {
-		WriteBlankOK(w)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 	} else {
-		Error{
-			ErrorCode:  ErrUnknown,
-			HTTPStatus: http.StatusNotFound,
-		}.Write(w)
+		mautrix.MNotFound.WithMessage("User not found").Write(w)
 	}
 }
 
@@ -301,11 +268,7 @@ func (as *AppService) PostPing(w http.ResponseWriter, r *http.Request) {
 	}
 	body, err := io.ReadAll(r.Body)
 	if err != nil || len(body) == 0 || !json.Valid(body) {
-		Error{
-			ErrorCode:  ErrNotJSON,
-			HTTPStatus: http.StatusBadRequest,
-			Message:    "Missing request body",
-		}.Write(w)
+		mautrix.MNotJSON.WithMessage("Invalid or missing request body").Write(w)
 		return
 	}
 
@@ -313,27 +276,21 @@ func (as *AppService) PostPing(w http.ResponseWriter, r *http.Request) {
 	_ = json.Unmarshal(body, &txn)
 	as.Log.Debug().Str("txn_id", txn.TxnID).Msg("Received ping from homeserver")
 
-	w.Header().Add("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("{}"))
+	exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 }
 
 func (as *AppService) GetLive(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", "application/json")
 	if as.Live {
-		w.WriteHeader(http.StatusOK)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 	} else {
-		w.WriteHeader(http.StatusInternalServerError)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusInternalServerError)
 	}
-	w.Write([]byte("{}"))
 }
 
 func (as *AppService) GetReady(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", "application/json")
 	if as.Ready {
-		w.WriteHeader(http.StatusOK)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusOK)
 	} else {
-		w.WriteHeader(http.StatusInternalServerError)
+		exhttp.WriteEmptyJSONResponse(w, http.StatusInternalServerError)
 	}
-	w.Write([]byte("{}"))
 }

appservice/intent.go

@@ -51,7 +51,7 @@ func (as *AppService) NewIntentAPI(localpart string) *IntentAPI {
 }
 
 func (intent *IntentAPI) Register(ctx context.Context) error {
-	_, err := intent.Client.MakeRequest(ctx, http.MethodPost, intent.BuildClientURL("v3", "register"), &mautrix.ReqRegister{
+	_, err := intent.Client.MakeRequest(ctx, http.MethodPost, intent.BuildClientURL("v3", "register"), &mautrix.ReqRegister[any]{
 		Username:     intent.Localpart,
 		Type:         mautrix.AuthTypeAppservice,
 		InhibitLogin: true,
@@ -86,6 +86,7 @@ func (intent *IntentAPI) EnsureRegistered(ctx context.Context) error {
 type EnsureJoinedParams struct {
 	IgnoreCache bool
 	BotOverride *mautrix.Client
+	Via         []string
 }
 
 func (intent *IntentAPI) EnsureJoined(ctx context.Context, roomID id.RoomID, extra ...EnsureJoinedParams) error {
@@ -99,11 +100,17 @@ func (intent *IntentAPI) EnsureJoined(ctx context.Context, roomID id.RoomID, ext
 		return nil
 	}
 
-	if err := intent.EnsureRegistered(ctx); err != nil {
+	err := intent.EnsureRegistered(ctx)
+	if err != nil {
 		return fmt.Errorf("failed to ensure joined: %w", err)
 	}
 
-	resp, err := intent.JoinRoomByID(ctx, roomID)
+	var resp *mautrix.RespJoinRoom
+	if len(params.Via) > 0 {
+		resp, err = intent.JoinRoom(ctx, roomID.String(), &mautrix.ReqJoinRoom{Via: params.Via})
+	} else {
+		resp, err = intent.JoinRoomByID(ctx, roomID)
+	}
 	if err != nil {
 		bot := intent.bot
 		if params.BotOverride != nil {
@@ -142,12 +149,16 @@ func (intent *IntentAPI) EnsureJoined(ctx context.Context, roomID id.RoomID, ext
 	return nil
 }
 
+func (intent *IntentAPI) IsDoublePuppet() bool {
+	return intent.IsCustomPuppet && intent.as.DoublePuppetValue != ""
+}
+
 func (intent *IntentAPI) AddDoublePuppetValue(into any) any {
 	return intent.AddDoublePuppetValueWithTS(into, 0)
 }
 
 func (intent *IntentAPI) AddDoublePuppetValueWithTS(into any, ts int64) any {
-	if !intent.IsCustomPuppet || intent.as.DoublePuppetValue == "" {
+	if !intent.IsDoublePuppet() {
 		return into
 	}
 	// Only use ts deduplication feature with appservice double puppeting
@@ -203,38 +214,45 @@ func (intent *IntentAPI) AddDoublePuppetValueWithTS(into any, ts int64) any {
 	}
 }
 
-func (intent *IntentAPI) SendMessageEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, contentJSON interface{}) (*mautrix.RespSendEvent, error) {
+func (intent *IntentAPI) SendMessageEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, contentJSON any, extra ...mautrix.ReqSendEvent) (*mautrix.RespSendEvent, error) {
 	if err := intent.EnsureJoined(ctx, roomID); err != nil {
 		return nil, err
 	}
 	contentJSON = intent.AddDoublePuppetValue(contentJSON)
-	return intent.Client.SendMessageEvent(ctx, roomID, eventType, contentJSON)
+	return intent.Client.SendMessageEvent(ctx, roomID, eventType, contentJSON, extra...)
 }
 
-func (intent *IntentAPI) SendMassagedMessageEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, contentJSON interface{}, ts int64) (*mautrix.RespSendEvent, error) {
+func (intent *IntentAPI) BeeperSendEphemeralEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, contentJSON any, extra ...mautrix.ReqSendEvent) (*mautrix.RespSendEvent, error) {
 	if err := intent.EnsureJoined(ctx, roomID); err != nil {
 		return nil, err
 	}
-	contentJSON = intent.AddDoublePuppetValueWithTS(contentJSON, ts)
-	return intent.Client.SendMessageEvent(ctx, roomID, eventType, contentJSON, mautrix.ReqSendEvent{Timestamp: ts})
+	if !intent.SpecVersions.Supports(mautrix.BeeperFeatureEphemeralEvents) {
+		return nil, mautrix.MUnrecognized.WithMessage("Homeserver does not advertise com.beeper.ephemeral support")
+	}
+	contentJSON = intent.AddDoublePuppetValue(contentJSON)
+	return intent.Client.BeeperSendEphemeralEvent(ctx, roomID, eventType, contentJSON, extra...)
 }
 
-func (intent *IntentAPI) SendStateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, contentJSON interface{}) (*mautrix.RespSendEvent, error) {
+// Deprecated: use SendMessageEvent with mautrix.ReqSendEvent.Timestamp instead
+func (intent *IntentAPI) SendMassagedMessageEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, contentJSON interface{}, ts int64) (*mautrix.RespSendEvent, error) {
+	return intent.SendMessageEvent(ctx, roomID, eventType, contentJSON, mautrix.ReqSendEvent{Timestamp: ts})
+}
+
+func (intent *IntentAPI) SendStateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, contentJSON any, extra ...mautrix.ReqSendEvent) (*mautrix.RespSendEvent, error) {
 	if eventType != event.StateMember || stateKey != string(intent.UserID) {
 		if err := intent.EnsureJoined(ctx, roomID); err != nil {
 			return nil, err
 		}
-	}
-	contentJSON = intent.AddDoublePuppetValue(contentJSON)
-	return intent.Client.SendStateEvent(ctx, roomID, eventType, stateKey, contentJSON)
-}
-
-func (intent *IntentAPI) SendMassagedStateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, contentJSON interface{}, ts int64) (*mautrix.RespSendEvent, error) {
-	if err := intent.EnsureJoined(ctx, roomID); err != nil {
+	} else if err := intent.EnsureRegistered(ctx); err != nil {
 		return nil, err
 	}
-	contentJSON = intent.AddDoublePuppetValueWithTS(contentJSON, ts)
-	return intent.Client.SendMassagedStateEvent(ctx, roomID, eventType, stateKey, contentJSON, ts)
+	contentJSON = intent.AddDoublePuppetValue(contentJSON)
+	return intent.Client.SendStateEvent(ctx, roomID, eventType, stateKey, contentJSON, extra...)
+}
+
+// Deprecated: use SendStateEvent with mautrix.ReqSendEvent.Timestamp instead
+func (intent *IntentAPI) SendMassagedStateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, contentJSON interface{}, ts int64) (*mautrix.RespSendEvent, error) {
+	return intent.SendStateEvent(ctx, roomID, eventType, stateKey, contentJSON, mautrix.ReqSendEvent{Timestamp: ts})
 }
 
 func (intent *IntentAPI) StateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, outContent interface{}) error {
@@ -293,7 +311,7 @@ func (intent *IntentAPI) SendCustomMembershipEvent(ctx context.Context, roomID i
 func (intent *IntentAPI) JoinRoomByID(ctx context.Context, roomID id.RoomID, extraContent ...map[string]interface{}) (resp *mautrix.RespJoinRoom, err error) {
 	if intent.IsCustomPuppet || len(extraContent) > 0 {
 		_, err = intent.SendCustomMembershipEvent(ctx, roomID, intent.UserID, event.MembershipJoin, "", extraContent...)
-		return &mautrix.RespJoinRoom{}, err
+		return &mautrix.RespJoinRoom{RoomID: roomID}, err
 	}
 	return intent.Client.JoinRoomByID(ctx, roomID)
 }
@@ -362,6 +380,24 @@ func (intent *IntentAPI) Member(ctx context.Context, roomID id.RoomID, userID id
 	return member
 }
 
+func (intent *IntentAPI) FillPowerLevelCreateEvent(ctx context.Context, roomID id.RoomID, pl *event.PowerLevelsEventContent) error {
+	if pl.CreateEvent != nil {
+		return nil
+	}
+	var err error
+	pl.CreateEvent, err = intent.StateStore.GetCreate(ctx, roomID)
+	if err != nil {
+		return fmt.Errorf("failed to get create event from cache: %w", err)
+	} else if pl.CreateEvent != nil {
+		return nil
+	}
+	pl.CreateEvent, err = intent.FullStateEvent(ctx, roomID, event.StateCreate, "")
+	if err != nil {
+		return fmt.Errorf("failed to get create event from server: %w", err)
+	}
+	return nil
+}
+
 func (intent *IntentAPI) PowerLevels(ctx context.Context, roomID id.RoomID) (pl *event.PowerLevelsEventContent, err error) {
 	pl, err = intent.as.StateStore.GetPowerLevels(ctx, roomID)
 	if err != nil {
@@ -371,6 +407,12 @@ func (intent *IntentAPI) PowerLevels(ctx context.Context, roomID id.RoomID) (pl
 	if pl == nil {
 		pl = &event.PowerLevelsEventContent{}
 		err = intent.StateEvent(ctx, roomID, event.StatePowerLevels, "", pl)
+		if err != nil {
+			return
+		}
+	}
+	if pl.CreateEvent == nil {
+		pl.CreateEvent, err = intent.FullStateEvent(ctx, roomID, event.StateCreate, "")
 	}
 	return
 }
@@ -385,8 +427,7 @@ func (intent *IntentAPI) SetPowerLevel(ctx context.Context, roomID id.RoomID, us
 		return nil, err
 	}
 
-	if pl.GetUserLevel(userID) != level {
-		pl.SetUserLevel(userID, level)
+	if pl.EnsureUserLevelAs(intent.UserID, userID, level) {
 		return intent.SendStateEvent(ctx, roomID, event.StatePowerLevels, "", &pl)
 	}
 	return nil, nil
@@ -436,6 +477,20 @@ func (intent *IntentAPI) SetRoomTopic(ctx context.Context, roomID id.RoomID, top
 	})
 }
 
+func (intent *IntentAPI) UploadMedia(ctx context.Context, data mautrix.ReqUploadMedia) (*mautrix.RespMediaUpload, error) {
+	if err := intent.EnsureRegistered(ctx); err != nil {
+		return nil, err
+	}
+	return intent.Client.UploadMedia(ctx, data)
+}
+
+func (intent *IntentAPI) UploadAsync(ctx context.Context, data mautrix.ReqUploadMedia) (*mautrix.RespCreateMXC, error) {
+	if err := intent.EnsureRegistered(ctx); err != nil {
+		return nil, err
+	}
+	return intent.Client.UploadAsync(ctx, data)
+}
+
 func (intent *IntentAPI) SetDisplayName(ctx context.Context, displayName string) error {
 	if err := intent.EnsureRegistered(ctx); err != nil {
 		return err
@@ -461,7 +516,7 @@ func (intent *IntentAPI) SetAvatarURL(ctx context.Context, avatarURL id.ContentU
 		// No need to update
 		return nil
 	}
-	if !avatarURL.IsEmpty() {
+	if !avatarURL.IsEmpty() && !intent.SpecVersions.Supports(mautrix.BeeperFeatureHungry) {
 		// Some homeservers require the avatar to be downloaded before setting it
 		resp, _ := intent.Download(ctx, avatarURL)
 		if resp != nil {

appservice/ping.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2025 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package appservice
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	"maunium.net/go/mautrix"
+)
+
+func (intent *IntentAPI) EnsureAppserviceConnection(ctx context.Context) {
+	var pingResp *mautrix.RespAppservicePing
+	var txnID string
+	var retryCount int
+	var err error
+	const maxRetries = 6
+	for {
+		txnID = intent.TxnID()
+		pingResp, err = intent.AppservicePing(ctx, intent.as.Registration.ID, txnID)
+		if err == nil {
+			break
+		}
+		var httpErr mautrix.HTTPError
+		var pingErrBody string
+		if errors.As(err, &httpErr) && httpErr.RespError != nil {
+			if val, ok := httpErr.RespError.ExtraData["body"].(string); ok {
+				pingErrBody = strings.TrimSpace(val)
+			}
+		}
+		outOfRetries := retryCount >= maxRetries
+		level := zerolog.ErrorLevel
+		if outOfRetries {
+			level = zerolog.FatalLevel
+		}
+		evt := zerolog.Ctx(ctx).WithLevel(level).Err(err).Str("txn_id", txnID)
+		if pingErrBody != "" {
+			bodyBytes := []byte(pingErrBody)
+			if json.Valid(bodyBytes) {
+				evt.RawJSON("body", bodyBytes)
+			} else {
+				evt.Str("body", pingErrBody)
+			}
+		}
+		if outOfRetries {
+			evt.Msg("Homeserver -> appservice connection is not working")
+			zerolog.Ctx(ctx).Info().Msg("See https://docs.mau.fi/faq/as-ping for more info")
+			os.Exit(13)
+		}
+		evt.Msg("Homeserver -> appservice connection is not working, retrying in 5 seconds...")
+		time.Sleep(5 * time.Second)
+		retryCount++
+	}
+	zerolog.Ctx(ctx).Debug().
+		Str("txn_id", txnID).
+		Int64("duration_ms", pingResp.DurationMS).
+		Msg("Homeserver -> appservice connection works")
+}

appservice/protocol.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tulir Asokan
+// Copyright (c) 2025 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -7,9 +7,7 @@
 package appservice
 
 import (
-	"encoding/json"
 	"fmt"
-	"net/http"
 	"strings"
 
 	"github.com/rs/zerolog"
@@ -103,50 +101,3 @@ func (txn *Transaction) ContentString() string {
 
 // EventListener is a function that receives events.
 type EventListener func(evt *event.Event)
-
-// WriteBlankOK writes a blank OK message as a reply to a HTTP request.
-func WriteBlankOK(w http.ResponseWriter) {
-	w.Header().Add("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write([]byte("{}"))
-}
-
-// Respond responds to a HTTP request with a JSON object.
-func Respond(w http.ResponseWriter, data interface{}) error {
-	w.Header().Add("Content-Type", "application/json")
-	dataStr, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-	_, err = w.Write(dataStr)
-	return err
-}
-
-// Error represents a Matrix protocol error.
-type Error struct {
-	HTTPStatus int       `json:"-"`
-	ErrorCode  ErrorCode `json:"errcode"`
-	Message    string    `json:"error"`
-}
-
-func (err Error) Write(w http.ResponseWriter) {
-	w.Header().Add("Content-Type", "application/json")
-	w.WriteHeader(err.HTTPStatus)
-	_ = Respond(w, &err)
-}
-
-// ErrorCode is the machine-readable code in an Error.
-type ErrorCode string
-
-// Native ErrorCodes
-const (
-	ErrUnknownToken ErrorCode = "M_UNKNOWN_TOKEN"
-	ErrBadJSON      ErrorCode = "M_BAD_JSON"
-	ErrNotJSON      ErrorCode = "M_NOT_JSON"
-	ErrUnknown      ErrorCode = "M_UNKNOWN"
-)
-
-// Custom ErrorCodes
-const (
-	ErrNoTransactionID ErrorCode = "NET.MAUNIUM.NO_TRANSACTION_ID"
-)

appservice/registration.go

@@ -27,7 +27,9 @@ type Registration struct {
 	Protocols       []string   `yaml:"protocols,omitempty" json:"protocols,omitempty"`
 
 	SoruEphemeralEvents bool `yaml:"de.sorunome.msc2409.push_ephemeral,omitempty" json:"de.sorunome.msc2409.push_ephemeral,omitempty"`
-	EphemeralEvents     bool `yaml:"push_ephemeral,omitempty" json:"push_ephemeral,omitempty"`
+	EphemeralEvents     bool `yaml:"receive_ephemeral,omitempty" json:"receive_ephemeral,omitempty"`
+	MSC3202             bool `yaml:"org.matrix.msc3202,omitempty" json:"org.matrix.msc3202,omitempty"`
+	MSC4190             bool `yaml:"io.element.msc4190,omitempty" json:"io.element.msc4190,omitempty"`
 }
 
 // CreateRegistration creates a Registration with random appservice and homeserver tokens.

appservice/websocket.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tulir Asokan
+// Copyright (c) 2025 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -11,26 +11,26 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
-	"path/filepath"
+	"path"
 	"strings"
 	"sync"
 	"sync/atomic"
-	"time"
 
-	"github.com/gorilla/websocket"
+	"github.com/coder/websocket"
 	"github.com/rs/zerolog"
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
+
+	"maunium.net/go/mautrix"
 )
 
 type WebsocketRequest struct {
-	ReqID   int         `json:"id,omitempty"`
-	Command string      `json:"command"`
-	Data    interface{} `json:"data"`
-
-	Deadline time.Duration `json:"-"`
+	ReqID   int    `json:"id,omitempty"`
+	Command string `json:"command"`
+	Data    any    `json:"data"`
 }
 
 type WebsocketCommand struct {
@@ -41,7 +41,7 @@ type WebsocketCommand struct {
 	Ctx context.Context `json:"-"`
 }
 
-func (wsc *WebsocketCommand) MakeResponse(ok bool, data interface{}) *WebsocketRequest {
+func (wsc *WebsocketCommand) MakeResponse(ok bool, data any) *WebsocketRequest {
 	if wsc.ReqID == 0 || wsc.Command == "response" || wsc.Command == "error" {
 		return nil
 	}
@@ -56,7 +56,7 @@ func (wsc *WebsocketCommand) MakeResponse(ok bool, data interface{}) *WebsocketR
 		var prefixMessage string
 		for unwrappedErr != nil {
 			errorData, jsonErr = json.Marshal(unwrappedErr)
-			if errorData != nil && len(errorData) > 2 && jsonErr == nil {
+			if len(errorData) > 2 && jsonErr == nil {
 				prefixMessage = strings.Replace(err.Error(), unwrappedErr.Error(), "", 1)
 				prefixMessage = strings.TrimRight(prefixMessage, ": ")
 				break
@@ -98,8 +98,8 @@ type WebsocketMessage struct {
 }
 
 const (
-	WebsocketCloseConnReplaced       = 4001
-	WebsocketCloseTxnNotAcknowledged = 4002
+	WebsocketCloseConnReplaced       websocket.StatusCode = 4001
+	WebsocketCloseTxnNotAcknowledged websocket.StatusCode = 4002
 )
 
 type MeowWebsocketCloseCode string
@@ -133,7 +133,7 @@ func (mwcc MeowWebsocketCloseCode) String() string {
 }
 
 type CloseCommand struct {
-	Code    int                    `json:"-"`
+	Code    websocket.StatusCode   `json:"-"`
 	Command string                 `json:"command"`
 	Status  MeowWebsocketCloseCode `json:"status"`
 }
@@ -143,15 +143,15 @@ func (cc CloseCommand) Error() string {
 }
 
 func parseCloseError(err error) error {
-	closeError := &websocket.CloseError{}
+	var closeError websocket.CloseError
 	if !errors.As(err, &closeError) {
 		return err
 	}
 	var closeCommand CloseCommand
 	closeCommand.Code = closeError.Code
 	closeCommand.Command = "disconnect"
-	if len(closeError.Text) > 0 {
-		jsonErr := json.Unmarshal([]byte(closeError.Text), &closeCommand)
+	if len(closeError.Reason) > 0 {
+		jsonErr := json.Unmarshal([]byte(closeError.Reason), &closeCommand)
 		if jsonErr != nil {
 			return err
 		}
@@ -159,7 +159,7 @@ func parseCloseError(err error) error {
 	if len(closeCommand.Status) == 0 {
 		if closeCommand.Code == WebsocketCloseConnReplaced {
 			closeCommand.Status = MeowConnectionReplaced
-		} else if closeCommand.Code == websocket.CloseServiceRestart {
+		} else if closeCommand.Code == websocket.StatusServiceRestart {
 			closeCommand.Status = MeowServerShuttingDown
 		}
 	}
@@ -170,20 +170,23 @@ func (as *AppService) HasWebsocket() bool {
 	return as.ws != nil
 }
 
-func (as *AppService) SendWebsocket(cmd *WebsocketRequest) error {
+func (as *AppService) SendWebsocket(ctx context.Context, cmd *WebsocketRequest) error {
 	ws := as.ws
 	if cmd == nil {
 		return nil
 	} else if ws == nil {
 		return ErrWebsocketNotConnected
 	}
-	as.wsWriteLock.Lock()
-	defer as.wsWriteLock.Unlock()
-	if cmd.Deadline == 0 {
-		cmd.Deadline = 3 * time.Minute
+	wr, err := ws.Writer(ctx, websocket.MessageText)
+	if err != nil {
+		return err
 	}
-	_ = ws.SetWriteDeadline(time.Now().Add(cmd.Deadline))
-	return ws.WriteJSON(cmd)
+	err = json.NewEncoder(wr).Encode(cmd)
+	if err != nil {
+		_ = wr.Close()
+		return err
+	}
+	return wr.Close()
 }
 
 func (as *AppService) clearWebsocketResponseWaiters() {
@@ -220,12 +223,12 @@ func (er *ErrorResponse) Error() string {
 	return fmt.Sprintf("%s: %s", er.Code, er.Message)
 }
 
-func (as *AppService) RequestWebsocket(ctx context.Context, cmd *WebsocketRequest, response interface{}) error {
+func (as *AppService) RequestWebsocket(ctx context.Context, cmd *WebsocketRequest, response any) error {
 	cmd.ReqID = int(atomic.AddInt32(&as.websocketRequestID, 1))
 	respChan := make(chan *WebsocketCommand, 1)
 	as.addWebsocketResponseWaiter(cmd.ReqID, respChan)
 	defer as.removeWebsocketResponseWaiter(cmd.ReqID, respChan)
-	err := as.SendWebsocket(cmd)
+	err := as.SendWebsocket(ctx, cmd)
 	if err != nil {
 		return err
 	}
@@ -254,7 +257,7 @@ func (as *AppService) RequestWebsocket(ctx context.Context, cmd *WebsocketReques
 	}
 }
 
-func (as *AppService) unknownCommandHandler(cmd WebsocketCommand) (bool, interface{}) {
+func (as *AppService) unknownCommandHandler(cmd WebsocketCommand) (bool, any) {
 	zerolog.Ctx(cmd.Ctx).Warn().Msg("No handler for websocket command")
 	return false, fmt.Errorf("unknown request type")
 }
@@ -278,14 +281,28 @@ func (as *AppService) defaultHandleWebsocketTransaction(ctx context.Context, msg
 	return true, &WebsocketTransactionResponse{TxnID: msg.TxnID}
 }
 
-func (as *AppService) consumeWebsocket(stopFunc func(error), ws *websocket.Conn) {
+func (as *AppService) consumeWebsocket(ctx context.Context, stopFunc func(error), ws *websocket.Conn) {
 	defer stopFunc(ErrWebsocketUnknownError)
-	ctx := context.Background()
 	for {
-		var msg WebsocketMessage
-		err := ws.ReadJSON(&msg)
+		msgType, reader, err := ws.Reader(ctx)
 		if err != nil {
-			as.Log.Debug().Err(err).Msg("Error reading from websocket")
+			as.Log.Debug().Err(err).Msg("Error getting reader from websocket")
+			stopFunc(parseCloseError(err))
+			return
+		} else if msgType != websocket.MessageText {
+			as.Log.Debug().Msg("Ignoring non-text message from websocket")
+			continue
+		}
+		data, err := io.ReadAll(reader)
+		if err != nil {
+			as.Log.Debug().Err(err).Msg("Error reading data from websocket")
+			stopFunc(parseCloseError(err))
+			return
+		}
+		var msg WebsocketMessage
+		err = json.Unmarshal(data, &msg)
+		if err != nil {
+			as.Log.Debug().Err(err).Msg("Error parsing JSON received from websocket")
 			stopFunc(parseCloseError(err))
 			return
 		}
@@ -296,11 +313,11 @@ func (as *AppService) consumeWebsocket(stopFunc func(error), ws *websocket.Conn)
 			with = with.Str("transaction_id", msg.TxnID)
 		}
 		log := with.Logger()
-		ctx = log.WithContext(ctx)
+		ctx := log.WithContext(ctx)
 		if msg.Command == "" || msg.Command == "transaction" {
 			ok, resp := as.WebsocketTransactionHandler(ctx, msg)
 			go func() {
-				err := as.SendWebsocket(msg.MakeResponse(ok, resp))
+				err := as.SendWebsocket(ctx, msg.MakeResponse(ok, resp))
 				if err != nil {
 					log.Warn().Err(err).Msg("Failed to send response to websocket transaction")
 				} else {
@@ -332,7 +349,7 @@ func (as *AppService) consumeWebsocket(stopFunc func(error), ws *websocket.Conn)
 			}
 			go func() {
 				okResp, data := handler(msg.WebsocketCommand)
-				err := as.SendWebsocket(msg.MakeResponse(okResp, data))
+				err := as.SendWebsocket(ctx, msg.MakeResponse(okResp, data))
 				if err != nil {
 					log.Error().Err(err).Msg("Failed to send response to websocket command")
 				} else if okResp {
@@ -345,7 +362,7 @@ func (as *AppService) consumeWebsocket(stopFunc func(error), ws *websocket.Conn)
 	}
 }
 
-func (as *AppService) StartWebsocket(baseURL string, onConnect func()) error {
+func (as *AppService) StartWebsocket(ctx context.Context, baseURL string, onConnect func()) error {
 	var parsed *url.URL
 	if baseURL != "" {
 		var err error
@@ -357,26 +374,29 @@ func (as *AppService) StartWebsocket(baseURL string, onConnect func()) error {
 		copiedURL := *as.hsURLForClient
 		parsed = &copiedURL
 	}
-	parsed.Path = filepath.Join(parsed.Path, "_matrix/client/unstable/fi.mau.as_sync")
+	parsed.Path = path.Join(parsed.Path, "_matrix/client/unstable/fi.mau.as_sync")
 	if parsed.Scheme == "http" {
 		parsed.Scheme = "ws"
 	} else if parsed.Scheme == "https" {
 		parsed.Scheme = "wss"
 	}
-	ws, resp, err := websocket.DefaultDialer.Dial(parsed.String(), http.Header{
-		"Authorization": []string{fmt.Sprintf("Bearer %s", as.Registration.AppToken)},
-		"User-Agent":    []string{as.BotClient().UserAgent},
+	ws, resp, err := websocket.Dial(ctx, parsed.String(), &websocket.DialOptions{
+		HTTPClient: as.HTTPClient,
+		HTTPHeader: http.Header{
+			"Authorization": []string{fmt.Sprintf("Bearer %s", as.Registration.AppToken)},
+			"User-Agent":    []string{as.BotClient().UserAgent},
 
-		"X-Mautrix-Process-ID":        []string{as.ProcessID},
-		"X-Mautrix-Websocket-Version": []string{"3"},
+			"X-Mautrix-Process-ID":        []string{as.ProcessID},
+			"X-Mautrix-Websocket-Version": []string{"3"},
+		},
 	})
 	if resp != nil && resp.StatusCode >= 400 {
-		var errResp Error
+		var errResp mautrix.RespError
 		err = json.NewDecoder(resp.Body).Decode(&errResp)
 		if err != nil {
 			return fmt.Errorf("websocket request returned HTTP %d with non-JSON body", resp.StatusCode)
 		} else {
-			return fmt.Errorf("websocket request returned %s (HTTP %d): %s", errResp.ErrorCode, resp.StatusCode, errResp.Message)
+			return fmt.Errorf("websocket request returned %s (HTTP %d): %s", errResp.ErrCode, resp.StatusCode, errResp.Err)
 		}
 	} else if err != nil {
 		return fmt.Errorf("failed to open websocket: %w", err)
@@ -399,12 +419,13 @@ func (as *AppService) StartWebsocket(baseURL string, onConnect func()) error {
 			}
 		})
 	}
+	ws.SetReadLimit(50 * 1024 * 1024)
 	as.ws = ws
 	as.StopWebsocket = stopFunc
 	as.PrepareWebsocket()
 	as.Log.Debug().Msg("Appservice transaction websocket opened")
 
-	go as.consumeWebsocket(stopFunc, ws)
+	go as.consumeWebsocket(ctx, stopFunc, ws)
 
 	var onConnectDone atomic.Bool
 	if onConnect != nil {
@@ -426,12 +447,7 @@ func (as *AppService) StartWebsocket(baseURL string, onConnect func()) error {
 		as.ws = nil
 	}
 
-	_ = ws.SetWriteDeadline(time.Now().Add(3 * time.Second))
-	err = ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, ""))
-	if err != nil && !errors.Is(err, websocket.ErrCloseSent) {
-		as.Log.Warn().Err(err).Msg("Error writing close message to websocket")
-	}
-	err = ws.Close()
+	err = ws.Close(websocket.StatusGoingAway, "")
 	if err != nil {
 		as.Log.Warn().Err(err).Msg("Error closing websocket")
 	}

appservice/wshttp.go

@@ -61,6 +61,11 @@ func (as *AppService) WebsocketHTTPProxy(cmd WebsocketCommand) (bool, interface{
 	if err != nil {
 		return false, fmt.Errorf("failed to create fake HTTP request: %w", err)
 	}
+	httpReq.RequestURI = req.Path
+	if req.Query != "" {
+		httpReq.RequestURI += "?" + req.Query
+	}
+	httpReq.RemoteAddr = "websocket"
 	httpReq.Header = req.Headers
 
 	var resp HTTPProxyResponse

bridge/bridge.go

@@ -1,932 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridge
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/url"
-	"os"
-	"os/signal"
-	"runtime"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/lib/pq"
-	"github.com/mattn/go-sqlite3"
-	"github.com/rs/zerolog"
-	"go.mau.fi/util/configupgrade"
-	"go.mau.fi/util/dbutil"
-	_ "go.mau.fi/util/dbutil/litestream"
-	"go.mau.fi/util/exzerolog"
-	"gopkg.in/yaml.v3"
-	flag "maunium.net/go/mauflag"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
-	"maunium.net/go/mautrix/bridge/status"
-	"maunium.net/go/mautrix/event"
-	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/sqlstatestore"
-)
-
-var configPath = flag.MakeFull("c", "config", "The path to your config file.", "config.yaml").String()
-var dontSaveConfig = flag.MakeFull("n", "no-update", "Don't save updated config to disk.", "false").Bool()
-var registrationPath = flag.MakeFull("r", "registration", "The path where to save the appservice registration.", "registration.yaml").String()
-var generateRegistration = flag.MakeFull("g", "generate-registration", "Generate registration and quit.", "false").Bool()
-var version = flag.MakeFull("v", "version", "View bridge version and quit.", "false").Bool()
-var versionJSON = flag.Make().LongKey("version-json").Usage("Print a JSON object representing the bridge version and quit.").Default("false").Bool()
-var ignoreUnsupportedDatabase = flag.Make().LongKey("ignore-unsupported-database").Usage("Run even if the database schema is too new").Default("false").Bool()
-var ignoreForeignTables = flag.Make().LongKey("ignore-foreign-tables").Usage("Run even if the database contains tables from other programs (like Synapse)").Default("false").Bool()
-var ignoreUnsupportedServer = flag.Make().LongKey("ignore-unsupported-server").Usage("Run even if the Matrix homeserver is outdated").Default("false").Bool()
-var wantHelp, _ = flag.MakeHelpFlag()
-
-var _ appservice.StateStore = (*sqlstatestore.SQLStateStore)(nil)
-
-type Portal interface {
-	IsEncrypted() bool
-	IsPrivateChat() bool
-	MarkEncrypted()
-	MainIntent() *appservice.IntentAPI
-
-	ReceiveMatrixEvent(user User, evt *event.Event)
-	UpdateBridgeInfo(ctx context.Context)
-}
-
-type MembershipHandlingPortal interface {
-	Portal
-	HandleMatrixLeave(sender User, evt *event.Event)
-	HandleMatrixKick(sender User, ghost Ghost, evt *event.Event)
-	HandleMatrixInvite(sender User, ghost Ghost, evt *event.Event)
-}
-
-type ReadReceiptHandlingPortal interface {
-	Portal
-	HandleMatrixReadReceipt(sender User, eventID id.EventID, receipt event.ReadReceipt)
-}
-
-type TypingPortal interface {
-	Portal
-	HandleMatrixTyping(userIDs []id.UserID)
-}
-
-type MetaHandlingPortal interface {
-	Portal
-	HandleMatrixMeta(sender User, evt *event.Event)
-}
-
-type DisappearingPortal interface {
-	Portal
-	ScheduleDisappearing()
-}
-
-type PowerLevelHandlingPortal interface {
-	Portal
-	HandleMatrixPowerLevels(sender User, evt *event.Event)
-}
-
-type JoinRuleHandlingPortal interface {
-	Portal
-	HandleMatrixJoinRule(sender User, evt *event.Event)
-}
-
-type BanHandlingPortal interface {
-	Portal
-	HandleMatrixBan(sender User, ghost Ghost, evt *event.Event)
-	HandleMatrixUnban(sender User, ghost Ghost, evt *event.Event)
-}
-
-type KnockHandlingPortal interface {
-	Portal
-	HandleMatrixKnock(sender User, evt *event.Event)
-	HandleMatrixRetractKnock(sender User, evt *event.Event)
-	HandleMatrixAcceptKnock(sender User, ghost Ghost, evt *event.Event)
-	HandleMatrixRejectKnock(sender User, ghost Ghost, evt *event.Event)
-}
-
-type InviteHandlingPortal interface {
-	Portal
-	HandleMatrixAcceptInvite(sender User, evt *event.Event)
-	HandleMatrixRejectInvite(sender User, evt *event.Event)
-	HandleMatrixRetractInvite(sender User, ghost Ghost, evt *event.Event)
-}
-
-type User interface {
-	GetPermissionLevel() bridgeconfig.PermissionLevel
-	IsLoggedIn() bool
-	GetManagementRoomID() id.RoomID
-	SetManagementRoom(id.RoomID)
-	GetMXID() id.UserID
-	GetIDoublePuppet() DoublePuppet
-	GetIGhost() Ghost
-}
-
-type DoublePuppet interface {
-	CustomIntent() *appservice.IntentAPI
-	SwitchCustomMXID(accessToken string, userID id.UserID) error
-	ClearCustomMXID()
-}
-
-type Ghost interface {
-	DoublePuppet
-	DefaultIntent() *appservice.IntentAPI
-	GetMXID() id.UserID
-}
-
-type GhostWithProfile interface {
-	Ghost
-	GetDisplayname() string
-	GetAvatarURL() id.ContentURI
-}
-
-type ChildOverride interface {
-	GetExampleConfig() string
-	GetConfigPtr() interface{}
-
-	Init()
-	Start()
-	Stop()
-
-	GetIPortal(id.RoomID) Portal
-	GetAllIPortals() []Portal
-	GetIUser(id id.UserID, create bool) User
-	IsGhost(id.UserID) bool
-	GetIGhost(id.UserID) Ghost
-	CreatePrivatePortal(id.RoomID, User, Ghost)
-}
-
-type ConfigValidatingBridge interface {
-	ChildOverride
-	ValidateConfig() error
-}
-
-type FlagHandlingBridge interface {
-	ChildOverride
-	HandleFlags() bool
-}
-
-type PreInitableBridge interface {
-	ChildOverride
-	PreInit()
-}
-
-type WebsocketStartingBridge interface {
-	ChildOverride
-	OnWebsocketConnect()
-}
-
-type CSFeatureRequirer interface {
-	CheckFeatures(versions *mautrix.RespVersions) (string, bool)
-}
-
-type Bridge struct {
-	Name              string
-	URL               string
-	Description       string
-	Version           string
-	ProtocolName      string
-	BeeperServiceName string
-	BeeperNetworkName string
-
-	AdditionalShortFlags string
-	AdditionalLongFlags  string
-
-	VersionDesc      string
-	LinkifiedVersion string
-	BuildTime        string
-	commit           string
-	baseVersion      string
-
-	PublicHSAddress *url.URL
-
-	DoublePuppet *doublePuppetUtil
-
-	AS               *appservice.AppService
-	EventProcessor   *appservice.EventProcessor
-	CommandProcessor CommandProcessor
-	MatrixHandler    *MatrixHandler
-	Bot              *appservice.IntentAPI
-	Config           bridgeconfig.BaseConfig
-	ConfigPath       string
-	RegistrationPath string
-	SaveConfig       bool
-	ConfigUpgrader   configupgrade.BaseUpgrader
-	DB               *dbutil.Database
-	StateStore       *sqlstatestore.SQLStateStore
-	Crypto           Crypto
-	CryptoPickleKey  string
-
-	ZLog *zerolog.Logger
-
-	MediaConfig  mautrix.RespMediaConfig
-	SpecVersions mautrix.RespVersions
-
-	Child ChildOverride
-
-	manualStop chan int
-	Stopping   bool
-
-	latestState *status.BridgeState
-
-	Websocket                      bool
-	wsStopPinger                   chan struct{}
-	wsStarted                      chan struct{}
-	wsStopped                      chan struct{}
-	wsShortCircuitReconnectBackoff chan struct{}
-	wsStartupWait                  *sync.WaitGroup
-}
-
-type Crypto interface {
-	HandleMemberEvent(context.Context, *event.Event)
-	Decrypt(context.Context, *event.Event) (*event.Event, error)
-	Encrypt(context.Context, id.RoomID, event.Type, *event.Content) error
-	WaitForSession(context.Context, id.RoomID, id.SenderKey, id.SessionID, time.Duration) bool
-	RequestSession(context.Context, id.RoomID, id.SenderKey, id.SessionID, id.UserID, id.DeviceID)
-	ResetSession(context.Context, id.RoomID)
-	Init(ctx context.Context) error
-	Start()
-	Stop()
-	Reset(ctx context.Context, startAfterReset bool)
-	Client() *mautrix.Client
-	ShareKeys(context.Context) error
-}
-
-func (br *Bridge) GenerateRegistration() {
-	if !br.SaveConfig {
-		// We need to save the generated as_token and hs_token in the config
-		_, _ = fmt.Fprintln(os.Stderr, "--no-update is not compatible with --generate-registration")
-		os.Exit(5)
-	} else if br.Config.Homeserver.Domain == "example.com" {
-		_, _ = fmt.Fprintln(os.Stderr, "Homeserver domain is not set")
-		os.Exit(20)
-	}
-	reg := br.Config.GenerateRegistration()
-	err := reg.Save(br.RegistrationPath)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to save registration:", err)
-		os.Exit(21)
-	}
-
-	updateTokens := func(helper configupgrade.Helper) {
-		helper.Set(configupgrade.Str, reg.AppToken, "appservice", "as_token")
-		helper.Set(configupgrade.Str, reg.ServerToken, "appservice", "hs_token")
-	}
-	_, _, err = configupgrade.Do(br.ConfigPath, true, br.ConfigUpgrader, configupgrade.SimpleUpgrader(updateTokens))
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to save config:", err)
-		os.Exit(22)
-	}
-	fmt.Println("Registration generated. See https://docs.mau.fi/bridges/general/registering-appservices.html for instructions on installing the registration.")
-	os.Exit(0)
-}
-
-func (br *Bridge) InitVersion(tag, commit, buildTime string) {
-	br.baseVersion = br.Version
-	if len(tag) > 0 && tag[0] == 'v' {
-		tag = tag[1:]
-	}
-	if tag != br.Version {
-		suffix := ""
-		if !strings.HasSuffix(br.Version, "+dev") {
-			suffix = "+dev"
-		}
-		if len(commit) > 8 {
-			br.Version = fmt.Sprintf("%s%s.%s", br.Version, suffix, commit[:8])
-		} else {
-			br.Version = fmt.Sprintf("%s%s.unknown", br.Version, suffix)
-		}
-	}
-
-	br.LinkifiedVersion = fmt.Sprintf("v%s", br.Version)
-	if tag == br.Version {
-		br.LinkifiedVersion = fmt.Sprintf("[v%s](%s/releases/v%s)", br.Version, br.URL, tag)
-	} else if len(commit) > 8 {
-		br.LinkifiedVersion = strings.Replace(br.LinkifiedVersion, commit[:8], fmt.Sprintf("[%s](%s/commit/%s)", commit[:8], br.URL, commit), 1)
-	}
-	mautrix.DefaultUserAgent = fmt.Sprintf("%s/%s %s", br.Name, br.Version, mautrix.DefaultUserAgent)
-	br.VersionDesc = fmt.Sprintf("%s %s (%s with %s)", br.Name, br.Version, buildTime, runtime.Version())
-	br.commit = commit
-	br.BuildTime = buildTime
-}
-
-var MinSpecVersion = mautrix.SpecV14
-
-func (br *Bridge) ensureConnection(ctx context.Context) {
-	for {
-		versions, err := br.Bot.Versions(ctx)
-		if err != nil {
-			if errors.Is(err, mautrix.MForbidden) {
-				br.ZLog.Debug().Msg("M_FORBIDDEN in /versions, trying to register before retrying")
-				err = br.Bot.EnsureRegistered(ctx)
-				if err != nil {
-					br.ZLog.Err(err).Msg("Failed to register after /versions failed")
-				}
-			} else {
-				br.ZLog.Err(err).Msg("Failed to connect to homeserver, retrying in 10 seconds...")
-				time.Sleep(10 * time.Second)
-			}
-		} else {
-			br.SpecVersions = *versions
-			*br.AS.SpecVersions = *versions
-			break
-		}
-	}
-
-	unsupportedServerLogLevel := zerolog.FatalLevel
-	if *ignoreUnsupportedServer {
-		unsupportedServerLogLevel = zerolog.ErrorLevel
-	}
-	if br.Config.Homeserver.Software == bridgeconfig.SoftwareHungry && !br.SpecVersions.Supports(mautrix.BeeperFeatureHungry) {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Msg("The config claims the homeserver is hungryserv, but the /versions response didn't confirm it")
-		os.Exit(18)
-	} else if !br.SpecVersions.ContainsGreaterOrEqual(MinSpecVersion) {
-		br.ZLog.WithLevel(unsupportedServerLogLevel).
-			Stringer("server_supports", br.SpecVersions.GetLatest()).
-			Stringer("bridge_requires", MinSpecVersion).
-			Msg("The homeserver is outdated (supported spec versions are below minimum required by bridge)")
-		if !*ignoreUnsupportedServer {
-			os.Exit(18)
-		}
-	} else if fr, ok := br.Child.(CSFeatureRequirer); ok {
-		if msg, hasFeatures := fr.CheckFeatures(&br.SpecVersions); !hasFeatures {
-			br.ZLog.WithLevel(unsupportedServerLogLevel).Msg(msg)
-			if !*ignoreUnsupportedServer {
-				os.Exit(18)
-			}
-		}
-	}
-
-	resp, err := br.Bot.Whoami(ctx)
-	if err != nil {
-		if errors.Is(err, mautrix.MUnknownToken) {
-			br.ZLog.WithLevel(zerolog.FatalLevel).Msg("The as_token was not accepted. Is the registration file installed in your homeserver correctly?")
-			br.ZLog.Info().Msg("See https://docs.mau.fi/faq/as-token for more info")
-		} else if errors.Is(err, mautrix.MExclusive) {
-			br.ZLog.WithLevel(zerolog.FatalLevel).Msg("The as_token was accepted, but the /register request was not. Are the homeserver domain, bot username and username template in the config correct, and do they match the values in the registration?")
-			br.ZLog.Info().Msg("See https://docs.mau.fi/faq/as-register for more info")
-		} else {
-			br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).Msg("/whoami request failed with unknown error")
-		}
-		os.Exit(16)
-	} else if resp.UserID != br.Bot.UserID {
-		br.ZLog.WithLevel(zerolog.FatalLevel).
-			Stringer("got_user_id", resp.UserID).
-			Stringer("expected_user_id", br.Bot.UserID).
-			Msg("Unexpected user ID in whoami call")
-		os.Exit(17)
-	}
-
-	if br.Websocket {
-		br.ZLog.Debug().Msg("Websocket mode: no need to check status of homeserver -> bridge connection")
-		return
-	} else if !br.SpecVersions.Supports(mautrix.FeatureAppservicePing) {
-		br.ZLog.Debug().Msg("Homeserver does not support checking status of homeserver -> bridge connection")
-		return
-	}
-	var pingResp *mautrix.RespAppservicePing
-	var txnID string
-	var retryCount int
-	const maxRetries = 6
-	for {
-		txnID = br.Bot.TxnID()
-		pingResp, err = br.Bot.AppservicePing(ctx, br.Config.AppService.ID, txnID)
-		if err == nil {
-			break
-		}
-		var httpErr mautrix.HTTPError
-		var pingErrBody string
-		if errors.As(err, &httpErr) && httpErr.RespError != nil {
-			if val, ok := httpErr.RespError.ExtraData["body"].(string); ok {
-				pingErrBody = strings.TrimSpace(val)
-			}
-		}
-		outOfRetries := retryCount >= maxRetries
-		level := zerolog.ErrorLevel
-		if outOfRetries {
-			level = zerolog.FatalLevel
-		}
-		evt := br.ZLog.WithLevel(level).Err(err).Str("txn_id", txnID)
-		if pingErrBody != "" {
-			bodyBytes := []byte(pingErrBody)
-			if json.Valid(bodyBytes) {
-				evt.RawJSON("body", bodyBytes)
-			} else {
-				evt.Str("body", pingErrBody)
-			}
-		}
-		if outOfRetries {
-			evt.Msg("Homeserver -> bridge connection is not working")
-			br.ZLog.Info().Msg("See https://docs.mau.fi/faq/as-ping for more info")
-			os.Exit(13)
-		}
-		evt.Msg("Homeserver -> bridge connection is not working, retrying in 5 seconds...")
-		time.Sleep(5 * time.Second)
-		retryCount++
-	}
-	br.ZLog.Debug().
-		Str("txn_id", txnID).
-		Int64("duration_ms", pingResp.DurationMS).
-		Msg("Homeserver -> bridge connection works")
-}
-
-func (br *Bridge) fetchMediaConfig(ctx context.Context) {
-	cfg, err := br.Bot.GetMediaConfig(ctx)
-	if err != nil {
-		br.ZLog.Warn().Err(err).Msg("Failed to fetch media config")
-	} else {
-		if cfg.UploadSize == 0 {
-			cfg.UploadSize = 50 * 1024 * 1024
-		}
-		br.MediaConfig = *cfg
-	}
-}
-
-func (br *Bridge) UpdateBotProfile(ctx context.Context) {
-	br.ZLog.Debug().Msg("Updating bot profile")
-	botConfig := &br.Config.AppService.Bot
-
-	var err error
-	var mxc id.ContentURI
-	if botConfig.Avatar == "remove" {
-		err = br.Bot.SetAvatarURL(ctx, mxc)
-	} else if !botConfig.ParsedAvatar.IsEmpty() {
-		err = br.Bot.SetAvatarURL(ctx, botConfig.ParsedAvatar)
-	}
-	if err != nil {
-		br.ZLog.Warn().Err(err).Msg("Failed to update bot avatar")
-	}
-
-	if botConfig.Displayname == "remove" {
-		err = br.Bot.SetDisplayName(ctx, "")
-	} else if len(botConfig.Displayname) > 0 {
-		err = br.Bot.SetDisplayName(ctx, botConfig.Displayname)
-	}
-	if err != nil {
-		br.ZLog.Warn().Err(err).Msg("Failed to update bot displayname")
-	}
-
-	if br.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) && br.BeeperNetworkName != "" {
-		br.ZLog.Debug().Msg("Setting contact info on the appservice bot")
-		br.Bot.BeeperUpdateProfile(ctx, map[string]any{
-			"com.beeper.bridge.service":       br.BeeperServiceName,
-			"com.beeper.bridge.network":       br.BeeperNetworkName,
-			"com.beeper.bridge.is_bridge_bot": true,
-		})
-	}
-}
-
-func (br *Bridge) loadConfig() {
-	configData, upgraded, err := configupgrade.Do(br.ConfigPath, br.SaveConfig, br.ConfigUpgrader)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Error updating config:", err)
-		if configData == nil {
-			os.Exit(10)
-		}
-	}
-
-	target := br.Child.GetConfigPtr()
-	if !upgraded {
-		// Fallback: if config upgrading failed, load example config for base values
-		err = yaml.Unmarshal([]byte(br.Child.GetExampleConfig()), &target)
-		if err != nil {
-			_, _ = fmt.Fprintln(os.Stderr, "Failed to unmarshal example config:", err)
-			os.Exit(10)
-		}
-	}
-	err = yaml.Unmarshal(configData, target)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to parse config:", err)
-		os.Exit(10)
-	}
-}
-
-func (br *Bridge) validateConfig() error {
-	switch {
-	case br.Config.Homeserver.Address == "https://matrix.example.com":
-		return errors.New("homeserver.address not configured")
-	case br.Config.Homeserver.Domain == "example.com":
-		return errors.New("homeserver.domain not configured")
-	case !bridgeconfig.AllowedHomeserverSoftware[br.Config.Homeserver.Software]:
-		return errors.New("invalid value for homeserver.software (use `standard` if you don't know what the field is for)")
-	case br.Config.AppService.ASToken == "This value is generated when generating the registration":
-		return errors.New("appservice.as_token not configured. Did you forget to generate the registration? ")
-	case br.Config.AppService.HSToken == "This value is generated when generating the registration":
-		return errors.New("appservice.hs_token not configured. Did you forget to generate the registration? ")
-	case br.Config.AppService.Database.URI == "postgres://user:password@host/database?sslmode=disable":
-		return errors.New("appservice.database not configured")
-	default:
-		err := br.Config.Bridge.Validate()
-		if err != nil {
-			return err
-		}
-		validator, ok := br.Child.(ConfigValidatingBridge)
-		if ok {
-			return validator.ValidateConfig()
-		}
-		return nil
-	}
-}
-
-func (br *Bridge) getProfile(userID id.UserID, roomID id.RoomID) *event.MemberEventContent {
-	ghost := br.Child.GetIGhost(userID)
-	if ghost == nil {
-		return nil
-	}
-	profilefulGhost, ok := ghost.(GhostWithProfile)
-	if ok {
-		return &event.MemberEventContent{
-			Displayname: profilefulGhost.GetDisplayname(),
-			AvatarURL:   profilefulGhost.GetAvatarURL().CUString(),
-		}
-	}
-	return nil
-}
-
-func (br *Bridge) init() {
-	pib, ok := br.Child.(PreInitableBridge)
-	if ok {
-		pib.PreInit()
-	}
-
-	var err error
-
-	br.MediaConfig.UploadSize = 50 * 1024 * 1024
-
-	br.ZLog, err = br.Config.Logging.Compile()
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to initialize logger:", err)
-		os.Exit(12)
-	}
-	exzerolog.SetupDefaults(br.ZLog)
-
-	br.DoublePuppet = &doublePuppetUtil{br: br, log: br.ZLog.With().Str("component", "double puppet").Logger()}
-
-	err = br.validateConfig()
-	if err != nil {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).Msg("Configuration error")
-		br.ZLog.Info().Msg("See https://docs.mau.fi/faq/field-unconfigured for more info")
-		os.Exit(11)
-	}
-
-	br.ZLog.Info().
-		Str("name", br.Name).
-		Str("version", br.Version).
-		Str("built_at", br.BuildTime).
-		Str("go_version", runtime.Version()).
-		Msg("Initializing bridge")
-
-	br.ZLog.Debug().Msg("Initializing database connection")
-	dbConfig := br.Config.AppService.Database
-	if (dbConfig.Type == "sqlite3-fk-wal" || dbConfig.Type == "litestream") && dbConfig.MaxOpenConns != 1 && !strings.Contains(dbConfig.URI, "_txlock=immediate") {
-		var fixedExampleURI string
-		if !strings.HasPrefix(dbConfig.URI, "file:") {
-			fixedExampleURI = fmt.Sprintf("file:%s?_txlock=immediate", dbConfig.URI)
-		} else if !strings.ContainsRune(dbConfig.URI, '?') {
-			fixedExampleURI = fmt.Sprintf("%s?_txlock=immediate", dbConfig.URI)
-		} else {
-			fixedExampleURI = fmt.Sprintf("%s&_txlock=immediate", dbConfig.URI)
-		}
-		br.ZLog.Warn().
-			Str("fixed_uri_example", fixedExampleURI).
-			Msg("Using SQLite without _txlock=immediate is not recommended")
-	}
-	br.DB, err = dbutil.NewFromConfig(br.Name, dbConfig, dbutil.ZeroLogger(br.ZLog.With().Str("db_section", "main").Logger()))
-	if err != nil {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to initialize database connection")
-		if sqlError := (&sqlite3.Error{}); errors.As(err, sqlError) && sqlError.Code == sqlite3.ErrCorrupt {
-			os.Exit(18)
-		}
-		os.Exit(14)
-	}
-	br.DB.IgnoreUnsupportedDatabase = *ignoreUnsupportedDatabase
-	br.DB.IgnoreForeignTables = *ignoreForeignTables
-
-	br.ZLog.Debug().Msg("Initializing state store")
-	br.StateStore = sqlstatestore.NewSQLStateStore(br.DB, dbutil.ZeroLogger(br.ZLog.With().Str("db_section", "matrix_state").Logger()), true)
-
-	br.AS, err = appservice.CreateFull(appservice.CreateOpts{
-		Registration:     br.Config.AppService.GetRegistration(),
-		HomeserverDomain: br.Config.Homeserver.Domain,
-		HomeserverURL:    br.Config.Homeserver.Address,
-		HostConfig: appservice.HostConfig{
-			Hostname: br.Config.AppService.Hostname,
-			Port:     br.Config.AppService.Port,
-		},
-		StateStore: br.StateStore,
-	})
-	if err != nil {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).
-			Msg("Failed to initialize appservice")
-		os.Exit(15)
-	}
-	br.AS.Log = *br.ZLog
-	br.AS.DoublePuppetValue = br.Name
-	br.AS.GetProfile = br.getProfile
-	br.Bot = br.AS.BotIntent()
-
-	br.ZLog.Debug().Msg("Initializing Matrix event processor")
-	br.EventProcessor = appservice.NewEventProcessor(br.AS)
-	if !br.Config.AppService.AsyncTransactions {
-		br.EventProcessor.ExecMode = appservice.Sync
-	}
-	br.ZLog.Debug().Msg("Initializing Matrix event handler")
-	br.MatrixHandler = NewMatrixHandler(br)
-
-	br.Crypto = NewCryptoHelper(br)
-
-	hsURL := br.Config.Homeserver.Address
-	if br.Config.Homeserver.PublicAddress != "" {
-		hsURL = br.Config.Homeserver.PublicAddress
-	}
-	br.PublicHSAddress, err = url.Parse(hsURL)
-	if err != nil {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).
-			Str("input", hsURL).
-			Msg("Failed to parse public homeserver URL")
-		os.Exit(15)
-	}
-
-	br.Child.Init()
-}
-
-type zerologPQError pq.Error
-
-func (zpe *zerologPQError) MarshalZerologObject(evt *zerolog.Event) {
-	maybeStr := func(field, value string) {
-		if value != "" {
-			evt.Str(field, value)
-		}
-	}
-	maybeStr("severity", zpe.Severity)
-	if name := zpe.Code.Name(); name != "" {
-		evt.Str("code", name)
-	} else if zpe.Code != "" {
-		evt.Str("code", string(zpe.Code))
-	}
-	//maybeStr("message", zpe.Message)
-	maybeStr("detail", zpe.Detail)
-	maybeStr("hint", zpe.Hint)
-	maybeStr("position", zpe.Position)
-	maybeStr("internal_position", zpe.InternalPosition)
-	maybeStr("internal_query", zpe.InternalQuery)
-	maybeStr("where", zpe.Where)
-	maybeStr("schema", zpe.Schema)
-	maybeStr("table", zpe.Table)
-	maybeStr("column", zpe.Column)
-	maybeStr("data_type_name", zpe.DataTypeName)
-	maybeStr("constraint", zpe.Constraint)
-	maybeStr("file", zpe.File)
-	maybeStr("line", zpe.Line)
-	maybeStr("routine", zpe.Routine)
-}
-
-func (br *Bridge) LogDBUpgradeErrorAndExit(name string, err error) {
-	logEvt := br.ZLog.WithLevel(zerolog.FatalLevel).
-		Err(err).
-		Str("db_section", name)
-	var errWithLine *dbutil.PQErrorWithLine
-	if errors.As(err, &errWithLine) {
-		logEvt.Str("sql_line", errWithLine.Line)
-	}
-	var pqe *pq.Error
-	if errors.As(err, &pqe) {
-		logEvt.Object("pq_error", (*zerologPQError)(pqe))
-	}
-	logEvt.Msg("Failed to initialize database")
-	if sqlError := (&sqlite3.Error{}); errors.As(err, sqlError) && sqlError.Code == sqlite3.ErrCorrupt {
-		os.Exit(18)
-	} else if errors.Is(err, dbutil.ErrForeignTables) {
-		br.ZLog.Info().Msg("You can use --ignore-foreign-tables to ignore this error")
-		br.ZLog.Info().Msg("See https://docs.mau.fi/faq/foreign-tables for more info")
-	} else if errors.Is(err, dbutil.ErrNotOwned) {
-		br.ZLog.Info().Msg("Sharing the same database with different programs is not supported")
-	} else if errors.Is(err, dbutil.ErrUnsupportedDatabaseVersion) {
-		br.ZLog.Info().Msg("Downgrading the bridge is not supported")
-	}
-	os.Exit(15)
-}
-
-func (br *Bridge) WaitWebsocketConnected() {
-	if br.wsStartupWait != nil {
-		br.wsStartupWait.Wait()
-	}
-}
-
-func (br *Bridge) start() {
-	br.ZLog.Debug().Msg("Running database upgrades")
-	err := br.DB.Upgrade(br.ZLog.With().Str("db_section", "main").Logger().WithContext(context.TODO()))
-	if err != nil {
-		br.LogDBUpgradeErrorAndExit("main", err)
-	} else if err = br.StateStore.Upgrade(br.ZLog.With().Str("db_section", "matrix_state").Logger().WithContext(context.TODO())); err != nil {
-		br.LogDBUpgradeErrorAndExit("matrix_state", err)
-	}
-
-	if br.Config.Homeserver.Websocket || len(br.Config.Homeserver.WSProxy) > 0 {
-		br.Websocket = true
-		br.ZLog.Debug().Msg("Starting application service websocket")
-		var wg sync.WaitGroup
-		wg.Add(1)
-		br.wsStartupWait = &wg
-		br.wsShortCircuitReconnectBackoff = make(chan struct{})
-		go br.startWebsocket(&wg)
-	} else if br.AS.Host.IsConfigured() {
-		br.ZLog.Debug().Msg("Starting application service HTTP server")
-		go br.AS.Start()
-	} else {
-		br.ZLog.WithLevel(zerolog.FatalLevel).Msg("Neither appservice HTTP listener nor websocket is enabled")
-		os.Exit(23)
-	}
-	br.ZLog.Debug().Msg("Checking connection to homeserver")
-
-	ctx := br.ZLog.WithContext(context.Background())
-	br.ensureConnection(ctx)
-	go br.fetchMediaConfig(ctx)
-
-	if br.Crypto != nil {
-		err = br.Crypto.Init(ctx)
-		if err != nil {
-			br.ZLog.WithLevel(zerolog.FatalLevel).Err(err).Msg("Error initializing end-to-bridge encryption")
-			os.Exit(19)
-		}
-	}
-
-	br.ZLog.Debug().Msg("Starting event processor")
-	br.EventProcessor.Start(ctx)
-
-	go br.UpdateBotProfile(ctx)
-	if br.Crypto != nil {
-		go br.Crypto.Start()
-	}
-
-	br.Child.Start()
-	br.WaitWebsocketConnected()
-	br.AS.Ready = true
-
-	if br.Config.Bridge.GetResendBridgeInfo() {
-		go br.ResendBridgeInfo()
-	}
-	if br.Websocket && br.Config.Homeserver.WSPingInterval > 0 {
-		br.wsStopPinger = make(chan struct{}, 1)
-		go br.websocketServerPinger()
-	}
-}
-
-func (br *Bridge) ResendBridgeInfo() {
-	if !br.SaveConfig {
-		br.ZLog.Warn().Msg("Not setting resend_bridge_info to false in config due to --no-update flag")
-	} else {
-		_, _, err := configupgrade.Do(br.ConfigPath, true, br.ConfigUpgrader, configupgrade.SimpleUpgrader(func(helper configupgrade.Helper) {
-			helper.Set(configupgrade.Bool, "false", "bridge", "resend_bridge_info")
-		}))
-		if err != nil {
-			br.ZLog.Err(err).Msg("Failed to save config after setting resend_bridge_info to false")
-		}
-	}
-	br.ZLog.Info().Msg("Re-sending bridge info state event to all portals")
-	for _, portal := range br.Child.GetAllIPortals() {
-		portal.UpdateBridgeInfo(context.TODO())
-	}
-	br.ZLog.Info().Msg("Finished re-sending bridge info state events")
-}
-
-func sendStopSignal(ch chan struct{}) {
-	if ch != nil {
-		select {
-		case ch <- struct{}{}:
-		default:
-		}
-	}
-}
-
-func (br *Bridge) stop() {
-	br.Stopping = true
-	if br.Crypto != nil {
-		br.Crypto.Stop()
-	}
-	waitForWS := false
-	if br.AS.StopWebsocket != nil {
-		br.ZLog.Debug().Msg("Stopping application service websocket")
-		br.AS.StopWebsocket(appservice.ErrWebsocketManualStop)
-		waitForWS = true
-	}
-	br.AS.Stop()
-	sendStopSignal(br.wsStopPinger)
-	sendStopSignal(br.wsShortCircuitReconnectBackoff)
-	br.EventProcessor.Stop()
-	br.Child.Stop()
-	err := br.DB.Close()
-	if err != nil {
-		br.ZLog.Warn().Err(err).Msg("Error closing database")
-	}
-	if waitForWS {
-		select {
-		case <-br.wsStopped:
-		case <-time.After(4 * time.Second):
-			br.ZLog.Warn().Msg("Timed out waiting for websocket to close")
-		}
-	}
-}
-
-func (br *Bridge) ManualStop(exitCode int) {
-	if br.manualStop != nil {
-		br.manualStop <- exitCode
-	} else {
-		os.Exit(exitCode)
-	}
-}
-
-type VersionJSONOutput struct {
-	Name string
-	URL  string
-
-	Version          string
-	IsRelease        bool
-	Commit           string
-	FormattedVersion string
-	BuildTime        string
-
-	OS   string
-	Arch string
-
-	Mautrix struct {
-		Version string
-		Commit  string
-	}
-}
-
-func (br *Bridge) Main() {
-	flag.SetHelpTitles(
-		fmt.Sprintf("%s - %s", br.Name, br.Description),
-		fmt.Sprintf("%s [-hgvn%s] [-c <path>] [-r <path>]%s", br.Name, br.AdditionalShortFlags, br.AdditionalLongFlags))
-	err := flag.Parse()
-	br.ConfigPath = *configPath
-	br.RegistrationPath = *registrationPath
-	br.SaveConfig = !*dontSaveConfig
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, err)
-		flag.PrintHelp()
-		os.Exit(1)
-	} else if *wantHelp {
-		flag.PrintHelp()
-		os.Exit(0)
-	} else if *version {
-		fmt.Println(br.VersionDesc)
-		return
-	} else if *versionJSON {
-		output := VersionJSONOutput{
-			URL:  br.URL,
-			Name: br.Name,
-
-			Version:          br.baseVersion,
-			IsRelease:        br.Version == br.baseVersion,
-			Commit:           br.commit,
-			FormattedVersion: br.Version,
-			BuildTime:        br.BuildTime,
-
-			OS:   runtime.GOOS,
-			Arch: runtime.GOARCH,
-		}
-		output.Mautrix.Commit = mautrix.Commit
-		output.Mautrix.Version = mautrix.Version
-		_ = json.NewEncoder(os.Stdout).Encode(output)
-		return
-	} else if flagHandler, ok := br.Child.(FlagHandlingBridge); ok && flagHandler.HandleFlags() {
-		return
-	}
-
-	br.loadConfig()
-
-	if *generateRegistration {
-		br.GenerateRegistration()
-		return
-	}
-
-	br.manualStop = make(chan int, 1)
-	br.init()
-	br.ZLog.Info().Msg("Bridge initialization complete, starting...")
-	br.start()
-	br.ZLog.Info().Msg("Bridge started!")
-
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-	var exitCode int
-	select {
-	case <-c:
-		br.ZLog.Info().Msg("Interrupt received, stopping...")
-	case exitCode = <-br.manualStop:
-		br.ZLog.Info().Int("exit_code", exitCode).Msg("Manual stop requested")
-	}
-
-	br.stop()
-	br.ZLog.Info().Msg("Bridge stopped.")
-	os.Exit(exitCode)
-}

bridge/bridgeconfig/config.go

@@ -1,337 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridgeconfig
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-
-	"github.com/rs/zerolog"
-	up "go.mau.fi/util/configupgrade"
-	"go.mau.fi/util/dbutil"
-	"go.mau.fi/util/random"
-	"go.mau.fi/zeroconfig"
-	"gopkg.in/yaml.v3"
-
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/id"
-)
-
-type HomeserverSoftware string
-
-const (
-	SoftwareStandard HomeserverSoftware = "standard"
-	SoftwareAsmux    HomeserverSoftware = "asmux"
-	SoftwareHungry   HomeserverSoftware = "hungry"
-)
-
-var AllowedHomeserverSoftware = map[HomeserverSoftware]bool{
-	SoftwareStandard: true,
-	SoftwareAsmux:    true,
-	SoftwareHungry:   true,
-}
-
-type HomeserverConfig struct {
-	Address    string `yaml:"address"`
-	Domain     string `yaml:"domain"`
-	AsyncMedia bool   `yaml:"async_media"`
-
-	PublicAddress string `yaml:"public_address,omitempty"`
-
-	Software HomeserverSoftware `yaml:"software"`
-
-	StatusEndpoint                string `yaml:"status_endpoint"`
-	MessageSendCheckpointEndpoint string `yaml:"message_send_checkpoint_endpoint"`
-
-	Websocket      bool   `yaml:"websocket"`
-	WSProxy        string `yaml:"websocket_proxy"`
-	WSPingInterval int    `yaml:"ping_interval_seconds"`
-}
-
-type AppserviceConfig struct {
-	Address  string `yaml:"address"`
-	Hostname string `yaml:"hostname"`
-	Port     uint16 `yaml:"port"`
-
-	Database dbutil.Config `yaml:"database"`
-
-	ID  string        `yaml:"id"`
-	Bot BotUserConfig `yaml:"bot"`
-
-	ASToken string `yaml:"as_token"`
-	HSToken string `yaml:"hs_token"`
-
-	EphemeralEvents   bool `yaml:"ephemeral_events"`
-	AsyncTransactions bool `yaml:"async_transactions"`
-}
-
-func (config *BaseConfig) MakeUserIDRegex(matcher string) *regexp.Regexp {
-	usernamePlaceholder := strings.ToLower(random.String(16))
-	usernameTemplate := fmt.Sprintf("@%s:%s",
-		config.Bridge.FormatUsername(usernamePlaceholder),
-		config.Homeserver.Domain)
-	usernameTemplate = regexp.QuoteMeta(usernameTemplate)
-	usernameTemplate = strings.Replace(usernameTemplate, usernamePlaceholder, matcher, 1)
-	usernameTemplate = fmt.Sprintf("^%s$", usernameTemplate)
-	return regexp.MustCompile(usernameTemplate)
-}
-
-// GenerateRegistration generates a registration file for the homeserver.
-func (config *BaseConfig) GenerateRegistration() *appservice.Registration {
-	registration := appservice.CreateRegistration()
-	config.AppService.HSToken = registration.ServerToken
-	config.AppService.ASToken = registration.AppToken
-	config.AppService.copyToRegistration(registration)
-
-	registration.SenderLocalpart = random.String(32)
-	botRegex := regexp.MustCompile(fmt.Sprintf("^@%s:%s$",
-		regexp.QuoteMeta(config.AppService.Bot.Username),
-		regexp.QuoteMeta(config.Homeserver.Domain)))
-	registration.Namespaces.UserIDs.Register(botRegex, true)
-	registration.Namespaces.UserIDs.Register(config.MakeUserIDRegex(".*"), true)
-
-	return registration
-}
-
-func (config *BaseConfig) MakeAppService() *appservice.AppService {
-	as := appservice.Create()
-	as.HomeserverDomain = config.Homeserver.Domain
-	_ = as.SetHomeserverURL(config.Homeserver.Address)
-	as.Host.Hostname = config.AppService.Hostname
-	as.Host.Port = config.AppService.Port
-	as.Registration = config.AppService.GetRegistration()
-	return as
-}
-
-// GetRegistration copies the data from the bridge config into an *appservice.Registration struct.
-// This can't be used with the homeserver, see GenerateRegistration for generating files for the homeserver.
-func (asc *AppserviceConfig) GetRegistration() *appservice.Registration {
-	reg := &appservice.Registration{}
-	asc.copyToRegistration(reg)
-	reg.SenderLocalpart = asc.Bot.Username
-	reg.ServerToken = asc.HSToken
-	reg.AppToken = asc.ASToken
-	return reg
-}
-
-func (asc *AppserviceConfig) copyToRegistration(registration *appservice.Registration) {
-	registration.ID = asc.ID
-	registration.URL = asc.Address
-	falseVal := false
-	registration.RateLimited = &falseVal
-	registration.EphemeralEvents = asc.EphemeralEvents
-	registration.SoruEphemeralEvents = asc.EphemeralEvents
-}
-
-type BotUserConfig struct {
-	Username    string `yaml:"username"`
-	Displayname string `yaml:"displayname"`
-	Avatar      string `yaml:"avatar"`
-
-	ParsedAvatar id.ContentURI `yaml:"-"`
-}
-
-type serializableBUC BotUserConfig
-
-func (buc *BotUserConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var sbuc serializableBUC
-	err := unmarshal(&sbuc)
-	if err != nil {
-		return err
-	}
-	*buc = (BotUserConfig)(sbuc)
-	if buc.Avatar != "" && buc.Avatar != "remove" {
-		buc.ParsedAvatar, err = id.ParseContentURI(buc.Avatar)
-		if err != nil {
-			return fmt.Errorf("%w in bot avatar", err)
-		}
-	}
-	return nil
-}
-
-type BridgeConfig interface {
-	FormatUsername(username string) string
-	GetEncryptionConfig() EncryptionConfig
-	GetCommandPrefix() string
-	GetManagementRoomTexts() ManagementRoomTexts
-	GetDoublePuppetConfig() DoublePuppetConfig
-	GetResendBridgeInfo() bool
-	EnableMessageStatusEvents() bool
-	EnableMessageErrorNotices() bool
-	Validate() error
-}
-
-type DoublePuppetConfig struct {
-	ServerMap       map[string]string `yaml:"double_puppet_server_map"`
-	AllowDiscovery  bool              `yaml:"double_puppet_allow_discovery"`
-	SharedSecretMap map[string]string `yaml:"login_shared_secret_map"`
-}
-
-type EncryptionConfig struct {
-	Allow      bool `yaml:"allow"`
-	Default    bool `yaml:"default"`
-	Require    bool `yaml:"require"`
-	Appservice bool `yaml:"appservice"`
-
-	PlaintextMentions bool `yaml:"plaintext_mentions"`
-
-	DeleteKeys struct {
-		DeleteOutboundOnAck       bool `yaml:"delete_outbound_on_ack"`
-		DontStoreOutbound         bool `yaml:"dont_store_outbound"`
-		RatchetOnDecrypt          bool `yaml:"ratchet_on_decrypt"`
-		DeleteFullyUsedOnDecrypt  bool `yaml:"delete_fully_used_on_decrypt"`
-		DeletePrevOnNewSession    bool `yaml:"delete_prev_on_new_session"`
-		DeleteOnDeviceDelete      bool `yaml:"delete_on_device_delete"`
-		PeriodicallyDeleteExpired bool `yaml:"periodically_delete_expired"`
-		DeleteOutdatedInbound     bool `yaml:"delete_outdated_inbound"`
-	} `yaml:"delete_keys"`
-
-	VerificationLevels struct {
-		Receive id.TrustState `yaml:"receive"`
-		Send    id.TrustState `yaml:"send"`
-		Share   id.TrustState `yaml:"share"`
-	} `yaml:"verification_levels"`
-	AllowKeySharing bool `yaml:"allow_key_sharing"`
-
-	Rotation struct {
-		EnableCustom bool  `yaml:"enable_custom"`
-		Milliseconds int64 `yaml:"milliseconds"`
-		Messages     int   `yaml:"messages"`
-
-		DisableDeviceChangeKeyRotation bool `yaml:"disable_device_change_key_rotation"`
-	} `yaml:"rotation"`
-}
-
-type ManagementRoomTexts struct {
-	Welcome            string `yaml:"welcome"`
-	WelcomeConnected   string `yaml:"welcome_connected"`
-	WelcomeUnconnected string `yaml:"welcome_unconnected"`
-	AdditionalHelp     string `yaml:"additional_help"`
-}
-
-type BaseConfig struct {
-	Homeserver HomeserverConfig  `yaml:"homeserver"`
-	AppService AppserviceConfig  `yaml:"appservice"`
-	Bridge     BridgeConfig      `yaml:"-"`
-	Logging    zeroconfig.Config `yaml:"logging"`
-}
-
-func doUpgrade(helper up.Helper) {
-	helper.Copy(up.Str, "homeserver", "address")
-	helper.Copy(up.Str, "homeserver", "domain")
-	if legacyAsmuxFlag, ok := helper.Get(up.Bool, "homeserver", "asmux"); ok && legacyAsmuxFlag == "true" {
-		helper.Set(up.Str, string(SoftwareAsmux), "homeserver", "software")
-	} else {
-		helper.Copy(up.Str, "homeserver", "software")
-	}
-	helper.Copy(up.Str|up.Null, "homeserver", "status_endpoint")
-	helper.Copy(up.Str|up.Null, "homeserver", "message_send_checkpoint_endpoint")
-	helper.Copy(up.Bool, "homeserver", "async_media")
-	helper.Copy(up.Str|up.Null, "homeserver", "websocket_proxy")
-	helper.Copy(up.Bool, "homeserver", "websocket")
-	helper.Copy(up.Int, "homeserver", "ping_interval_seconds")
-
-	helper.Copy(up.Str|up.Null, "appservice", "address")
-	helper.Copy(up.Str|up.Null, "appservice", "hostname")
-	helper.Copy(up.Int|up.Null, "appservice", "port")
-	if dbType, ok := helper.Get(up.Str, "appservice", "database", "type"); ok && dbType == "sqlite3" {
-		helper.Set(up.Str, "sqlite3-fk-wal", "appservice", "database", "type")
-	} else {
-		helper.Copy(up.Str, "appservice", "database", "type")
-	}
-	helper.Copy(up.Str, "appservice", "database", "uri")
-	helper.Copy(up.Int, "appservice", "database", "max_open_conns")
-	helper.Copy(up.Int, "appservice", "database", "max_idle_conns")
-	helper.Copy(up.Str|up.Null, "appservice", "database", "max_conn_idle_time")
-	helper.Copy(up.Str|up.Null, "appservice", "database", "max_conn_lifetime")
-	helper.Copy(up.Str, "appservice", "id")
-	helper.Copy(up.Str, "appservice", "bot", "username")
-	helper.Copy(up.Str, "appservice", "bot", "displayname")
-	helper.Copy(up.Str, "appservice", "bot", "avatar")
-	helper.Copy(up.Bool, "appservice", "ephemeral_events")
-	helper.Copy(up.Bool, "appservice", "async_transactions")
-	helper.Copy(up.Str, "appservice", "as_token")
-	helper.Copy(up.Str, "appservice", "hs_token")
-
-	if helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "print_level") != nil || helper.GetNode("logging", "file_name_format") != nil) {
-		_, _ = fmt.Fprintln(os.Stderr, "Migrating legacy log config")
-		migrateLegacyLogConfig(helper)
-	} else if helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "handlers") != nil) {
-		_, _ = fmt.Fprintln(os.Stderr, "Migrating Python log config is not currently supported")
-		// TODO implement?
-		//migratePythonLogConfig(helper)
-	} else {
-		helper.Copy(up.Map, "logging")
-	}
-}
-
-type legacyLogConfig struct {
-	Directory       string `yaml:"directory"`
-	FileNameFormat  string `yaml:"file_name_format"`
-	FileDateFormat  string `yaml:"file_date_format"`
-	FileMode        uint32 `yaml:"file_mode"`
-	TimestampFormat string `yaml:"timestamp_format"`
-	RawPrintLevel   string `yaml:"print_level"`
-	JSONStdout      bool   `yaml:"print_json"`
-	JSONFile        bool   `yaml:"file_json"`
-}
-
-func migrateLegacyLogConfig(helper up.Helper) {
-	var llc legacyLogConfig
-	var newConfig zeroconfig.Config
-	err := helper.GetBaseNode("logging").Decode(&newConfig)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Base config is corrupted: failed to decode example log config:", err)
-		return
-	} else if len(newConfig.Writers) != 2 || newConfig.Writers[0].Type != "stdout" || newConfig.Writers[1].Type != "file" {
-		_, _ = fmt.Fprintln(os.Stderr, "Base log config is not in expected format")
-		return
-	}
-	err = helper.GetNode("logging").Decode(&llc)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to decode legacy log config:", err)
-		return
-	}
-	if llc.RawPrintLevel != "" {
-		level, err := zerolog.ParseLevel(llc.RawPrintLevel)
-		if err != nil {
-			_, _ = fmt.Fprintln(os.Stderr, "Failed to parse minimum stdout log level:", err)
-		} else {
-			newConfig.Writers[0].MinLevel = &level
-		}
-	}
-	if llc.Directory != "" && llc.FileNameFormat != "" {
-		if llc.FileNameFormat == "{{.Date}}-{{.Index}}.log" {
-			llc.FileNameFormat = "bridge.log"
-		} else {
-			llc.FileNameFormat = strings.ReplaceAll(llc.FileNameFormat, "{{.Date}}", "")
-			llc.FileNameFormat = strings.ReplaceAll(llc.FileNameFormat, "{{.Index}}", "")
-		}
-		newConfig.Writers[1].Filename = filepath.Join(llc.Directory, llc.FileNameFormat)
-	} else if llc.FileNameFormat == "" {
-		newConfig.Writers = newConfig.Writers[0:1]
-	}
-	if llc.JSONStdout {
-		newConfig.Writers[0].TimeFormat = ""
-		newConfig.Writers[0].Format = "json"
-	} else if llc.TimestampFormat != "" {
-		newConfig.Writers[0].TimeFormat = llc.TimestampFormat
-	}
-	var updatedConfig yaml.Node
-	err = updatedConfig.Encode(&newConfig)
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Failed to encode migrated log config:", err)
-		return
-	}
-	*helper.GetBaseNode("logging").Node = updatedConfig
-}
-
-// Upgrader is a config upgrader that copies the default fields in the homeserver, appservice and logging blocks.
-var Upgrader = up.SimpleUpgrader(doUpgrade)

bridge/bridgeconfig/permissions.go

@@ -1,71 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridgeconfig
-
-import (
-	"strconv"
-	"strings"
-
-	"maunium.net/go/mautrix/id"
-)
-
-type PermissionConfig map[string]PermissionLevel
-
-type PermissionLevel int
-
-const (
-	PermissionLevelBlock PermissionLevel = 0
-	PermissionLevelRelay PermissionLevel = 5
-	PermissionLevelUser  PermissionLevel = 10
-	PermissionLevelAdmin PermissionLevel = 100
-)
-
-var namesToLevels = map[string]PermissionLevel{
-	"block": PermissionLevelBlock,
-	"relay": PermissionLevelRelay,
-	"user":  PermissionLevelUser,
-	"admin": PermissionLevelAdmin,
-}
-
-func RegisterPermissionLevel(name string, level PermissionLevel) {
-	namesToLevels[name] = level
-}
-
-func (pc *PermissionConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	rawPC := make(map[string]string)
-	err := unmarshal(&rawPC)
-	if err != nil {
-		return err
-	}
-
-	if *pc == nil {
-		*pc = make(map[string]PermissionLevel)
-	}
-	for key, value := range rawPC {
-		level, ok := namesToLevels[strings.ToLower(value)]
-		if ok {
-			(*pc)[key] = level
-		} else if val, err := strconv.Atoi(value); err == nil {
-			(*pc)[key] = PermissionLevel(val)
-		} else {
-			(*pc)[key] = PermissionLevelBlock
-		}
-	}
-	return nil
-}
-
-func (pc PermissionConfig) Get(userID id.UserID) PermissionLevel {
-	if level, ok := pc[string(userID)]; ok {
-		return level
-	} else if level, ok = pc[userID.Homeserver()]; len(userID.Homeserver()) > 0 && ok {
-		return level
-	} else if level, ok = pc["*"]; ok {
-		return level
-	} else {
-		return PermissionLevelBlock
-	}
-}

bridge/bridgestate.go

@@ -1,156 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridge
-
-import (
-	"context"
-	"runtime/debug"
-	"time"
-
-	"github.com/rs/zerolog"
-
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/status"
-)
-
-func (br *Bridge) SendBridgeState(ctx context.Context, state *status.BridgeState) error {
-	if br.Websocket {
-		// FIXME this doesn't account for multiple users
-		br.latestState = state
-
-		return br.AS.SendWebsocket(&appservice.WebsocketRequest{
-			Command: "bridge_status",
-			Data:    state,
-		})
-	} else if br.Config.Homeserver.StatusEndpoint != "" {
-		return state.SendHTTP(ctx, br.Config.Homeserver.StatusEndpoint, br.Config.AppService.ASToken)
-	} else {
-		return nil
-	}
-}
-
-func (br *Bridge) SendGlobalBridgeState(state status.BridgeState) {
-	if len(br.Config.Homeserver.StatusEndpoint) == 0 && !br.Websocket {
-		return
-	}
-
-	for {
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		if err := br.SendBridgeState(ctx, &state); err != nil {
-			br.ZLog.Warn().Err(err).Msg("Failed to update global bridge state")
-			cancel()
-			time.Sleep(5 * time.Second)
-			continue
-		} else {
-			br.ZLog.Debug().Interface("bridge_state", state).Msg("Sent new global bridge state")
-			cancel()
-			break
-		}
-	}
-}
-
-type BridgeStateQueue struct {
-	prev   *status.BridgeState
-	ch     chan status.BridgeState
-	bridge *Bridge
-	user   status.BridgeStateFiller
-}
-
-func (br *Bridge) NewBridgeStateQueue(user status.BridgeStateFiller) *BridgeStateQueue {
-	if len(br.Config.Homeserver.StatusEndpoint) == 0 && !br.Websocket {
-		return nil
-	}
-	bsq := &BridgeStateQueue{
-		ch:     make(chan status.BridgeState, 10),
-		bridge: br,
-		user:   user,
-	}
-	go bsq.loop()
-	return bsq
-}
-
-func (bsq *BridgeStateQueue) loop() {
-	defer func() {
-		err := recover()
-		if err != nil {
-			bsq.bridge.ZLog.Error().
-				Str(zerolog.ErrorStackFieldName, string(debug.Stack())).
-				Interface(zerolog.ErrorFieldName, err).
-				Msg("Panic in bridge state loop")
-		}
-	}()
-	for state := range bsq.ch {
-		bsq.immediateSendBridgeState(state)
-	}
-}
-
-func (bsq *BridgeStateQueue) immediateSendBridgeState(state status.BridgeState) {
-	retryIn := 2
-	for {
-		if bsq.prev != nil && bsq.prev.ShouldDeduplicate(&state) {
-			bsq.bridge.ZLog.Debug().
-				Str("state_event", string(state.StateEvent)).
-				Msg("Not sending bridge state as it's a duplicate")
-			return
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		err := bsq.bridge.SendBridgeState(ctx, &state)
-		cancel()
-
-		if err != nil {
-			bsq.bridge.ZLog.Warn().Err(err).
-				Int("retry_in_seconds", retryIn).
-				Msg("Failed to update bridge state")
-			time.Sleep(time.Duration(retryIn) * time.Second)
-			retryIn *= 2
-			if retryIn > 64 {
-				retryIn = 64
-			}
-		} else {
-			bsq.prev = &state
-			bsq.bridge.ZLog.Debug().
-				Interface("bridge_state", state).
-				Msg("Sent new bridge state")
-			return
-		}
-	}
-}
-
-func (bsq *BridgeStateQueue) Send(state status.BridgeState) {
-	if bsq == nil {
-		return
-	}
-
-	state = state.Fill(bsq.user)
-
-	if len(bsq.ch) >= 8 {
-		bsq.bridge.ZLog.Warn().Msg("Bridge state queue is nearly full, discarding an item")
-		select {
-		case <-bsq.ch:
-		default:
-		}
-	}
-	select {
-	case bsq.ch <- state:
-	default:
-		bsq.bridge.ZLog.Error().Msg("Bridge state queue is full, dropped new state")
-	}
-}
-
-func (bsq *BridgeStateQueue) GetPrev() status.BridgeState {
-	if bsq != nil && bsq.prev != nil {
-		return *bsq.prev
-	}
-	return status.BridgeState{}
-}
-
-func (bsq *BridgeStateQueue) SetPrev(prev status.BridgeState) {
-	if bsq != nil {
-		bsq.prev = &prev
-	}
-}

bridge/commands/admin.go

@@ -1,77 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-import (
-	"strconv"
-
-	"maunium.net/go/mautrix/id"
-)
-
-var CommandDiscardMegolmSession = &FullHandler{
-	Func: func(ce *Event) {
-		if ce.Bridge.Crypto == nil {
-			ce.Reply("This bridge instance doesn't have end-to-bridge encryption enabled")
-		} else {
-			ce.Bridge.Crypto.ResetSession(ce.Ctx, ce.RoomID)
-			ce.Reply("Successfully reset Megolm session in this room. New decryption keys will be shared the next time a message is sent from the remote network.")
-		}
-	},
-	Name:    "discard-megolm-session",
-	Aliases: []string{"discard-session"},
-	Help: HelpMeta{
-		Section:     HelpSectionAdmin,
-		Description: "Discard the Megolm session in the room",
-	},
-	RequiresAdmin: true,
-}
-
-func fnSetPowerLevel(ce *Event) {
-	var level int
-	var userID id.UserID
-	var err error
-	if len(ce.Args) == 1 {
-		level, err = strconv.Atoi(ce.Args[0])
-		if err != nil {
-			ce.Reply("Invalid power level \"%s\"", ce.Args[0])
-			return
-		}
-		userID = ce.User.GetMXID()
-	} else if len(ce.Args) == 2 {
-		userID = id.UserID(ce.Args[0])
-		_, _, err := userID.Parse()
-		if err != nil {
-			ce.Reply("Invalid user ID \"%s\"", ce.Args[0])
-			return
-		}
-		level, err = strconv.Atoi(ce.Args[1])
-		if err != nil {
-			ce.Reply("Invalid power level \"%s\"", ce.Args[1])
-			return
-		}
-	} else {
-		ce.Reply("**Usage:** `set-pl [user] <level>`")
-		return
-	}
-	_, err = ce.Portal.MainIntent().SetPowerLevel(ce.Ctx, ce.RoomID, userID, level)
-	if err != nil {
-		ce.Reply("Failed to set power levels: %v", err)
-	}
-}
-
-var CommandSetPowerLevel = &FullHandler{
-	Func:    fnSetPowerLevel,
-	Name:    "set-pl",
-	Aliases: []string{"set-power-level"},
-	Help: HelpMeta{
-		Section:     HelpSectionAdmin,
-		Description: "Change the power level in a portal room.",
-		Args:        "[_user ID_] <_power level_>",
-	},
-	RequiresAdmin:  true,
-	RequiresPortal: true,
-}

bridge/commands/doublepuppet.go

@@ -1,83 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-var CommandLoginMatrix = &FullHandler{
-	Func: fnLoginMatrix,
-	Name: "login-matrix",
-	Help: HelpMeta{
-		Section:     HelpSectionAuth,
-		Description: "Enable double puppeting.",
-		Args:        "<_access token_>",
-	},
-	RequiresLogin: true,
-}
-
-func fnLoginMatrix(ce *Event) {
-	if len(ce.Args) == 0 {
-		ce.Reply("**Usage:** `login-matrix <access token>`")
-		return
-	}
-	puppet := ce.User.GetIDoublePuppet()
-	if puppet == nil {
-		puppet = ce.User.GetIGhost()
-		if puppet == nil {
-			ce.Reply("Didn't get a ghost :(")
-			return
-		}
-	}
-	err := puppet.SwitchCustomMXID(ce.Args[0], ce.User.GetMXID())
-	if err != nil {
-		ce.Reply("Failed to enable double puppeting: %v", err)
-	} else {
-		ce.Reply("Successfully switched puppet")
-	}
-}
-
-var CommandPingMatrix = &FullHandler{
-	Func: fnPingMatrix,
-	Name: "ping-matrix",
-	Help: HelpMeta{
-		Section:     HelpSectionAuth,
-		Description: "Ping the Matrix server with the double puppet.",
-	},
-	RequiresLogin: true,
-}
-
-func fnPingMatrix(ce *Event) {
-	puppet := ce.User.GetIDoublePuppet()
-	if puppet == nil || puppet.CustomIntent() == nil {
-		ce.Reply("You are not logged in with your Matrix account.")
-		return
-	}
-	resp, err := puppet.CustomIntent().Whoami(ce.Ctx)
-	if err != nil {
-		ce.Reply("Failed to validate Matrix login: %v", err)
-	} else {
-		ce.Reply("Confirmed valid access token for %s / %s", resp.UserID, resp.DeviceID)
-	}
-}
-
-var CommandLogoutMatrix = &FullHandler{
-	Func: fnLogoutMatrix,
-	Name: "logout-matrix",
-	Help: HelpMeta{
-		Section:     HelpSectionAuth,
-		Description: "Disable double puppeting.",
-	},
-	RequiresLogin: true,
-}
-
-func fnLogoutMatrix(ce *Event) {
-	puppet := ce.User.GetIDoublePuppet()
-	if puppet == nil || puppet.CustomIntent() == nil {
-		ce.Reply("You don't have double puppeting enabled.")
-		return
-	}
-	puppet.ClearCustomMXID()
-	ce.Reply("Successfully disabled double puppeting.")
-}

bridge/commands/event.go

@@ -1,95 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/rs/zerolog"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge"
-	"maunium.net/go/mautrix/event"
-	"maunium.net/go/mautrix/format"
-	"maunium.net/go/mautrix/id"
-)
-
-// Event stores all data which might be used to handle commands
-type Event struct {
-	Bot       *appservice.IntentAPI
-	Bridge    *bridge.Bridge
-	Portal    bridge.Portal
-	Processor *Processor
-	Handler   MinimalHandler
-	RoomID    id.RoomID
-	EventID   id.EventID
-	User      bridge.User
-	Command   string
-	Args      []string
-	RawArgs   string
-	ReplyTo   id.EventID
-	Ctx       context.Context
-	ZLog      *zerolog.Logger
-}
-
-// MainIntent returns the intent to use when replying to the command.
-//
-// It prefers the bridge bot, but falls back to the other user in DMs if the bridge bot is not present.
-func (ce *Event) MainIntent() *appservice.IntentAPI {
-	intent := ce.Bot
-	if ce.Portal != nil && ce.Portal.IsPrivateChat() && !ce.Portal.IsEncrypted() {
-		intent = ce.Portal.MainIntent()
-	}
-	return intent
-}
-
-// Reply sends a reply to command as notice, with optional string formatting and automatic $cmdprefix replacement.
-func (ce *Event) Reply(msg string, args ...interface{}) {
-	msg = strings.ReplaceAll(msg, "$cmdprefix ", ce.Bridge.Config.Bridge.GetCommandPrefix()+" ")
-	if len(args) > 0 {
-		msg = fmt.Sprintf(msg, args...)
-	}
-	ce.ReplyAdvanced(msg, true, false)
-}
-
-// ReplyAdvanced sends a reply to command as notice. It allows using HTML and disabling markdown,
-// but doesn't have built-in string formatting.
-func (ce *Event) ReplyAdvanced(msg string, allowMarkdown, allowHTML bool) {
-	content := format.RenderMarkdown(msg, allowMarkdown, allowHTML)
-	content.MsgType = event.MsgNotice
-	_, err := ce.MainIntent().SendMessageEvent(ce.Ctx, ce.RoomID, event.EventMessage, content)
-	if err != nil {
-		ce.ZLog.Error().Err(err).Msgf("Failed to reply to command")
-	}
-}
-
-// React sends a reaction to the command.
-func (ce *Event) React(key string) {
-	_, err := ce.MainIntent().SendReaction(ce.Ctx, ce.RoomID, ce.EventID, key)
-	if err != nil {
-		ce.ZLog.Error().Err(err).Msgf("Failed to react to command")
-	}
-}
-
-// Redact redacts the command.
-func (ce *Event) Redact(req ...mautrix.ReqRedact) {
-	_, err := ce.MainIntent().RedactEvent(ce.Ctx, ce.RoomID, ce.EventID, req...)
-	if err != nil {
-		ce.ZLog.Error().Err(err).Msgf("Failed to redact command")
-	}
-}
-
-// MarkRead marks the command event as read.
-func (ce *Event) MarkRead() {
-	err := ce.MainIntent().SendReceipt(ce.Ctx, ce.RoomID, ce.EventID, event.ReceiptTypeRead, nil)
-	if err != nil {
-		ce.ZLog.Error().Err(err).Msgf("Failed to mark command as read")
-	}
-}

bridge/commands/handler.go

@@ -1,100 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-import (
-	"maunium.net/go/mautrix/bridge"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
-	"maunium.net/go/mautrix/event"
-)
-
-type MinimalHandler interface {
-	Run(*Event)
-}
-
-type MinimalHandlerFunc func(*Event)
-
-func (mhf MinimalHandlerFunc) Run(ce *Event) {
-	mhf(ce)
-}
-
-type CommandState struct {
-	Next   MinimalHandler
-	Action string
-	Meta   interface{}
-}
-
-type CommandingUser interface {
-	bridge.User
-	GetCommandState() *CommandState
-	SetCommandState(*CommandState)
-}
-
-type Handler interface {
-	MinimalHandler
-	GetName() string
-}
-
-type AliasedHandler interface {
-	Handler
-	GetAliases() []string
-}
-
-type FullHandler struct {
-	Func func(*Event)
-
-	Name    string
-	Aliases []string
-	Help    HelpMeta
-
-	RequiresAdmin  bool
-	RequiresPortal bool
-	RequiresLogin  bool
-
-	RequiresEventLevel event.Type
-}
-
-func (fh *FullHandler) GetHelp() HelpMeta {
-	fh.Help.Command = fh.Name
-	return fh.Help
-}
-
-func (fh *FullHandler) GetName() string {
-	return fh.Name
-}
-
-func (fh *FullHandler) GetAliases() []string {
-	return fh.Aliases
-}
-
-func (fh *FullHandler) ShowInHelp(ce *Event) bool {
-	return !fh.RequiresAdmin || ce.User.GetPermissionLevel() >= bridgeconfig.PermissionLevelAdmin
-}
-
-func (fh *FullHandler) userHasRoomPermission(ce *Event) bool {
-	levels, err := ce.MainIntent().PowerLevels(ce.Ctx, ce.RoomID)
-	if err != nil {
-		ce.ZLog.Warn().Err(err).Msg("Failed to check room power levels")
-		ce.Reply("Failed to get room power levels to see if you're allowed to use that command")
-		return false
-	}
-	return levels.GetUserLevel(ce.User.GetMXID()) >= levels.GetEventLevel(fh.RequiresEventLevel)
-}
-
-func (fh *FullHandler) Run(ce *Event) {
-	if fh.RequiresAdmin && ce.User.GetPermissionLevel() < bridgeconfig.PermissionLevelAdmin {
-		ce.Reply("That command is limited to bridge administrators.")
-	} else if fh.RequiresEventLevel.Type != "" && ce.User.GetPermissionLevel() < bridgeconfig.PermissionLevelAdmin && !fh.userHasRoomPermission(ce) {
-		ce.Reply("That command requires room admin rights.")
-	} else if fh.RequiresPortal && ce.Portal == nil {
-		ce.Reply("That command can only be ran in portal rooms.")
-	} else if fh.RequiresLogin && !ce.User.IsLoggedIn() {
-		ce.Reply("That command requires you to be logged in.")
-	} else {
-		fh.Func(ce)
-	}
-}

bridge/commands/help.go

@@ -1,129 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-type HelpfulHandler interface {
-	Handler
-	GetHelp() HelpMeta
-	ShowInHelp(*Event) bool
-}
-
-type HelpSection struct {
-	Name  string
-	Order int
-}
-
-var (
-	// Deprecated: this should be used as a placeholder that needs to be fixed
-	HelpSectionUnclassified = HelpSection{"Unclassified", -1}
-
-	HelpSectionGeneral = HelpSection{"General", 0}
-	HelpSectionAuth    = HelpSection{"Authentication", 10}
-	HelpSectionAdmin   = HelpSection{"Administration", 50}
-)
-
-type HelpMeta struct {
-	Command     string
-	Section     HelpSection
-	Description string
-	Args        string
-}
-
-func (hm *HelpMeta) String() string {
-	if len(hm.Args) == 0 {
-		return fmt.Sprintf("**%s** - %s", hm.Command, hm.Description)
-	}
-	return fmt.Sprintf("**%s** %s - %s", hm.Command, hm.Args, hm.Description)
-}
-
-type helpSectionList []HelpSection
-
-func (h helpSectionList) Len() int {
-	return len(h)
-}
-
-func (h helpSectionList) Less(i, j int) bool {
-	return h[i].Order < h[j].Order
-}
-
-func (h helpSectionList) Swap(i, j int) {
-	h[i], h[j] = h[j], h[i]
-}
-
-type helpMetaList []HelpMeta
-
-func (h helpMetaList) Len() int {
-	return len(h)
-}
-
-func (h helpMetaList) Less(i, j int) bool {
-	return h[i].Command < h[j].Command
-}
-
-func (h helpMetaList) Swap(i, j int) {
-	h[i], h[j] = h[j], h[i]
-}
-
-var _ sort.Interface = (helpSectionList)(nil)
-var _ sort.Interface = (helpMetaList)(nil)
-
-func FormatHelp(ce *Event) string {
-	sections := make(map[HelpSection]helpMetaList)
-	for _, handler := range ce.Processor.handlers {
-		helpfulHandler, ok := handler.(HelpfulHandler)
-		if !ok || !helpfulHandler.ShowInHelp(ce) {
-			continue
-		}
-		help := helpfulHandler.GetHelp()
-		if help.Description == "" {
-			continue
-		}
-		sections[help.Section] = append(sections[help.Section], help)
-	}
-
-	sortedSections := make(helpSectionList, 0, len(sections))
-	for section := range sections {
-		sortedSections = append(sortedSections, section)
-	}
-	sort.Sort(sortedSections)
-
-	var output strings.Builder
-	output.Grow(10240)
-
-	var prefixMsg string
-	if ce.RoomID == ce.User.GetManagementRoomID() {
-		prefixMsg = "This is your management room: prefixing commands with `%s` is not required."
-	} else if ce.Portal != nil {
-		prefixMsg = "**This is a portal room**: you must always prefix commands with `%s`. Management commands will not be bridged."
-	} else {
-		prefixMsg = "This is not your management room: prefixing commands with `%s` is required."
-	}
-	_, _ = fmt.Fprintf(&output, prefixMsg, ce.Bridge.Config.Bridge.GetCommandPrefix())
-	output.WriteByte('\n')
-	output.WriteString("Parameters in [square brackets] are optional, while parameters in <angle brackets> are required.")
-	output.WriteByte('\n')
-	output.WriteByte('\n')
-
-	for _, section := range sortedSections {
-		output.WriteString("#### ")
-		output.WriteString(section.Name)
-		output.WriteByte('\n')
-		sort.Sort(sections[section])
-		for _, command := range sections[section] {
-			output.WriteString(command.String())
-			output.WriteByte('\n')
-		}
-		output.WriteByte('\n')
-	}
-	return output.String()
-}

bridge/commands/meta.go

@@ -1,56 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-var CommandHelp = &FullHandler{
-	Func: func(ce *Event) {
-		ce.Reply(FormatHelp(ce))
-	},
-	Name: "help",
-	Help: HelpMeta{
-		Section:     HelpSectionGeneral,
-		Description: "Show this help message.",
-	},
-}
-
-var CommandVersion = &FullHandler{
-	Func: func(ce *Event) {
-		ce.Reply("[%s](%s) %s (%s)", ce.Bridge.Name, ce.Bridge.URL, ce.Bridge.LinkifiedVersion, ce.Bridge.BuildTime)
-	},
-	Name: "version",
-	Help: HelpMeta{
-		Section:     HelpSectionGeneral,
-		Description: "Get the bridge version.",
-	},
-}
-
-var CommandCancel = &FullHandler{
-	Func: func(ce *Event) {
-		commandingUser, ok := ce.User.(CommandingUser)
-		if !ok {
-			ce.Reply("This bridge does not implement cancelable commands")
-			return
-		}
-		state := commandingUser.GetCommandState()
-
-		if state != nil {
-			action := state.Action
-			if action == "" {
-				action = "Unknown action"
-			}
-			commandingUser.SetCommandState(nil)
-			ce.Reply("%s cancelled.", action)
-		} else {
-			ce.Reply("No ongoing command.")
-		}
-	},
-	Name: "cancel",
-	Help: HelpMeta{
-		Section:     HelpSectionGeneral,
-		Description: "Cancel an ongoing action.",
-	},
-}

bridge/commands/processor.go

@@ -1,122 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package commands
-
-import (
-	"context"
-	"runtime/debug"
-	"strings"
-
-	"github.com/rs/zerolog"
-
-	"maunium.net/go/mautrix/bridge"
-	"maunium.net/go/mautrix/id"
-)
-
-type Processor struct {
-	bridge *bridge.Bridge
-	log    *zerolog.Logger
-
-	handlers map[string]Handler
-	aliases  map[string]string
-}
-
-// NewProcessor creates a Processor
-func NewProcessor(bridge *bridge.Bridge) *Processor {
-	proc := &Processor{
-		bridge: bridge,
-		log:    bridge.ZLog,
-
-		handlers: make(map[string]Handler),
-		aliases:  make(map[string]string),
-	}
-	proc.AddHandlers(
-		CommandHelp, CommandVersion, CommandCancel,
-		CommandLoginMatrix, CommandLogoutMatrix, CommandPingMatrix,
-		CommandDiscardMegolmSession, CommandSetPowerLevel)
-	return proc
-}
-
-func (proc *Processor) AddHandlers(handlers ...Handler) {
-	for _, handler := range handlers {
-		proc.AddHandler(handler)
-	}
-}
-
-func (proc *Processor) AddHandler(handler Handler) {
-	proc.handlers[handler.GetName()] = handler
-	aliased, ok := handler.(AliasedHandler)
-	if ok {
-		for _, alias := range aliased.GetAliases() {
-			proc.aliases[alias] = handler.GetName()
-		}
-	}
-}
-
-// Handle handles messages to the bridge
-func (proc *Processor) Handle(ctx context.Context, roomID id.RoomID, eventID id.EventID, user bridge.User, message string, replyTo id.EventID) {
-	defer func() {
-		err := recover()
-		if err != nil {
-			zerolog.Ctx(ctx).Error().
-				Str(zerolog.ErrorStackFieldName, string(debug.Stack())).
-				Interface(zerolog.ErrorFieldName, err).
-				Msg("Panic in Matrix command handler")
-		}
-	}()
-	args := strings.Fields(message)
-	if len(args) == 0 {
-		args = []string{"unknown-command"}
-	}
-	command := strings.ToLower(args[0])
-	rawArgs := strings.TrimLeft(strings.TrimPrefix(message, command), " ")
-	log := zerolog.Ctx(ctx).With().Str("mx_command", command).Logger()
-	ctx = log.WithContext(ctx)
-	ce := &Event{
-		Bot:       proc.bridge.Bot,
-		Bridge:    proc.bridge,
-		Portal:    proc.bridge.Child.GetIPortal(roomID),
-		Processor: proc,
-		RoomID:    roomID,
-		EventID:   eventID,
-		User:      user,
-		Command:   command,
-		Args:      args[1:],
-		RawArgs:   rawArgs,
-		ReplyTo:   replyTo,
-		Ctx:       ctx,
-		ZLog:      &log,
-	}
-	log.Debug().Msg("Received command")
-
-	realCommand, ok := proc.aliases[ce.Command]
-	if !ok {
-		realCommand = ce.Command
-	}
-	commandingUser, ok := ce.User.(CommandingUser)
-
-	var handler MinimalHandler
-	handler, ok = proc.handlers[realCommand]
-	if !ok {
-		var state *CommandState
-		if commandingUser != nil {
-			state = commandingUser.GetCommandState()
-		}
-		if state != nil && state.Next != nil {
-			ce.Command = ""
-			ce.RawArgs = message
-			ce.Args = args
-			ce.Handler = state.Next
-			state.Next.Run(ce)
-		} else {
-			ce.Reply("Unknown command, use the `help` command for help.")
-		}
-	} else {
-		ce.Handler = handler
-		handler.Run(ce)
-	}
-}

bridge/crypto.go

@@ -1,511 +0,0 @@
-// Copyright (c) 2024 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:build cgo && !nocrypto
-
-package bridge
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"runtime/debug"
-	"sync"
-	"time"
-
-	"github.com/rs/zerolog"
-	"go.mau.fi/util/dbutil"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
-	"maunium.net/go/mautrix/crypto"
-	"maunium.net/go/mautrix/crypto/olm"
-	"maunium.net/go/mautrix/event"
-	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/sqlstatestore"
-)
-
-var _ crypto.StateStore = (*sqlstatestore.SQLStateStore)(nil)
-
-var NoSessionFound = crypto.NoSessionFound
-var DuplicateMessageIndex = crypto.DuplicateMessageIndex
-var UnknownMessageIndex = olm.UnknownMessageIndex
-
-type CryptoHelper struct {
-	bridge *Bridge
-	client *mautrix.Client
-	mach   *crypto.OlmMachine
-	store  *SQLCryptoStore
-	log    *zerolog.Logger
-
-	lock       sync.RWMutex
-	syncDone   sync.WaitGroup
-	cancelSync func()
-
-	cancelPeriodicDeleteLoop func()
-}
-
-func NewCryptoHelper(bridge *Bridge) Crypto {
-	if !bridge.Config.Bridge.GetEncryptionConfig().Allow {
-		bridge.ZLog.Debug().Msg("Bridge built with end-to-bridge encryption, but disabled in config")
-		return nil
-	}
-	log := bridge.ZLog.With().Str("component", "crypto").Logger()
-	return &CryptoHelper{
-		bridge: bridge,
-		log:    &log,
-	}
-}
-
-func (helper *CryptoHelper) Init(ctx context.Context) error {
-	if len(helper.bridge.CryptoPickleKey) == 0 {
-		panic("CryptoPickleKey not set")
-	}
-	helper.log.Debug().Msg("Initializing end-to-bridge encryption...")
-
-	helper.store = NewSQLCryptoStore(
-		helper.bridge.DB,
-		dbutil.ZeroLogger(helper.bridge.ZLog.With().Str("db_section", "crypto").Logger()),
-		helper.bridge.AS.BotMXID(),
-		fmt.Sprintf("@%s:%s", helper.bridge.Config.Bridge.FormatUsername("%"), helper.bridge.AS.HomeserverDomain),
-		helper.bridge.CryptoPickleKey,
-	)
-
-	err := helper.store.DB.Upgrade(ctx)
-	if err != nil {
-		helper.bridge.LogDBUpgradeErrorAndExit("crypto", err)
-	}
-
-	var isExistingDevice bool
-	helper.client, isExistingDevice, err = helper.loginBot(ctx)
-	if err != nil {
-		return err
-	}
-
-	helper.log.Debug().
-		Str("device_id", helper.client.DeviceID.String()).
-		Msg("Logged in as bridge bot")
-	stateStore := &cryptoStateStore{helper.bridge}
-	helper.mach = crypto.NewOlmMachine(helper.client, helper.log, helper.store, stateStore)
-	helper.mach.AllowKeyShare = helper.allowKeyShare
-
-	encryptionConfig := helper.bridge.Config.Bridge.GetEncryptionConfig()
-	helper.mach.SendKeysMinTrust = encryptionConfig.VerificationLevels.Receive
-	helper.mach.PlaintextMentions = encryptionConfig.PlaintextMentions
-
-	helper.mach.DeleteOutboundKeysOnAck = encryptionConfig.DeleteKeys.DeleteOutboundOnAck
-	helper.mach.DontStoreOutboundKeys = encryptionConfig.DeleteKeys.DontStoreOutbound
-	helper.mach.RatchetKeysOnDecrypt = encryptionConfig.DeleteKeys.RatchetOnDecrypt
-	helper.mach.DeleteFullyUsedKeysOnDecrypt = encryptionConfig.DeleteKeys.DeleteFullyUsedOnDecrypt
-	helper.mach.DeletePreviousKeysOnReceive = encryptionConfig.DeleteKeys.DeletePrevOnNewSession
-	helper.mach.DeleteKeysOnDeviceDelete = encryptionConfig.DeleteKeys.DeleteOnDeviceDelete
-	helper.mach.DisableDeviceChangeKeyRotation = encryptionConfig.Rotation.DisableDeviceChangeKeyRotation
-	if encryptionConfig.DeleteKeys.PeriodicallyDeleteExpired {
-		ctx, cancel := context.WithCancel(context.Background())
-		helper.cancelPeriodicDeleteLoop = cancel
-		go helper.mach.ExpiredKeyDeleteLoop(ctx)
-	}
-
-	if encryptionConfig.DeleteKeys.DeleteOutdatedInbound {
-		deleted, err := helper.store.RedactOutdatedGroupSessions(ctx)
-		if err != nil {
-			return err
-		}
-		if len(deleted) > 0 {
-			helper.log.Debug().Int("deleted", len(deleted)).Msg("Deleted inbound keys which lacked expiration metadata")
-		}
-	}
-
-	helper.client.Syncer = &cryptoSyncer{helper.mach}
-	helper.client.Store = helper.store
-
-	err = helper.mach.Load(ctx)
-	if err != nil {
-		return err
-	}
-	if isExistingDevice {
-		helper.verifyKeysAreOnServer(ctx)
-	}
-
-	go helper.resyncEncryptionInfo(context.TODO())
-
-	return nil
-}
-
-func (helper *CryptoHelper) resyncEncryptionInfo(ctx context.Context) {
-	log := helper.log.With().Str("action", "resync encryption event").Logger()
-	rows, err := helper.bridge.DB.Query(ctx, `SELECT room_id FROM mx_room_state WHERE encryption='{"resync":true}'`)
-	if err != nil {
-		log.Err(err).Msg("Failed to query rooms for resync")
-		return
-	}
-	roomIDs, err := dbutil.NewRowIter(rows, dbutil.ScanSingleColumn[id.RoomID]).AsList()
-	if err != nil {
-		log.Err(err).Msg("Failed to scan rooms for resync")
-		return
-	}
-	if len(roomIDs) > 0 {
-		log.Debug().Interface("room_ids", roomIDs).Msg("Resyncing rooms")
-		for _, roomID := range roomIDs {
-			var evt event.EncryptionEventContent
-			err = helper.client.StateEvent(ctx, roomID, event.StateEncryption, "", &evt)
-			if err != nil {
-				log.Err(err).Str("room_id", roomID.String()).Msg("Failed to get encryption event")
-				_, err = helper.bridge.DB.Exec(ctx, `
-					UPDATE mx_room_state SET encryption=NULL WHERE room_id=$1 AND encryption='{"resync":true}'
-				`, roomID)
-				if err != nil {
-					log.Err(err).Str("room_id", roomID.String()).Msg("Failed to unmark room for resync after failed sync")
-				}
-			} else {
-				maxAge := evt.RotationPeriodMillis
-				if maxAge <= 0 {
-					maxAge = (7 * 24 * time.Hour).Milliseconds()
-				}
-				maxMessages := evt.RotationPeriodMessages
-				if maxMessages <= 0 {
-					maxMessages = 100
-				}
-				log.Debug().
-					Str("room_id", roomID.String()).
-					Int64("max_age_ms", maxAge).
-					Int("max_messages", maxMessages).
-					Interface("content", &evt).
-					Msg("Resynced encryption event")
-				_, err = helper.bridge.DB.Exec(ctx, `
-					UPDATE crypto_megolm_inbound_session
-					SET max_age=$1, max_messages=$2
-					WHERE room_id=$3 AND max_age IS NULL AND max_messages IS NULL
-				`, maxAge, maxMessages, roomID)
-				if err != nil {
-					log.Err(err).Str("room_id", roomID.String()).Msg("Failed to update megolm session table")
-				} else {
-					log.Debug().Str("room_id", roomID.String()).Msg("Updated megolm session table")
-				}
-			}
-		}
-	}
-}
-
-func (helper *CryptoHelper) allowKeyShare(ctx context.Context, device *id.Device, info event.RequestedKeyInfo) *crypto.KeyShareRejection {
-	cfg := helper.bridge.Config.Bridge.GetEncryptionConfig()
-	if !cfg.AllowKeySharing {
-		return &crypto.KeyShareRejectNoResponse
-	} else if device.Trust == id.TrustStateBlacklisted {
-		return &crypto.KeyShareRejectBlacklisted
-	} else if trustState := helper.mach.ResolveTrust(device); trustState >= cfg.VerificationLevels.Share {
-		portal := helper.bridge.Child.GetIPortal(info.RoomID)
-		if portal == nil {
-			zerolog.Ctx(ctx).Debug().Msg("Rejecting key request: room is not a portal")
-			return &crypto.KeyShareRejection{Code: event.RoomKeyWithheldUnavailable, Reason: "Requested room is not a portal room"}
-		}
-		user := helper.bridge.Child.GetIUser(device.UserID, true)
-		// FIXME reimplement IsInPortal
-		if user.GetPermissionLevel() < bridgeconfig.PermissionLevelAdmin /*&& !user.IsInPortal(portal.Key)*/ {
-			zerolog.Ctx(ctx).Debug().Msg("Rejecting key request: user is not in portal")
-			return &crypto.KeyShareRejection{Code: event.RoomKeyWithheldUnauthorized, Reason: "You're not in that portal"}
-		}
-		zerolog.Ctx(ctx).Debug().Msg("Accepting key request")
-		return nil
-	} else {
-		return &crypto.KeyShareRejectUnverified
-	}
-}
-
-func (helper *CryptoHelper) loginBot(ctx context.Context) (*mautrix.Client, bool, error) {
-	deviceID, err := helper.store.FindDeviceID(ctx)
-	if err != nil {
-		return nil, false, fmt.Errorf("failed to find existing device ID: %w", err)
-	} else if len(deviceID) > 0 {
-		helper.log.Debug().Str("device_id", deviceID.String()).Msg("Found existing device ID for bot in database")
-	}
-	// Create a new client instance with the default AS settings (including as_token),
-	// the Login call will then override the access token in the client.
-	client := helper.bridge.AS.NewMautrixClient(helper.bridge.AS.BotMXID())
-	flows, err := client.GetLoginFlows(ctx)
-	if err != nil {
-		return nil, deviceID != "", fmt.Errorf("failed to get supported login flows: %w", err)
-	} else if !flows.HasFlow(mautrix.AuthTypeAppservice) {
-		return nil, deviceID != "", fmt.Errorf("homeserver does not support appservice login")
-	}
-	resp, err := client.Login(ctx, &mautrix.ReqLogin{
-		Type: mautrix.AuthTypeAppservice,
-		Identifier: mautrix.UserIdentifier{
-			Type: mautrix.IdentifierTypeUser,
-			User: string(helper.bridge.AS.BotMXID()),
-		},
-		DeviceID:         deviceID,
-		StoreCredentials: true,
-
-		InitialDeviceDisplayName: fmt.Sprintf("%s bridge", helper.bridge.ProtocolName),
-	})
-	if err != nil {
-		return nil, deviceID != "", fmt.Errorf("failed to log in as bridge bot: %w", err)
-	}
-	helper.store.DeviceID = resp.DeviceID
-	return client, deviceID != "", nil
-}
-
-func (helper *CryptoHelper) verifyKeysAreOnServer(ctx context.Context) {
-	helper.log.Debug().Msg("Making sure keys are still on server")
-	resp, err := helper.client.QueryKeys(ctx, &mautrix.ReqQueryKeys{
-		DeviceKeys: map[id.UserID]mautrix.DeviceIDList{
-			helper.client.UserID: {helper.client.DeviceID},
-		},
-	})
-	if err != nil {
-		helper.log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to query own keys to make sure device still exists")
-		os.Exit(33)
-	}
-	device, ok := resp.DeviceKeys[helper.client.UserID][helper.client.DeviceID]
-	if ok && len(device.Keys) > 0 {
-		return
-	}
-	helper.log.Warn().Msg("Existing device doesn't have keys on server, resetting crypto")
-	helper.Reset(ctx, false)
-}
-
-func (helper *CryptoHelper) Start() {
-	if helper.bridge.Config.Bridge.GetEncryptionConfig().Appservice {
-		helper.log.Debug().Msg("End-to-bridge encryption is in appservice mode, registering event listeners and not starting syncer")
-		helper.bridge.AS.Registration.EphemeralEvents = true
-		helper.mach.AddAppserviceListener(helper.bridge.EventProcessor)
-		return
-	}
-	helper.syncDone.Add(1)
-	defer helper.syncDone.Done()
-	helper.log.Debug().Msg("Starting syncer for receiving to-device messages")
-	var ctx context.Context
-	ctx, helper.cancelSync = context.WithCancel(context.Background())
-	err := helper.client.SyncWithContext(ctx)
-	if err != nil && !errors.Is(err, context.Canceled) {
-		helper.log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Fatal error syncing")
-		os.Exit(51)
-	} else {
-		helper.log.Info().Msg("Bridge bot to-device syncer stopped without error")
-	}
-}
-
-func (helper *CryptoHelper) Stop() {
-	helper.log.Debug().Msg("CryptoHelper.Stop() called, stopping bridge bot sync")
-	helper.client.StopSync()
-	if helper.cancelSync != nil {
-		helper.cancelSync()
-	}
-	if helper.cancelPeriodicDeleteLoop != nil {
-		helper.cancelPeriodicDeleteLoop()
-	}
-	helper.syncDone.Wait()
-}
-
-func (helper *CryptoHelper) clearDatabase(ctx context.Context) {
-	_, err := helper.store.DB.Exec(ctx, "DELETE FROM crypto_account")
-	if err != nil {
-		helper.log.Warn().Err(err).Msg("Failed to clear crypto_account table")
-	}
-	_, err = helper.store.DB.Exec(ctx, "DELETE FROM crypto_olm_session")
-	if err != nil {
-		helper.log.Warn().Err(err).Msg("Failed to clear crypto_olm_session table")
-	}
-	_, err = helper.store.DB.Exec(ctx, "DELETE FROM crypto_megolm_outbound_session")
-	if err != nil {
-		helper.log.Warn().Err(err).Msg("Failed to clear crypto_megolm_outbound_session table")
-	}
-	//_, _ = helper.store.DB.Exec("DELETE FROM crypto_device")
-	//_, _ = helper.store.DB.Exec("DELETE FROM crypto_tracked_user")
-	//_, _ = helper.store.DB.Exec("DELETE FROM crypto_cross_signing_keys")
-	//_, _ = helper.store.DB.Exec("DELETE FROM crypto_cross_signing_signatures")
-}
-
-func (helper *CryptoHelper) Reset(ctx context.Context, startAfterReset bool) {
-	helper.lock.Lock()
-	defer helper.lock.Unlock()
-	helper.log.Info().Msg("Resetting end-to-bridge encryption device")
-	helper.Stop()
-	helper.log.Debug().Msg("Crypto syncer stopped, clearing database")
-	helper.clearDatabase(ctx)
-	helper.log.Debug().Msg("Crypto database cleared, logging out of all sessions")
-	_, err := helper.client.LogoutAll(ctx)
-	if err != nil {
-		helper.log.Warn().Err(err).Msg("Failed to log out all devices")
-	}
-	helper.client = nil
-	helper.store = nil
-	helper.mach = nil
-	err = helper.Init(ctx)
-	if err != nil {
-		helper.log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Error reinitializing end-to-bridge encryption")
-		os.Exit(50)
-	}
-	helper.log.Info().Msg("End-to-bridge encryption successfully reset")
-	if startAfterReset {
-		go helper.Start()
-	}
-}
-
-func (helper *CryptoHelper) Client() *mautrix.Client {
-	return helper.client
-}
-
-func (helper *CryptoHelper) Decrypt(ctx context.Context, evt *event.Event) (*event.Event, error) {
-	return helper.mach.DecryptMegolmEvent(ctx, evt)
-}
-
-func (helper *CryptoHelper) Encrypt(ctx context.Context, roomID id.RoomID, evtType event.Type, content *event.Content) (err error) {
-	helper.lock.RLock()
-	defer helper.lock.RUnlock()
-	var encrypted *event.EncryptedEventContent
-	encrypted, err = helper.mach.EncryptMegolmEvent(ctx, roomID, evtType, content)
-	if err != nil {
-		if !errors.Is(err, crypto.SessionExpired) && !errors.Is(err, crypto.SessionNotShared) && !errors.Is(err, crypto.NoGroupSession) {
-			return
-		}
-		helper.log.Debug().Err(err).
-			Str("room_id", roomID.String()).
-			Msg("Got error while encrypting event for room, sharing group session and trying again...")
-		var users []id.UserID
-		users, err = helper.store.GetRoomJoinedOrInvitedMembers(ctx, roomID)
-		if err != nil {
-			err = fmt.Errorf("failed to get room member list: %w", err)
-		} else if err = helper.mach.ShareGroupSession(ctx, roomID, users); err != nil {
-			err = fmt.Errorf("failed to share group session: %w", err)
-		} else if encrypted, err = helper.mach.EncryptMegolmEvent(ctx, roomID, evtType, content); err != nil {
-			err = fmt.Errorf("failed to encrypt event after re-sharing group session: %w", err)
-		}
-	}
-	if encrypted != nil {
-		content.Parsed = encrypted
-		content.Raw = nil
-	}
-	return
-}
-
-func (helper *CryptoHelper) WaitForSession(ctx context.Context, roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID, timeout time.Duration) bool {
-	helper.lock.RLock()
-	defer helper.lock.RUnlock()
-	return helper.mach.WaitForSession(ctx, roomID, senderKey, sessionID, timeout)
-}
-
-func (helper *CryptoHelper) RequestSession(ctx context.Context, roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID, userID id.UserID, deviceID id.DeviceID) {
-	helper.lock.RLock()
-	defer helper.lock.RUnlock()
-	if deviceID == "" {
-		deviceID = "*"
-	}
-	err := helper.mach.SendRoomKeyRequest(ctx, roomID, senderKey, sessionID, "", map[id.UserID][]id.DeviceID{userID: {deviceID}})
-	if err != nil {
-		helper.log.Warn().Err(err).
-			Str("user_id", userID.String()).
-			Str("device_id", deviceID.String()).
-			Str("session_id", sessionID.String()).
-			Str("room_id", roomID.String()).
-			Msg("Failed to send key request")
-	} else {
-		helper.log.Debug().
-			Str("user_id", userID.String()).
-			Str("device_id", deviceID.String()).
-			Str("session_id", sessionID.String()).
-			Str("room_id", roomID.String()).
-			Msg("Sent key request")
-	}
-}
-
-func (helper *CryptoHelper) ResetSession(ctx context.Context, roomID id.RoomID) {
-	helper.lock.RLock()
-	defer helper.lock.RUnlock()
-	err := helper.mach.CryptoStore.RemoveOutboundGroupSession(ctx, roomID)
-	if err != nil {
-		helper.log.Debug().Err(err).
-			Str("room_id", roomID.String()).
-			Msg("Error manually removing outbound group session in room")
-	}
-}
-
-func (helper *CryptoHelper) HandleMemberEvent(ctx context.Context, evt *event.Event) {
-	helper.lock.RLock()
-	defer helper.lock.RUnlock()
-	helper.mach.HandleMemberEvent(ctx, evt)
-}
-
-// ShareKeys uploads the given number of one-time-keys to the server.
-func (helper *CryptoHelper) ShareKeys(ctx context.Context) error {
-	return helper.mach.ShareKeys(ctx, -1)
-}
-
-type cryptoSyncer struct {
-	*crypto.OlmMachine
-}
-
-func (syncer *cryptoSyncer) ProcessResponse(ctx context.Context, resp *mautrix.RespSync, since string) error {
-	done := make(chan struct{})
-	go func() {
-		defer func() {
-			if err := recover(); err != nil {
-				syncer.Log.Error().
-					Str("since", since).
-					Interface("error", err).
-					Str("stack", string(debug.Stack())).
-					Msg("Processing sync response panicked")
-			}
-			done <- struct{}{}
-		}()
-		syncer.Log.Trace().Str("since", since).Msg("Starting sync response handling")
-		syncer.ProcessSyncResponse(ctx, resp, since)
-		syncer.Log.Trace().Str("since", since).Msg("Successfully handled sync response")
-	}()
-	select {
-	case <-done:
-	case <-time.After(30 * time.Second):
-		syncer.Log.Warn().Str("since", since).Msg("Handling sync response is taking unusually long")
-	}
-	return nil
-}
-
-func (syncer *cryptoSyncer) OnFailedSync(_ *mautrix.RespSync, err error) (time.Duration, error) {
-	if errors.Is(err, mautrix.MUnknownToken) {
-		return 0, err
-	}
-	syncer.Log.Error().Err(err).Msg("Error /syncing, waiting 10 seconds")
-	return 10 * time.Second, nil
-}
-
-func (syncer *cryptoSyncer) GetFilterJSON(_ id.UserID) *mautrix.Filter {
-	everything := []event.Type{{Type: "*"}}
-	return &mautrix.Filter{
-		Presence:    mautrix.FilterPart{NotTypes: everything},
-		AccountData: mautrix.FilterPart{NotTypes: everything},
-		Room: mautrix.RoomFilter{
-			IncludeLeave: false,
-			Ephemeral:    mautrix.FilterPart{NotTypes: everything},
-			AccountData:  mautrix.FilterPart{NotTypes: everything},
-			State:        mautrix.FilterPart{NotTypes: everything},
-			Timeline:     mautrix.FilterPart{NotTypes: everything},
-		},
-	}
-}
-
-type cryptoStateStore struct {
-	bridge *Bridge
-}
-
-var _ crypto.StateStore = (*cryptoStateStore)(nil)
-
-func (c *cryptoStateStore) IsEncrypted(ctx context.Context, id id.RoomID) (bool, error) {
-	portal := c.bridge.Child.GetIPortal(id)
-	if portal != nil {
-		return portal.IsEncrypted(), nil
-	}
-	return c.bridge.StateStore.IsEncrypted(ctx, id)
-}
-
-func (c *cryptoStateStore) FindSharedRooms(ctx context.Context, id id.UserID) ([]id.RoomID, error) {
-	return c.bridge.StateStore.FindSharedRooms(ctx, id)
-}
-
-func (c *cryptoStateStore) GetEncryptionEvent(ctx context.Context, id id.RoomID) (*event.EncryptionEventContent, error) {
-	return c.bridge.StateStore.GetEncryptionEvent(ctx, id)
-}

bridge/cryptostore.go

@@ -1,63 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:build cgo && !nocrypto
-
-package bridge
-
-import (
-	"context"
-
-	"github.com/lib/pq"
-	"go.mau.fi/util/dbutil"
-
-	"maunium.net/go/mautrix/crypto"
-	"maunium.net/go/mautrix/id"
-)
-
-func init() {
-	crypto.PostgresArrayWrapper = pq.Array
-}
-
-type SQLCryptoStore struct {
-	*crypto.SQLCryptoStore
-	UserID        id.UserID
-	GhostIDFormat string
-}
-
-var _ crypto.Store = (*SQLCryptoStore)(nil)
-
-func NewSQLCryptoStore(db *dbutil.Database, log dbutil.DatabaseLogger, userID id.UserID, ghostIDFormat, pickleKey string) *SQLCryptoStore {
-	return &SQLCryptoStore{
-		SQLCryptoStore: crypto.NewSQLCryptoStore(db, log, "", "", []byte(pickleKey)),
-		UserID:         userID,
-		GhostIDFormat:  ghostIDFormat,
-	}
-}
-
-func (store *SQLCryptoStore) GetRoomJoinedOrInvitedMembers(ctx context.Context, roomID id.RoomID) (members []id.UserID, err error) {
-	var rows dbutil.Rows
-	rows, err = store.DB.Query(ctx, `
-		SELECT user_id FROM mx_user_profile
-		WHERE room_id=$1
-			AND (membership='join' OR membership='invite')
-			AND user_id<>$2
-			AND user_id NOT LIKE $3
-	`, roomID, store.UserID, store.GhostIDFormat)
-	if err != nil {
-		return
-	}
-	for rows.Next() {
-		var userID id.UserID
-		err = rows.Scan(&userID)
-		if err != nil {
-			return members, err
-		} else {
-			members = append(members, userID)
-		}
-	}
-	return
-}

bridge/doublepuppet.go

@@ -1,173 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridge
-
-import (
-	"context"
-	"crypto/hmac"
-	"crypto/sha512"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/rs/zerolog"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/id"
-)
-
-type doublePuppetUtil struct {
-	br  *Bridge
-	log zerolog.Logger
-}
-
-func (dp *doublePuppetUtil) newClient(ctx context.Context, mxid id.UserID, accessToken string) (*mautrix.Client, error) {
-	_, homeserver, err := mxid.Parse()
-	if err != nil {
-		return nil, err
-	}
-	homeserverURL, found := dp.br.Config.Bridge.GetDoublePuppetConfig().ServerMap[homeserver]
-	if !found {
-		if homeserver == dp.br.AS.HomeserverDomain {
-			homeserverURL = ""
-		} else if dp.br.Config.Bridge.GetDoublePuppetConfig().AllowDiscovery {
-			resp, err := mautrix.DiscoverClientAPI(ctx, homeserver)
-			if err != nil {
-				return nil, fmt.Errorf("failed to find homeserver URL for %s: %v", homeserver, err)
-			}
-			homeserverURL = resp.Homeserver.BaseURL
-			dp.log.Debug().
-				Str("homeserver", homeserver).
-				Str("url", homeserverURL).
-				Str("user_id", mxid.String()).
-				Msg("Discovered URL to enable double puppeting for user")
-		} else {
-			return nil, fmt.Errorf("double puppeting from %s is not allowed", homeserver)
-		}
-	}
-	return dp.br.AS.NewExternalMautrixClient(mxid, accessToken, homeserverURL)
-}
-
-func (dp *doublePuppetUtil) newIntent(ctx context.Context, mxid id.UserID, accessToken string) (*appservice.IntentAPI, error) {
-	client, err := dp.newClient(ctx, mxid, accessToken)
-	if err != nil {
-		return nil, err
-	}
-
-	ia := dp.br.AS.NewIntentAPI("custom")
-	ia.Client = client
-	ia.Localpart, _, _ = mxid.Parse()
-	ia.UserID = mxid
-	ia.IsCustomPuppet = true
-	return ia, nil
-}
-
-func (dp *doublePuppetUtil) autoLogin(ctx context.Context, mxid id.UserID, loginSecret string) (string, error) {
-	dp.log.Debug().Str("user_id", mxid.String()).Msg("Logging into user account with shared secret")
-	client, err := dp.newClient(ctx, mxid, "")
-	if err != nil {
-		return "", fmt.Errorf("failed to create mautrix client to log in: %v", err)
-	}
-	bridgeName := fmt.Sprintf("%s Bridge", dp.br.ProtocolName)
-	req := mautrix.ReqLogin{
-		Identifier:               mautrix.UserIdentifier{Type: mautrix.IdentifierTypeUser, User: string(mxid)},
-		DeviceID:                 id.DeviceID(bridgeName),
-		InitialDeviceDisplayName: bridgeName,
-	}
-	if loginSecret == "appservice" {
-		client.AccessToken = dp.br.AS.Registration.AppToken
-		req.Type = mautrix.AuthTypeAppservice
-	} else {
-		loginFlows, err := client.GetLoginFlows(ctx)
-		if err != nil {
-			return "", fmt.Errorf("failed to get supported login flows: %w", err)
-		}
-		mac := hmac.New(sha512.New, []byte(loginSecret))
-		mac.Write([]byte(mxid))
-		token := hex.EncodeToString(mac.Sum(nil))
-		switch {
-		case loginFlows.HasFlow(mautrix.AuthTypeDevtureSharedSecret):
-			req.Type = mautrix.AuthTypeDevtureSharedSecret
-			req.Token = token
-		case loginFlows.HasFlow(mautrix.AuthTypePassword):
-			req.Type = mautrix.AuthTypePassword
-			req.Password = token
-		default:
-			return "", fmt.Errorf("no supported auth types for shared secret auth found")
-		}
-	}
-	resp, err := client.Login(ctx, &req)
-	if err != nil {
-		return "", err
-	}
-	return resp.AccessToken, nil
-}
-
-var (
-	ErrMismatchingMXID = errors.New("whoami result does not match custom mxid")
-	ErrNoAccessToken   = errors.New("no access token provided")
-	ErrNoMXID          = errors.New("no mxid provided")
-)
-
-const useConfigASToken = "appservice-config"
-const asTokenModePrefix = "as_token:"
-
-func (dp *doublePuppetUtil) Setup(ctx context.Context, mxid id.UserID, savedAccessToken string, reloginOnFail bool) (intent *appservice.IntentAPI, newAccessToken string, err error) {
-	if len(mxid) == 0 {
-		err = ErrNoMXID
-		return
-	}
-	_, homeserver, _ := mxid.Parse()
-	loginSecret, hasSecret := dp.br.Config.Bridge.GetDoublePuppetConfig().SharedSecretMap[homeserver]
-	// Special case appservice: prefix to not login and use it as an as_token directly.
-	if hasSecret && strings.HasPrefix(loginSecret, asTokenModePrefix) {
-		intent, err = dp.newIntent(ctx, mxid, strings.TrimPrefix(loginSecret, asTokenModePrefix))
-		if err != nil {
-			return
-		}
-		intent.SetAppServiceUserID = true
-		if savedAccessToken != useConfigASToken {
-			var resp *mautrix.RespWhoami
-			resp, err = intent.Whoami(ctx)
-			if err == nil && resp.UserID != mxid {
-				err = ErrMismatchingMXID
-			}
-		}
-		return intent, useConfigASToken, err
-	}
-	if savedAccessToken == "" || savedAccessToken == useConfigASToken {
-		if reloginOnFail && hasSecret {
-			savedAccessToken, err = dp.autoLogin(ctx, mxid, loginSecret)
-		} else {
-			err = ErrNoAccessToken
-		}
-		if err != nil {
-			return
-		}
-	}
-	intent, err = dp.newIntent(ctx, mxid, savedAccessToken)
-	if err != nil {
-		return
-	}
-	var resp *mautrix.RespWhoami
-	resp, err = intent.Whoami(ctx)
-	if err != nil {
-		if reloginOnFail && hasSecret && errors.Is(err, mautrix.MUnknownToken) {
-			intent.AccessToken, err = dp.autoLogin(ctx, mxid, loginSecret)
-			if err == nil {
-				newAccessToken = intent.AccessToken
-			}
-		}
-	} else if resp.UserID != mxid {
-		err = ErrMismatchingMXID
-	} else {
-		newAccessToken = savedAccessToken
-	}
-	return
-}

bridge/matrix.go

@@ -1,755 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridge
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/rs/zerolog"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
-	"maunium.net/go/mautrix/bridge/status"
-	"maunium.net/go/mautrix/event"
-	"maunium.net/go/mautrix/format"
-	"maunium.net/go/mautrix/id"
-)
-
-type CommandProcessor interface {
-	Handle(ctx context.Context, roomID id.RoomID, eventID id.EventID, user User, message string, replyTo id.EventID)
-}
-
-type MatrixHandler struct {
-	bridge *Bridge
-	as     *appservice.AppService
-	log    *zerolog.Logger
-
-	TrackEventDuration func(event.Type) func()
-}
-
-func noop() {}
-
-func noopTrack(_ event.Type) func() {
-	return noop
-}
-
-func NewMatrixHandler(br *Bridge) *MatrixHandler {
-	handler := &MatrixHandler{
-		bridge: br,
-		as:     br.AS,
-		log:    br.ZLog,
-
-		TrackEventDuration: noopTrack,
-	}
-	for evtType := range status.CheckpointTypes {
-		br.EventProcessor.On(evtType, handler.sendBridgeCheckpoint)
-	}
-	br.EventProcessor.On(event.EventMessage, handler.HandleMessage)
-	br.EventProcessor.On(event.EventEncrypted, handler.HandleEncrypted)
-	br.EventProcessor.On(event.EventSticker, handler.HandleMessage)
-	br.EventProcessor.On(event.EventReaction, handler.HandleReaction)
-	br.EventProcessor.On(event.EventRedaction, handler.HandleRedaction)
-	br.EventProcessor.On(event.StateMember, handler.HandleMembership)
-	br.EventProcessor.On(event.StateRoomName, handler.HandleRoomMetadata)
-	br.EventProcessor.On(event.StateRoomAvatar, handler.HandleRoomMetadata)
-	br.EventProcessor.On(event.StateTopic, handler.HandleRoomMetadata)
-	br.EventProcessor.On(event.StateEncryption, handler.HandleEncryption)
-	br.EventProcessor.On(event.EphemeralEventReceipt, handler.HandleReceipt)
-	br.EventProcessor.On(event.EphemeralEventTyping, handler.HandleTyping)
-	br.EventProcessor.On(event.StatePowerLevels, handler.HandlePowerLevels)
-	br.EventProcessor.On(event.StateJoinRules, handler.HandleJoinRule)
-	return handler
-}
-
-func (mx *MatrixHandler) sendBridgeCheckpoint(_ context.Context, evt *event.Event) {
-	if !evt.Mautrix.CheckpointSent {
-		go mx.bridge.SendMessageSuccessCheckpoint(evt, status.MsgStepBridge, 0)
-	}
-}
-
-func (mx *MatrixHandler) HandleEncryption(ctx context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	if evt.Content.AsEncryption().Algorithm != id.AlgorithmMegolmV1 {
-		return
-	}
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal != nil && !portal.IsEncrypted() {
-		mx.log.Debug().
-			Str("user_id", evt.Sender.String()).
-			Str("room_id", evt.RoomID.String()).
-			Msg("Encryption was enabled in room")
-		portal.MarkEncrypted()
-		if portal.IsPrivateChat() {
-			err := mx.as.BotIntent().EnsureJoined(ctx, evt.RoomID, appservice.EnsureJoinedParams{BotOverride: portal.MainIntent().Client})
-			if err != nil {
-				mx.log.Err(err).
-					Str("room_id", evt.RoomID.String()).
-					Msg("Failed to join bot to room after encryption was enabled")
-			}
-		}
-	}
-}
-
-func (mx *MatrixHandler) joinAndCheckMembers(ctx context.Context, evt *event.Event, intent *appservice.IntentAPI) *mautrix.RespJoinedMembers {
-	log := zerolog.Ctx(ctx)
-	resp, err := intent.JoinRoomByID(ctx, evt.RoomID)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to join room with invite")
-		return nil
-	}
-
-	members, err := intent.JoinedMembers(ctx, resp.RoomID)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to get members in room after accepting invite, leaving room")
-		_, _ = intent.LeaveRoom(ctx, resp.RoomID)
-		return nil
-	}
-
-	if len(members.Joined) < 2 {
-		log.Debug().Msg("Leaving empty room after accepting invite")
-		_, _ = intent.LeaveRoom(ctx, resp.RoomID)
-		return nil
-	}
-	return members
-}
-
-func (mx *MatrixHandler) sendNoticeWithMarkdown(ctx context.Context, roomID id.RoomID, message string) (*mautrix.RespSendEvent, error) {
-	intent := mx.as.BotIntent()
-	content := format.RenderMarkdown(message, true, false)
-	content.MsgType = event.MsgNotice
-	return intent.SendMessageEvent(ctx, roomID, event.EventMessage, content)
-}
-
-func (mx *MatrixHandler) HandleBotInvite(ctx context.Context, evt *event.Event) {
-	intent := mx.as.BotIntent()
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil {
-		return
-	}
-
-	members := mx.joinAndCheckMembers(ctx, evt, intent)
-	if members == nil {
-		return
-	}
-
-	if user.GetPermissionLevel() < bridgeconfig.PermissionLevelUser {
-		_, _ = intent.SendNotice(ctx, evt.RoomID, "You are not whitelisted to use this bridge.\n"+
-			"If you're the owner of this bridge, see the bridge.permissions section in your config file.")
-		_, _ = intent.LeaveRoom(ctx, evt.RoomID)
-		return
-	}
-
-	texts := mx.bridge.Config.Bridge.GetManagementRoomTexts()
-	_, _ = mx.sendNoticeWithMarkdown(ctx, evt.RoomID, texts.Welcome)
-
-	if len(members.Joined) == 2 && (len(user.GetManagementRoomID()) == 0 || evt.Content.AsMember().IsDirect) {
-		user.SetManagementRoom(evt.RoomID)
-		_, _ = intent.SendNotice(ctx, user.GetManagementRoomID(), "This room has been registered as your bridge management/status room.")
-		zerolog.Ctx(ctx).Debug().Msg("Registered room as management room with inviter")
-	}
-
-	if evt.RoomID == user.GetManagementRoomID() {
-		if user.IsLoggedIn() {
-			_, _ = mx.sendNoticeWithMarkdown(ctx, evt.RoomID, texts.WelcomeConnected)
-		} else {
-			_, _ = mx.sendNoticeWithMarkdown(ctx, evt.RoomID, texts.WelcomeUnconnected)
-		}
-
-		additionalHelp := texts.AdditionalHelp
-		if len(additionalHelp) > 0 {
-			_, _ = mx.sendNoticeWithMarkdown(ctx, evt.RoomID, additionalHelp)
-		}
-	}
-}
-
-func (mx *MatrixHandler) HandleGhostInvite(ctx context.Context, evt *event.Event, inviter User, ghost Ghost) {
-	log := zerolog.Ctx(ctx)
-	intent := ghost.DefaultIntent()
-
-	if inviter.GetPermissionLevel() < bridgeconfig.PermissionLevelUser {
-		log.Debug().Msg("Rejecting invite: inviter is not whitelisted")
-		_, err := intent.LeaveRoom(ctx, evt.RoomID, &mautrix.ReqLeave{
-			Reason: "You're not whitelisted to use this bridge",
-		})
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to reject invite")
-		}
-		return
-	} else if !inviter.IsLoggedIn() {
-		log.Debug().Msg("Rejecting invite: inviter is not logged in")
-		_, err := intent.LeaveRoom(ctx, evt.RoomID, &mautrix.ReqLeave{
-			Reason: "You're not logged into this bridge",
-		})
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to reject invite")
-		}
-		return
-	}
-
-	members := mx.joinAndCheckMembers(ctx, evt, intent)
-	if members == nil {
-		return
-	}
-	var createEvent event.CreateEventContent
-	if err := intent.StateEvent(ctx, evt.RoomID, event.StateCreate, "", &createEvent); err != nil {
-		log.Warn().Err(err).Msg("Failed to check m.room.create event in room")
-	} else if createEvent.Type != "" {
-		log.Warn().Str("room_type", string(createEvent.Type)).Msg("Non-standard room type, leaving room")
-		_, err = intent.LeaveRoom(ctx, evt.RoomID, &mautrix.ReqLeave{
-			Reason: "Unsupported room type",
-		})
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to leave room")
-		}
-		return
-	}
-	var hasBridgeBot, hasOtherUsers bool
-	for mxid, _ := range members.Joined {
-		if mxid == intent.UserID || mxid == inviter.GetMXID() {
-			continue
-		} else if mxid == mx.bridge.Bot.UserID {
-			hasBridgeBot = true
-		} else {
-			hasOtherUsers = true
-		}
-	}
-	if !hasBridgeBot && !hasOtherUsers && evt.Content.AsMember().IsDirect {
-		mx.bridge.Child.CreatePrivatePortal(evt.RoomID, inviter, ghost)
-	} else if !hasBridgeBot {
-		log.Debug().Msg("Leaving multi-user room after accepting invite")
-		_, _ = intent.SendNotice(ctx, evt.RoomID, "Please invite the bridge bot first if you want to bridge to a remote chat.")
-		_, _ = intent.LeaveRoom(ctx, evt.RoomID)
-	} else {
-		_, _ = intent.SendNotice(ctx, evt.RoomID, "This puppet will remain inactive until this room is bridged to a remote chat.")
-	}
-}
-
-func (mx *MatrixHandler) HandleMembership(ctx context.Context, evt *event.Event) {
-	if evt.Sender == mx.bridge.Bot.UserID || mx.bridge.Child.IsGhost(evt.Sender) {
-		return
-	}
-	defer mx.TrackEventDuration(evt.Type)()
-
-	if mx.bridge.Crypto != nil {
-		mx.bridge.Crypto.HandleMemberEvent(ctx, evt)
-	}
-
-	log := mx.log.With().
-		Str("sender", evt.Sender.String()).
-		Str("target", evt.GetStateKey()).
-		Str("room_id", evt.RoomID.String()).
-		Logger()
-	ctx = log.WithContext(ctx)
-
-	content := evt.Content.AsMember()
-	if content.Membership == event.MembershipInvite && id.UserID(evt.GetStateKey()) == mx.as.BotMXID() {
-		mx.HandleBotInvite(ctx, evt)
-		return
-	}
-
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil {
-		return
-	}
-	isSelf := id.UserID(evt.GetStateKey()) == evt.Sender
-	ghost := mx.bridge.Child.GetIGhost(id.UserID(evt.GetStateKey()))
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil {
-		if ghost != nil && content.Membership == event.MembershipInvite {
-			mx.HandleGhostInvite(ctx, evt, user, ghost)
-		}
-		return
-	} else if user.GetPermissionLevel() < bridgeconfig.PermissionLevelUser || !user.IsLoggedIn() {
-		return
-	}
-	bhp, bhpOk := portal.(BanHandlingPortal)
-	mhp, mhpOk := portal.(MembershipHandlingPortal)
-	khp, khpOk := portal.(KnockHandlingPortal)
-	ihp, ihpOk := portal.(InviteHandlingPortal)
-	if !(mhpOk || bhpOk || khpOk) {
-		return
-	}
-	prevContent := &event.MemberEventContent{Membership: event.MembershipLeave}
-	if evt.Unsigned.PrevContent != nil {
-		_ = evt.Unsigned.PrevContent.ParseRaw(evt.Type)
-		prevContent, _ = evt.Unsigned.PrevContent.Parsed.(*event.MemberEventContent)
-	}
-	if ihpOk && prevContent.Membership == event.MembershipInvite && content.Membership != event.MembershipBan {
-		if content.Membership == event.MembershipJoin {
-			ihp.HandleMatrixAcceptInvite(user, evt)
-		}
-		if content.Membership == event.MembershipLeave {
-			if isSelf {
-				ihp.HandleMatrixRejectInvite(user, evt)
-			} else if ghost != nil {
-				ihp.HandleMatrixRetractInvite(user, ghost, evt)
-			}
-		}
-	}
-	if bhpOk && ghost != nil {
-		if content.Membership == event.MembershipBan {
-			bhp.HandleMatrixBan(user, ghost, evt)
-		} else if content.Membership == event.MembershipLeave && prevContent.Membership == event.MembershipBan {
-			bhp.HandleMatrixUnban(user, ghost, evt)
-		}
-	}
-	if khpOk {
-		if content.Membership == event.MembershipKnock {
-			khp.HandleMatrixKnock(user, evt)
-		} else if prevContent.Membership == event.MembershipKnock {
-			if content.Membership == event.MembershipInvite && ghost != nil {
-				khp.HandleMatrixAcceptKnock(user, ghost, evt)
-			} else if content.Membership == event.MembershipLeave {
-				if isSelf {
-					khp.HandleMatrixRetractKnock(user, evt)
-				} else if ghost != nil {
-					khp.HandleMatrixRejectKnock(user, ghost, evt)
-				}
-			}
-		}
-	}
-	if mhpOk {
-		if content.Membership == event.MembershipLeave && prevContent.Membership == event.MembershipJoin {
-			if isSelf {
-				mhp.HandleMatrixLeave(user, evt)
-			} else if ghost != nil {
-				mhp.HandleMatrixKick(user, ghost, evt)
-			}
-		} else if content.Membership == event.MembershipInvite && !isSelf && ghost != nil {
-			mhp.HandleMatrixInvite(user, ghost, evt)
-		}
-	}
-	// TODO kicking/inviting non-ghost users users
-}
-
-func (mx *MatrixHandler) HandleRoomMetadata(ctx context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil {
-		return
-	}
-
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil || portal.IsPrivateChat() {
-		return
-	}
-
-	metaPortal, ok := portal.(MetaHandlingPortal)
-	if !ok {
-		return
-	}
-
-	metaPortal.HandleMatrixMeta(user, evt)
-}
-
-func (mx *MatrixHandler) shouldIgnoreEvent(evt *event.Event) bool {
-	if evt.Sender == mx.bridge.Bot.UserID || mx.bridge.Child.IsGhost(evt.Sender) {
-		return true
-	}
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil || user.GetPermissionLevel() <= 0 {
-		return true
-	} else if val, ok := evt.Content.Raw[appservice.DoublePuppetKey]; ok && val == mx.bridge.Name && user.GetIDoublePuppet() != nil {
-		return true
-	}
-	return false
-}
-
-const initialSessionWaitTimeout = 3 * time.Second
-const extendedSessionWaitTimeout = 22 * time.Second
-
-func (mx *MatrixHandler) sendCryptoStatusError(ctx context.Context, evt *event.Event, editEvent id.EventID, err error, retryCount int, isFinal bool) id.EventID {
-	mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepDecrypted, err, isFinal, retryCount)
-
-	if mx.bridge.Config.Bridge.EnableMessageStatusEvents() {
-		statusEvent := &event.BeeperMessageStatusEventContent{
-			// TODO: network
-			RelatesTo: event.RelatesTo{
-				Type:    event.RelReference,
-				EventID: evt.ID,
-			},
-			Status:  event.MessageStatusRetriable,
-			Reason:  event.MessageStatusUndecryptable,
-			Error:   err.Error(),
-			Message: errorToHumanMessage(err),
-		}
-		if !isFinal {
-			statusEvent.Status = event.MessageStatusPending
-		}
-		_, sendErr := mx.bridge.Bot.SendMessageEvent(ctx, evt.RoomID, event.BeeperMessageStatus, statusEvent)
-		if sendErr != nil {
-			zerolog.Ctx(ctx).Error().Err(err).Msg("Failed to send message status event")
-		}
-	}
-	if mx.bridge.Config.Bridge.EnableMessageErrorNotices() {
-		update := event.MessageEventContent{
-			MsgType: event.MsgNotice,
-			Body:    fmt.Sprintf("\u26a0 Your message was not bridged: %v.", err),
-		}
-		if errors.Is(err, errNoCrypto) {
-			update.Body = "🔒 This bridge has not been configured to support encryption"
-		}
-		relatable, ok := evt.Content.Parsed.(event.Relatable)
-		if editEvent != "" {
-			update.SetEdit(editEvent)
-		} else if ok && relatable.OptionalGetRelatesTo().GetThreadParent() != "" {
-			update.GetRelatesTo().SetThread(relatable.OptionalGetRelatesTo().GetThreadParent(), evt.ID)
-		}
-		resp, sendErr := mx.bridge.Bot.SendMessageEvent(ctx, evt.RoomID, event.EventMessage, &update)
-		if sendErr != nil {
-			zerolog.Ctx(ctx).Error().Err(sendErr).Msg("Failed to send decryption error notice")
-		} else if resp != nil {
-			return resp.EventID
-		}
-	}
-	return ""
-}
-
-var (
-	errDeviceNotTrusted    = errors.New("your device is not trusted")
-	errMessageNotEncrypted = errors.New("unencrypted message")
-	errNoDecryptionKeys    = errors.New("the bridge hasn't received the decryption keys")
-	errNoCrypto            = errors.New("this bridge has not been configured to support encryption")
-)
-
-func errorToHumanMessage(err error) string {
-	var withheld *event.RoomKeyWithheldEventContent
-	switch {
-	case errors.Is(err, errDeviceNotTrusted), errors.Is(err, errNoDecryptionKeys):
-		return err.Error()
-	case errors.Is(err, UnknownMessageIndex):
-		return "the keys received by the bridge can't decrypt the message"
-	case errors.Is(err, DuplicateMessageIndex):
-		return "your client encrypted multiple messages with the same key"
-	case errors.As(err, &withheld):
-		if withheld.Code == event.RoomKeyWithheldBeeperRedacted {
-			return "your client used an outdated encryption session"
-		}
-		return "your client refused to share decryption keys with the bridge"
-	case errors.Is(err, errMessageNotEncrypted):
-		return "the message is not encrypted"
-	default:
-		return "the bridge failed to decrypt the message"
-	}
-}
-
-func deviceUnverifiedErrorWithExplanation(trust id.TrustState) error {
-	var explanation string
-	switch trust {
-	case id.TrustStateBlacklisted:
-		explanation = "device is blacklisted"
-	case id.TrustStateUnset:
-		explanation = "unverified"
-	case id.TrustStateUnknownDevice:
-		explanation = "device info not found"
-	case id.TrustStateForwarded:
-		explanation = "keys were forwarded from an unknown device"
-	case id.TrustStateCrossSignedUntrusted:
-		explanation = "cross-signing keys changed after setting up the bridge"
-	default:
-		return errDeviceNotTrusted
-	}
-	return fmt.Errorf("%w (%s)", errDeviceNotTrusted, explanation)
-}
-
-func copySomeKeys(original, decrypted *event.Event) {
-	isScheduled, _ := original.Content.Raw["com.beeper.scheduled"].(bool)
-	_, alreadyExists := decrypted.Content.Raw["com.beeper.scheduled"]
-	if isScheduled && !alreadyExists {
-		decrypted.Content.Raw["com.beeper.scheduled"] = true
-	}
-}
-
-func (mx *MatrixHandler) postDecrypt(ctx context.Context, original, decrypted *event.Event, retryCount int, errorEventID id.EventID, duration time.Duration) {
-	log := zerolog.Ctx(ctx)
-	minLevel := mx.bridge.Config.Bridge.GetEncryptionConfig().VerificationLevels.Send
-	if decrypted.Mautrix.TrustState < minLevel {
-		logEvt := log.Warn().
-			Str("user_id", decrypted.Sender.String()).
-			Bool("forwarded_keys", decrypted.Mautrix.ForwardedKeys).
-			Stringer("device_trust", decrypted.Mautrix.TrustState).
-			Stringer("min_trust", minLevel)
-		if decrypted.Mautrix.TrustSource != nil {
-			dev := decrypted.Mautrix.TrustSource
-			logEvt.
-				Str("device_id", dev.DeviceID.String()).
-				Str("device_signing_key", dev.SigningKey.String())
-		} else {
-			logEvt.Str("device_id", "unknown")
-		}
-		logEvt.Msg("Dropping event due to insufficient verification level")
-		err := deviceUnverifiedErrorWithExplanation(decrypted.Mautrix.TrustState)
-		go mx.sendCryptoStatusError(ctx, decrypted, errorEventID, err, retryCount, true)
-		return
-	}
-	copySomeKeys(original, decrypted)
-
-	mx.bridge.SendMessageSuccessCheckpoint(decrypted, status.MsgStepDecrypted, retryCount)
-	decrypted.Mautrix.CheckpointSent = true
-	decrypted.Mautrix.DecryptionDuration = duration
-	decrypted.Mautrix.EventSource |= event.SourceDecrypted
-	mx.bridge.EventProcessor.Dispatch(ctx, decrypted)
-	if errorEventID != "" {
-		_, _ = mx.bridge.Bot.RedactEvent(ctx, decrypted.RoomID, errorEventID)
-	}
-}
-
-func (mx *MatrixHandler) HandleEncrypted(ctx context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-	content := evt.Content.AsEncrypted()
-	log := zerolog.Ctx(ctx).With().
-		Str("event_id", evt.ID.String()).
-		Str("session_id", content.SessionID.String()).
-		Logger()
-	ctx = log.WithContext(ctx)
-	if mx.bridge.Crypto == nil {
-		go mx.sendCryptoStatusError(ctx, evt, "", errNoCrypto, 0, true)
-		return
-	}
-	log.Debug().Msg("Decrypting received event")
-
-	decryptionStart := time.Now()
-	decrypted, err := mx.bridge.Crypto.Decrypt(ctx, evt)
-	decryptionRetryCount := 0
-	if errors.Is(err, NoSessionFound) {
-		decryptionRetryCount = 1
-		log.Debug().
-			Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())).
-			Msg("Couldn't find session, waiting for keys to arrive...")
-		mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepDecrypted, err, false, 0)
-		if mx.bridge.Crypto.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
-			log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
-			decrypted, err = mx.bridge.Crypto.Decrypt(ctx, evt)
-		} else {
-			go mx.waitLongerForSession(ctx, evt, decryptionStart)
-			return
-		}
-	}
-	if err != nil {
-		mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepDecrypted, err, true, decryptionRetryCount)
-		log.Warn().Err(err).Msg("Failed to decrypt event")
-		go mx.sendCryptoStatusError(ctx, evt, "", err, decryptionRetryCount, true)
-		return
-	}
-	mx.postDecrypt(ctx, evt, decrypted, decryptionRetryCount, "", time.Since(decryptionStart))
-}
-
-func (mx *MatrixHandler) waitLongerForSession(ctx context.Context, evt *event.Event, decryptionStart time.Time) {
-	log := zerolog.Ctx(ctx)
-	content := evt.Content.AsEncrypted()
-	log.Debug().
-		Int("wait_seconds", int(extendedSessionWaitTimeout.Seconds())).
-		Msg("Couldn't find session, requesting keys and waiting longer...")
-
-	go mx.bridge.Crypto.RequestSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, evt.Sender, content.DeviceID)
-	errorEventID := mx.sendCryptoStatusError(ctx, evt, "", fmt.Errorf("%w. The bridge will retry for %d seconds", errNoDecryptionKeys, int(extendedSessionWaitTimeout.Seconds())), 1, false)
-
-	if !mx.bridge.Crypto.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, extendedSessionWaitTimeout) {
-		log.Debug().Msg("Didn't get session, giving up trying to decrypt event")
-		mx.sendCryptoStatusError(ctx, evt, errorEventID, errNoDecryptionKeys, 2, true)
-		return
-	}
-
-	log.Debug().Msg("Got keys after waiting longer, trying to decrypt event again")
-	decrypted, err := mx.bridge.Crypto.Decrypt(ctx, evt)
-	if err != nil {
-		log.Error().Err(err).Msg("Failed to decrypt event")
-		mx.sendCryptoStatusError(ctx, evt, errorEventID, err, 2, true)
-		return
-	}
-
-	mx.postDecrypt(ctx, evt, decrypted, 2, errorEventID, time.Since(decryptionStart))
-}
-
-func (mx *MatrixHandler) HandleMessage(ctx context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	log := zerolog.Ctx(ctx).With().
-		Str("event_id", evt.ID.String()).
-		Str("room_id", evt.RoomID.String()).
-		Str("sender", evt.Sender.String()).
-		Logger()
-	ctx = log.WithContext(ctx)
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	} else if !evt.Mautrix.WasEncrypted && mx.bridge.Config.Bridge.GetEncryptionConfig().Require {
-		log.Warn().Msg("Dropping unencrypted event")
-		mx.sendCryptoStatusError(ctx, evt, "", errMessageNotEncrypted, 0, true)
-		return
-	}
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil {
-		return
-	}
-
-	content := evt.Content.AsMessage()
-	content.RemoveReplyFallback()
-	if user.GetPermissionLevel() >= bridgeconfig.PermissionLevelUser && content.MsgType == event.MsgText {
-		commandPrefix := mx.bridge.Config.Bridge.GetCommandPrefix()
-		hasCommandPrefix := strings.HasPrefix(content.Body, commandPrefix)
-		if hasCommandPrefix {
-			content.Body = strings.TrimLeft(strings.TrimPrefix(content.Body, commandPrefix), " ")
-		}
-		if hasCommandPrefix || evt.RoomID == user.GetManagementRoomID() {
-			go mx.bridge.CommandProcessor.Handle(ctx, evt.RoomID, evt.ID, user, content.Body, content.RelatesTo.GetReplyTo())
-			go mx.bridge.SendMessageSuccessCheckpoint(evt, status.MsgStepCommand, 0)
-			if mx.bridge.Config.Bridge.EnableMessageStatusEvents() {
-				statusEvent := &event.BeeperMessageStatusEventContent{
-					// TODO: network
-					RelatesTo: event.RelatesTo{
-						Type:    event.RelReference,
-						EventID: evt.ID,
-					},
-					Status: event.MessageStatusSuccess,
-				}
-				_, sendErr := mx.bridge.Bot.SendMessageEvent(ctx, evt.RoomID, event.BeeperMessageStatus, statusEvent)
-				if sendErr != nil {
-					log.Warn().Err(sendErr).Msg("Failed to send message status event for command")
-				}
-			}
-			return
-		}
-	}
-
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal != nil {
-		portal.ReceiveMatrixEvent(user, evt)
-	} else {
-		mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepRemote, fmt.Errorf("unknown room"), true, 0)
-	}
-}
-
-func (mx *MatrixHandler) HandleReaction(_ context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil || user.GetPermissionLevel() < bridgeconfig.PermissionLevelUser || !user.IsLoggedIn() {
-		return
-	}
-
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal != nil {
-		portal.ReceiveMatrixEvent(user, evt)
-	} else {
-		mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepRemote, fmt.Errorf("unknown room"), true, 0)
-	}
-}
-
-func (mx *MatrixHandler) HandleRedaction(_ context.Context, evt *event.Event) {
-	defer mx.TrackEventDuration(evt.Type)()
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-
-	user := mx.bridge.Child.GetIUser(evt.Sender, true)
-	if user == nil {
-		return
-	}
-
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal != nil {
-		portal.ReceiveMatrixEvent(user, evt)
-	} else {
-		mx.bridge.SendMessageErrorCheckpoint(evt, status.MsgStepRemote, fmt.Errorf("unknown room"), true, 0)
-	}
-}
-
-func (mx *MatrixHandler) HandleReceipt(_ context.Context, evt *event.Event) {
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil {
-		return
-	}
-
-	rrPortal, ok := portal.(ReadReceiptHandlingPortal)
-	if !ok {
-		return
-	}
-
-	for eventID, receipts := range *evt.Content.AsReceipt() {
-		for userID, receipt := range receipts[event.ReceiptTypeRead] {
-			user := mx.bridge.Child.GetIUser(userID, false)
-			if user == nil {
-				// Not a bridge user
-				continue
-			}
-			customPuppet := user.GetIDoublePuppet()
-			if val, ok := receipt.Extra[appservice.DoublePuppetKey].(string); ok && customPuppet != nil && val == mx.bridge.Name {
-				// Ignore double puppeted read receipts.
-				mx.log.Debug().Interface("content", evt.Content.Raw).Msg("Ignoring double-puppeted read receipt")
-				// But do start disappearing messages, because the user read the chat
-				dp, ok := portal.(DisappearingPortal)
-				if ok {
-					dp.ScheduleDisappearing()
-				}
-			} else {
-				rrPortal.HandleMatrixReadReceipt(user, eventID, receipt)
-			}
-		}
-	}
-}
-
-func (mx *MatrixHandler) HandleTyping(_ context.Context, evt *event.Event) {
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil {
-		return
-	}
-	typingPortal, ok := portal.(TypingPortal)
-	if !ok {
-		return
-	}
-	typingPortal.HandleMatrixTyping(evt.Content.AsTyping().UserIDs)
-}
-
-func (mx *MatrixHandler) HandlePowerLevels(_ context.Context, evt *event.Event) {
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil {
-		return
-	}
-	powerLevelPortal, ok := portal.(PowerLevelHandlingPortal)
-	if ok {
-		user := mx.bridge.Child.GetIUser(evt.Sender, true)
-		powerLevelPortal.HandleMatrixPowerLevels(user, evt)
-	}
-}
-
-func (mx *MatrixHandler) HandleJoinRule(_ context.Context, evt *event.Event) {
-	if mx.shouldIgnoreEvent(evt) {
-		return
-	}
-	portal := mx.bridge.Child.GetIPortal(evt.RoomID)
-	if portal == nil {
-		return
-	}
-	joinRulePortal, ok := portal.(JoinRuleHandlingPortal)
-	if ok {
-		user := mx.bridge.Child.GetIUser(evt.Sender, true)
-		joinRulePortal.HandleMatrixJoinRule(user, evt)
-	}
-}

bridge/messagecheckpoint.go

@@ -1,61 +0,0 @@
-// Copyright (c) 2021 Sumner Evans
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package bridge
-
-import (
-	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/status"
-	"maunium.net/go/mautrix/event"
-)
-
-func (br *Bridge) SendMessageSuccessCheckpoint(evt *event.Event, step status.MessageCheckpointStep, retryNum int) {
-	br.SendMessageCheckpoint(evt, step, nil, status.MsgStatusSuccess, retryNum)
-}
-
-func (br *Bridge) SendMessageErrorCheckpoint(evt *event.Event, step status.MessageCheckpointStep, err error, permanent bool, retryNum int) {
-	s := status.MsgStatusWillRetry
-	if permanent {
-		s = status.MsgStatusPermFailure
-	}
-	br.SendMessageCheckpoint(evt, step, err, s, retryNum)
-}
-
-func (br *Bridge) SendMessageCheckpoint(evt *event.Event, step status.MessageCheckpointStep, err error, s status.MessageCheckpointStatus, retryNum int) {
-	checkpoint := status.NewMessageCheckpoint(evt, step, s, retryNum)
-	if err != nil {
-		checkpoint.Info = err.Error()
-	}
-	go br.SendRawMessageCheckpoint(checkpoint)
-}
-
-func (br *Bridge) SendRawMessageCheckpoint(cp *status.MessageCheckpoint) {
-	err := br.SendMessageCheckpoints([]*status.MessageCheckpoint{cp})
-	if err != nil {
-		br.ZLog.Warn().Err(err).Interface("message_checkpoint", cp).Msg("Error sending message checkpoint")
-	} else {
-		br.ZLog.Debug().Interface("message_checkpoint", cp).Msg("Sent message checkpoint")
-	}
-}
-
-func (br *Bridge) SendMessageCheckpoints(checkpoints []*status.MessageCheckpoint) error {
-	checkpointsJSON := status.CheckpointsJSON{Checkpoints: checkpoints}
-
-	if br.Websocket {
-		return br.AS.SendWebsocket(&appservice.WebsocketRequest{
-			Command: "message_checkpoint",
-			Data:    checkpointsJSON,
-		})
-	}
-
-	endpoint := br.Config.Homeserver.MessageSendCheckpointEndpoint
-	if endpoint == "" {
-		return nil
-	}
-
-	return checkpointsJSON.SendHTTP(endpoint, br.AS.Registration.AppToken)
-}

bridge/no-crypto.go

@@ -1,26 +0,0 @@
-// Copyright (c) 2023 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:build !cgo || nocrypto
-
-package bridge
-
-import (
-	"errors"
-)
-
-func NewCryptoHelper(bridge *Bridge) Crypto {
-	if bridge.Config.Bridge.GetEncryptionConfig().Allow {
-		bridge.ZLog.Warn().Msg("Bridge built without end-to-bridge encryption, but encryption is enabled in config")
-	} else {
-		bridge.ZLog.Debug().Msg("Bridge built without end-to-bridge encryption")
-	}
-	return nil
-}
-
-var NoSessionFound = errors.New("nil")
-var UnknownMessageIndex = NoSessionFound
-var DuplicateMessageIndex = NoSessionFound

bridge/websocket.go

@@ -1,163 +0,0 @@
-package bridge
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"time"
-
-	"go.mau.fi/util/jsontime"
-
-	"maunium.net/go/mautrix/appservice"
-)
-
-const defaultReconnectBackoff = 2 * time.Second
-const maxReconnectBackoff = 2 * time.Minute
-const reconnectBackoffReset = 5 * time.Minute
-
-func (br *Bridge) startWebsocket(wg *sync.WaitGroup) {
-	log := br.ZLog.With().Str("action", "appservice websocket").Logger()
-	var wgOnce sync.Once
-	onConnect := func() {
-		wssBr, ok := br.Child.(WebsocketStartingBridge)
-		if ok {
-			wssBr.OnWebsocketConnect()
-		}
-		if br.latestState != nil {
-			go func() {
-				ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-				defer cancel()
-				br.latestState.Timestamp = jsontime.UnixNow()
-				err := br.SendBridgeState(ctx, br.latestState)
-				if err != nil {
-					log.Err(err).Msg("Failed to resend latest bridge state after websocket reconnect")
-				} else {
-					log.Debug().Any("bridge_state", br.latestState).Msg("Resent bridge state after websocket reconnect")
-				}
-			}()
-		}
-		wgOnce.Do(wg.Done)
-		select {
-		case br.wsStarted <- struct{}{}:
-		default:
-		}
-	}
-	reconnectBackoff := defaultReconnectBackoff
-	lastDisconnect := time.Now().UnixNano()
-	br.wsStopped = make(chan struct{})
-	defer func() {
-		log.Debug().Msg("Appservice websocket loop finished")
-		close(br.wsStopped)
-	}()
-	addr := br.Config.Homeserver.WSProxy
-	if addr == "" {
-		addr = br.Config.Homeserver.Address
-	}
-	for {
-		err := br.AS.StartWebsocket(addr, onConnect)
-		if errors.Is(err, appservice.ErrWebsocketManualStop) {
-			return
-		} else if closeCommand := (&appservice.CloseCommand{}); errors.As(err, &closeCommand) && closeCommand.Status == appservice.MeowConnectionReplaced {
-			log.Info().Msg("Appservice websocket closed by another instance of the bridge, shutting down...")
-			br.ManualStop(0)
-			return
-		} else if err != nil {
-			log.Err(err).Msg("Error in appservice websocket")
-		}
-		if br.Stopping {
-			return
-		}
-		now := time.Now().UnixNano()
-		if lastDisconnect+reconnectBackoffReset.Nanoseconds() < now {
-			reconnectBackoff = defaultReconnectBackoff
-		} else {
-			reconnectBackoff *= 2
-			if reconnectBackoff > maxReconnectBackoff {
-				reconnectBackoff = maxReconnectBackoff
-			}
-		}
-		lastDisconnect = now
-		log.Info().
-			Int("backoff_seconds", int(reconnectBackoff.Seconds())).
-			Msg("Websocket disconnected, reconnecting...")
-		select {
-		case <-br.wsShortCircuitReconnectBackoff:
-			log.Debug().Msg("Reconnect backoff was short-circuited")
-		case <-time.After(reconnectBackoff):
-		}
-		if br.Stopping {
-			return
-		}
-	}
-}
-
-type wsPingData struct {
-	Timestamp int64 `json:"timestamp"`
-}
-
-func (br *Bridge) PingServer() (start, serverTs, end time.Time) {
-	if !br.Websocket {
-		panic(fmt.Errorf("PingServer called without websocket enabled"))
-	}
-	if !br.AS.HasWebsocket() {
-		br.ZLog.Debug().Msg("Received server ping request, but no websocket connected. Trying to short-circuit backoff sleep")
-		select {
-		case br.wsShortCircuitReconnectBackoff <- struct{}{}:
-		default:
-			br.ZLog.Warn().Msg("Failed to ping websocket: not connected and no backoff?")
-			return
-		}
-		select {
-		case <-br.wsStarted:
-		case <-time.After(15 * time.Second):
-			if !br.AS.HasWebsocket() {
-				br.ZLog.Warn().Msg("Failed to ping websocket: didn't connect after 15 seconds of waiting")
-				return
-			}
-		}
-	}
-	start = time.Now()
-	var resp wsPingData
-	br.ZLog.Debug().Msg("Pinging appservice websocket")
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	err := br.AS.RequestWebsocket(ctx, &appservice.WebsocketRequest{
-		Command: "ping",
-		Data:    &wsPingData{Timestamp: start.UnixMilli()},
-	}, &resp)
-	end = time.Now()
-	if err != nil {
-		br.ZLog.Warn().Err(err).Dur("duration", end.Sub(start)).Msg("Websocket ping returned error")
-		br.AS.StopWebsocket(fmt.Errorf("websocket ping returned error in %s: %w", end.Sub(start), err))
-	} else {
-		serverTs = time.Unix(0, resp.Timestamp*int64(time.Millisecond))
-		br.ZLog.Debug().
-			Dur("duration", end.Sub(start)).
-			Dur("req_duration", serverTs.Sub(start)).
-			Dur("resp_duration", end.Sub(serverTs)).
-			Msg("Websocket ping returned success")
-	}
-	return
-}
-
-func (br *Bridge) websocketServerPinger() {
-	interval := time.Duration(br.Config.Homeserver.WSPingInterval) * time.Second
-	clock := time.NewTicker(interval)
-	defer func() {
-		br.ZLog.Info().Msg("Stopping websocket pinger")
-		clock.Stop()
-	}()
-	br.ZLog.Info().Dur("interval_duration", interval).Msg("Starting websocket pinger")
-	for {
-		select {
-		case <-clock.C:
-			br.PingServer()
-		case <-br.wsStopPinger:
-			return
-		}
-		if br.Stopping {
-			return
-		}
-	}
-}

bridgev2/backfillqueue.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package bridgev2
+
+import (
+	"context"
+	"fmt"
+	"runtime/debug"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	"maunium.net/go/mautrix/bridgev2/database"
+)
+
+const BackfillMinBackoffAfterRoomCreate = 1 * time.Minute
+const BackfillQueueErrorBackoff = 1 * time.Minute
+const BackfillQueueMaxEmptyBackoff = 10 * time.Minute
+
+func (br *Bridge) WakeupBackfillQueue() {
+	select {
+	case br.wakeupBackfillQueue <- struct{}{}:
+	default:
+	}
+}
+
+func (br *Bridge) RunBackfillQueue() {
+	if !br.Config.Backfill.Queue.Enabled || !br.Config.Backfill.Enabled {
+		return
+	}
+	log := br.Log.With().Str("component", "backfill queue").Logger()
+	if !br.Matrix.GetCapabilities().BatchSending {
+		log.Warn().Msg("Backfill queue is enabled in config, but Matrix server doesn't support batch sending")
+		return
+	}
+	ctx, cancel := context.WithCancel(log.WithContext(context.Background()))
+	br.stopBackfillQueue.Clear()
+	stopChan := br.stopBackfillQueue.GetChan()
+	go func() {
+		<-stopChan
+		cancel()
+	}()
+	batchDelay := time.Duration(br.Config.Backfill.Queue.BatchDelay) * time.Second
+	log.Info().Stringer("batch_delay", batchDelay).Msg("Backfill queue starting")
+	noTasksFoundCount := 0
+	for {
+		nextDelay := batchDelay
+		if noTasksFoundCount > 0 {
+			extraDelay := batchDelay * time.Duration(noTasksFoundCount)
+			nextDelay += min(BackfillQueueMaxEmptyBackoff, extraDelay)
+		}
+		timer := time.NewTimer(nextDelay)
+		select {
+		case <-br.wakeupBackfillQueue:
+			if !timer.Stop() {
+				select {
+				case <-timer.C:
+				default:
+				}
+			}
+			noTasksFoundCount = 0
+		case <-stopChan:
+			if !timer.Stop() {
+				select {
+				case <-timer.C:
+				default:
+				}
+			}
+			log.Info().Msg("Stopping backfill queue")
+			return
+		case <-timer.C:
+		}
+		backfillTask, err := br.DB.BackfillTask.GetNext(ctx)
+		if err != nil {
+			log.Err(err).Msg("Failed to get next backfill queue entry")
+			time.Sleep(BackfillQueueErrorBackoff)
+			continue
+		} else if backfillTask != nil {
+			br.DoBackfillTask(ctx, backfillTask)
+			noTasksFoundCount = 0
+		}
+	}
+}
+
+func (br *Bridge) DoBackfillTask(ctx context.Context, task *database.BackfillTask) {
+	log := zerolog.Ctx(ctx).With().
+		Object("portal_key", task.PortalKey).
+		Str("login_id", string(task.UserLoginID)).
+		Logger()
+	defer func() {
+		err := recover()
+		if err != nil {
+			logEvt := log.Error().
+				Bytes(zerolog.ErrorStackFieldName, debug.Stack())
+			if realErr, ok := err.(error); ok {
+				logEvt = logEvt.Err(realErr)
+			} else {
+				logEvt = logEvt.Any(zerolog.ErrorFieldName, err)
+			}
+			logEvt.Msg("Panic in backfill queue")
+		}
+	}()
+	ctx = log.WithContext(ctx)
+	err := br.DB.BackfillTask.MarkDispatched(ctx, task)
+	if err != nil {
+		log.Err(err).Msg("Failed to mark backfill task as dispatched")
+		time.Sleep(BackfillQueueErrorBackoff)
+		return
+	}
+	completed, err := br.actuallyDoBackfillTask(ctx, task)
+	if err != nil {
+		log.Err(err).Msg("Failed to do backfill task")
+		time.Sleep(BackfillQueueErrorBackoff)
+		return
+	} else if completed {
+		log.Info().
+			Int("batch_count", task.BatchCount).
+			Bool("is_done", task.IsDone).
+			Msg("Backfill task completed successfully")
+	} else {
+		log.Info().
+			Int("batch_count", task.BatchCount).
+			Bool("is_done", task.IsDone).
+			Msg("Backfill task canceled")
+	}
+	err = br.DB.BackfillTask.Update(ctx, task)
+	if err != nil {
+		log.Err(err).Msg("Failed to update backfill task")
+		time.Sleep(BackfillQueueErrorBackoff)
+	}
+}
+
+func (portal *Portal) deleteBackfillQueueTaskIfRoomDoesNotExist(ctx context.Context) bool {
+	// Acquire the room create lock to ensure that task deletion doesn't race with room creation
+	portal.roomCreateLock.Lock()
+	defer portal.roomCreateLock.Unlock()
+	if portal.MXID == "" {
+		zerolog.Ctx(ctx).Debug().Msg("Portal for backfill task doesn't exist, deleting entry")
+		err := portal.Bridge.DB.BackfillTask.Delete(ctx, portal.PortalKey)
+		if err != nil {
+			zerolog.Ctx(ctx).Err(err).Msg("Failed to delete backfill task after portal wasn't found")
+		}
+		return true
+	}
+	return false
+}
+
+func (br *Bridge) actuallyDoBackfillTask(ctx context.Context, task *database.BackfillTask) (bool, error) {
+	log := zerolog.Ctx(ctx)
+	portal, err := br.GetExistingPortalByKey(ctx, task.PortalKey)
+	if err != nil {
+		return false, fmt.Errorf("failed to get portal for backfill task: %w", err)
+	} else if portal == nil {
+		log.Warn().Msg("Portal not found for backfill task")
+		err = br.DB.BackfillTask.Delete(ctx, task.PortalKey)
+		if err != nil {
+			log.Err(err).Msg("Failed to delete backfill task after portal wasn't found")
+			time.Sleep(BackfillQueueErrorBackoff)
+		}
+		return false, nil
+	} else if portal.MXID == "" {
+		portal.deleteBackfillQueueTaskIfRoomDoesNotExist(ctx)
+		return false, nil
+	}
+	login, err := br.GetExistingUserLoginByID(ctx, task.UserLoginID)
+	if err != nil {
+		return false, fmt.Errorf("failed to get user login for backfill task: %w", err)
+	} else if login == nil || !login.Client.IsLoggedIn() {
+		if login == nil {
+			log.Warn().Msg("User login not found for backfill task")
+		} else {
+			log.Warn().Msg("User login not logged in for backfill task")
+		}
+		logins, err := br.GetUserLoginsInPortal(ctx, portal.PortalKey)
+		if err != nil {
+			return false, fmt.Errorf("failed to get user portals for backfill task: %w", err)
+		} else if len(logins) == 0 {
+			log.Debug().Msg("No user logins found for backfill task")
+			task.NextDispatchMinTS = database.BackfillNextDispatchNever
+			if login == nil {
+				task.UserLoginID = ""
+			}
+			return false, nil
+		}
+		if login == nil {
+			task.UserLoginID = ""
+		}
+		foundLogin := false
+		for _, login = range logins {
+			if login.Client.IsLoggedIn() {
+				foundLogin = true
+				task.UserLoginID = login.ID
+				log.UpdateContext(func(c zerolog.Context) zerolog.Context {
+					return c.Str("overridden_login_id", string(login.ID))
+				})
+				log.Debug().Msg("Found user login for backfill task")
+				break
+			}
+		}
+		if !foundLogin {
+			log.Debug().Msg("No logged in user logins found for backfill task")
+			task.NextDispatchMinTS = database.BackfillNextDispatchNever
+			return false, nil
+		}
+	}
+	if task.BatchCount < 0 {
+		var msgCount int
+		msgCount, err = br.DB.Message.CountMessagesInPortal(ctx, task.PortalKey)
+		if err != nil {
+			return false, fmt.Errorf("failed to count messages in portal: %w", err)
+		}
+		task.BatchCount = msgCount / br.Config.Backfill.Queue.BatchSize
+		log.Debug().
+			Int("message_count", msgCount).
+			Int("batch_count", task.BatchCount).
+			Msg("Calculated existing batch count")
+	}
+	maxBatches := br.Config.Backfill.Queue.MaxBatches
+	api, ok := login.Client.(BackfillingNetworkAPI)
+	if !ok {
+		return false, fmt.Errorf("network API does not support backfilling")
+	}
+	limiterAPI, ok := api.(BackfillingNetworkAPIWithLimits)
+	if ok {
+		maxBatches = limiterAPI.GetBackfillMaxBatchCount(ctx, portal, task)
+	}
+	if maxBatches < 0 || maxBatches > task.BatchCount {
+		err = portal.DoBackwardsBackfill(ctx, login, task)
+		if err != nil {
+			return false, fmt.Errorf("failed to backfill: %w", err)
+		}
+		task.BatchCount++
+	} else {
+		log.Debug().
+			Int("max_batches", maxBatches).
+			Int("batch_count", task.BatchCount).
+			Msg("Not actually backfilling: max batches reached")
+	}
+	task.IsDone = task.IsDone || (maxBatches > 0 && task.BatchCount >= maxBatches)
+	batchDelay := time.Duration(br.Config.Backfill.Queue.BatchDelay) * time.Second
+	task.CompletedAt = time.Now()
+	task.NextDispatchMinTS = task.CompletedAt.Add(batchDelay)
+	return true, nil
+}

bridgev2/bridge.go

@@ -8,22 +8,24 @@ package bridgev2
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"os"
 	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/rs/zerolog"
 	"go.mau.fi/util/dbutil"
+	"go.mau.fi/util/exhttp"
+	"go.mau.fi/util/exsync"
 
-	"maunium.net/go/mautrix/bridge/status"
 	"maunium.net/go/mautrix/bridgev2/bridgeconfig"
 	"maunium.net/go/mautrix/bridgev2/database"
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/id"
 )
 
-var ErrNotLoggedIn = errors.New("not logged in")
-
 type CommandProcessor interface {
 	Handle(ctx context.Context, roomID id.RoomID, eventID id.EventID, user *User, message string, replyTo id.EventID)
 }
@@ -47,6 +49,18 @@ type Bridge struct {
 	portalsByMXID  map[id.RoomID]*Portal
 	ghostsByID     map[networkid.UserID]*Ghost
 	cacheLock      sync.Mutex
+
+	didSplitPortals bool
+
+	Background          bool
+	ExternallyManagedDB bool
+	stopping            atomic.Bool
+
+	wakeupBackfillQueue chan struct{}
+	stopBackfillQueue   *exsync.Event
+
+	BackgroundCtx       context.Context
+	cancelBackgroundCtx context.CancelFunc
 }
 
 func NewBridge(
@@ -72,6 +86,9 @@ func NewBridge(
 		portalsByKey:   make(map[networkid.PortalKey]*Portal),
 		portalsByMXID:  make(map[id.RoomID]*Portal),
 		ghostsByID:     make(map[networkid.UserID]*Ghost),
+
+		wakeupBackfillQueue: make(chan struct{}),
+		stopBackfillQueue:   exsync.NewEvent(),
 	}
 	if br.Config == nil {
 		br.Config = &bridgeconfig.BridgeConfig{CommandPrefix: "!bridge"}
@@ -97,16 +114,89 @@ func (e DBUpgradeError) Unwrap() error {
 	return e.Err
 }
 
-func (br *Bridge) Start() error {
-	br.Log.Info().Msg("Starting bridge")
-	ctx := br.Log.WithContext(context.Background())
-
-	err := br.DB.Upgrade(ctx)
+func (br *Bridge) Start(ctx context.Context) error {
+	ctx = br.Log.WithContext(ctx)
+	err := br.StartConnectors(ctx)
 	if err != nil {
-		return DBUpgradeError{Err: err, Section: "main"}
+		return err
+	}
+	err = br.StartLogins(ctx)
+	if err != nil {
+		return err
+	}
+	go br.PostStart(ctx)
+	return nil
+}
+
+func (br *Bridge) RunOnce(ctx context.Context, loginID networkid.UserLoginID, params *ConnectBackgroundParams) error {
+	br.Background = true
+	br.stopping.Store(false)
+	err := br.StartConnectors(ctx)
+	if err != nil {
+		return err
+	}
+
+	if loginID == "" {
+		br.Log.Info().Msg("No login ID provided to RunOnce, running all logins for 20 seconds")
+		err = br.StartLogins(ctx)
+		if err != nil {
+			return err
+		}
+		defer br.StopWithTimeout(5 * time.Second)
+		select {
+		case <-time.After(20 * time.Second):
+		case <-ctx.Done():
+		}
+		return nil
+	}
+
+	defer br.stop(true, 5*time.Second)
+	login, err := br.GetExistingUserLoginByID(ctx, loginID)
+	if err != nil {
+		return fmt.Errorf("failed to get user login: %w", err)
+	} else if login == nil {
+		return ErrNotLoggedIn
+	}
+	syncClient, ok := login.Client.(BackgroundSyncingNetworkAPI)
+	if !ok {
+		br.Log.Warn().Msg("Network connector doesn't implement background mode, using fallback mechanism for RunOnce")
+		login.Client.Connect(ctx)
+		defer login.DisconnectWithTimeout(5 * time.Second)
+		select {
+		case <-time.After(20 * time.Second):
+		case <-ctx.Done():
+		}
+		br.stopping.Store(true)
+		return nil
+	} else {
+		br.Log.Info().Str("user_login_id", string(login.ID)).Msg("Starting individual user login in background mode")
+		return syncClient.ConnectBackground(login.Log.WithContext(ctx), params)
+	}
+}
+
+func (br *Bridge) StartConnectors(ctx context.Context) error {
+	br.Log.Info().Msg("Starting bridge")
+	br.stopping.Store(false)
+	if br.BackgroundCtx == nil || br.BackgroundCtx.Err() != nil {
+		br.BackgroundCtx, br.cancelBackgroundCtx = context.WithCancel(context.Background())
+		br.BackgroundCtx = br.Log.WithContext(br.BackgroundCtx)
+	}
+
+	if !br.ExternallyManagedDB {
+		err := br.DB.Upgrade(ctx)
+		if err != nil {
+			return DBUpgradeError{Err: err, Section: "main"}
+		}
+	}
+	if !br.Background {
+		var postMigrate func()
+		br.didSplitPortals, postMigrate = br.MigrateToSplitPortals(ctx)
+		if postMigrate != nil {
+			defer postMigrate()
+		}
 	}
 	br.Log.Info().Msg("Starting Matrix connector")
-	err = br.Matrix.Start(ctx)
+	err := br.Matrix.Start(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to start Matrix connector: %w", err)
 	}
@@ -115,10 +205,144 @@ func (br *Bridge) Start() error {
 	if err != nil {
 		return fmt.Errorf("failed to start network connector: %w", err)
 	}
-	if br.Network.GetCapabilities().DisappearingMessages {
+	if br.Network.GetCapabilities().DisappearingMessages && !br.Background {
 		go br.DisappearLoop.Start()
 	}
+	return nil
+}
 
+func (br *Bridge) PostStart(ctx context.Context) {
+	if br.Background {
+		return
+	}
+	rawBridgeInfoVer := br.DB.KV.Get(ctx, database.KeyBridgeInfoVersion)
+	bridgeInfoVer, capVer, err := parseBridgeInfoVersion(rawBridgeInfoVer)
+	if err != nil {
+		br.Log.Err(err).Str("db_bridge_info_version", rawBridgeInfoVer).Msg("Failed to parse bridge info version")
+		return
+	}
+	expectedBridgeInfoVer, expectedCapVer := br.Network.GetBridgeInfoVersion()
+	doResendBridgeInfo := bridgeInfoVer != expectedBridgeInfoVer || br.didSplitPortals || br.Config.ResendBridgeInfo
+	doResendCapabilities := capVer != expectedCapVer || br.didSplitPortals
+	if doResendBridgeInfo || doResendCapabilities {
+		br.ResendBridgeInfo(ctx, doResendBridgeInfo, doResendCapabilities)
+	}
+	br.DB.KV.Set(ctx, database.KeyBridgeInfoVersion, fmt.Sprintf("%d,%d", expectedBridgeInfoVer, expectedCapVer))
+}
+
+func parseBridgeInfoVersion(version string) (info, capabilities int, err error) {
+	_, err = fmt.Sscanf(version, "%d,%d", &info, &capabilities)
+	if version == "" {
+		err = nil
+	}
+	return
+}
+
+func (br *Bridge) ResendBridgeInfo(ctx context.Context, resendInfo, resendCaps bool) {
+	log := zerolog.Ctx(ctx).With().Str("action", "resend bridge info").Logger()
+	portals, err := br.GetAllPortalsWithMXID(ctx)
+	if err != nil {
+		log.Err(err).Msg("Failed to get portals")
+		return
+	}
+	for _, portal := range portals {
+		if resendInfo {
+			portal.UpdateBridgeInfo(ctx)
+		}
+		if resendCaps {
+			logins, err := br.GetUserLoginsInPortal(ctx, portal.PortalKey)
+			if err != nil {
+				log.Err(err).
+					Stringer("room_id", portal.MXID).
+					Object("portal_key", portal.PortalKey).
+					Msg("Failed to get user logins in portal")
+			} else {
+				found := false
+				for _, login := range logins {
+					if portal.CapState.ID == "" || login.ID == portal.CapState.Source {
+						portal.UpdateCapabilities(ctx, login, true)
+						found = true
+					}
+				}
+				if !found && len(logins) > 0 {
+					portal.CapState.Source = ""
+					portal.UpdateCapabilities(ctx, logins[0], true)
+				} else if !found {
+					log.Warn().
+						Stringer("room_id", portal.MXID).
+						Object("portal_key", portal.PortalKey).
+						Msg("No user login found to update capabilities")
+				}
+			}
+		}
+	}
+	log.Info().
+		Bool("capabilities", resendCaps).
+		Bool("info", resendInfo).
+		Msg("Resent bridge info to all portals")
+}
+
+func (br *Bridge) MigrateToSplitPortals(ctx context.Context) (bool, func()) {
+	log := zerolog.Ctx(ctx).With().Str("action", "migrate to split portals").Logger()
+	ctx = log.WithContext(ctx)
+	if !br.Config.SplitPortals || br.DB.KV.Get(ctx, database.KeySplitPortalsEnabled) == "true" {
+		return false, nil
+	}
+	affected, err := br.DB.Portal.MigrateToSplitPortals(ctx)
+	if err != nil {
+		log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to migrate portals")
+		os.Exit(31)
+		return false, nil
+	}
+	log.Info().Int64("rows_affected", affected).Msg("Migrated to split portals")
+	affected2, err := br.DB.Portal.FixParentsAfterSplitPortalMigration(ctx)
+	if err != nil {
+		log.Err(err).Msg("Failed to fix parent portals after split portal migration")
+		os.Exit(31)
+		return false, nil
+	}
+	log.Info().Int64("rows_affected", affected2).Msg("Updated parent receivers after split portal migration")
+	withoutReceiver, err := br.DB.Portal.GetAllWithoutReceiver(ctx)
+	if err != nil {
+		log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to get portals that failed to migrate")
+		os.Exit(31)
+		return false, nil
+	}
+	var roomsToDelete []id.RoomID
+	log.Info().Int("remaining_portals", len(withoutReceiver)).Msg("Deleting remaining portals without receiver")
+	for _, portal := range withoutReceiver {
+		if err = br.DB.Portal.Delete(ctx, portal.PortalKey); err != nil {
+			log.Err(err).
+				Str("portal_id", string(portal.ID)).
+				Stringer("mxid", portal.MXID).
+				Msg("Failed to delete portal database row that failed to migrate")
+		} else if portal.MXID != "" {
+			log.Debug().
+				Str("portal_id", string(portal.ID)).
+				Stringer("mxid", portal.MXID).
+				Msg("Marked portal room for deletion from homeserver")
+			roomsToDelete = append(roomsToDelete, portal.MXID)
+		} else {
+			log.Debug().
+				Str("portal_id", string(portal.ID)).
+				Msg("Deleted portal row with no Matrix room")
+		}
+	}
+	br.DB.KV.Set(ctx, database.KeySplitPortalsEnabled, "true")
+	log.Info().Msg("Finished split portal migration successfully")
+	return affected > 0, func() {
+		for _, roomID := range roomsToDelete {
+			if err = br.Bot.DeleteRoom(ctx, roomID, true); err != nil {
+				log.Err(err).
+					Stringer("mxid", roomID).
+					Msg("Failed to delete portal room that failed to migrate")
+			}
+		}
+		log.Info().Int("room_count", len(roomsToDelete)).Msg("Finished deleting rooms that failed to migrate")
+	}
+}
+
+func (br *Bridge) StartLogins(ctx context.Context) error {
 	userIDs, err := br.DB.UserLogin.GetAllUserIDsWithLogins(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to get users with logins: %w", err)
@@ -131,13 +355,10 @@ func (br *Bridge) Start() error {
 		if err != nil {
 			br.Log.Err(err).Stringer("user_id", userID).Msg("Failed to load user")
 		} else {
-			for _, login := range user.GetCachedUserLogins() {
+			for _, login := range user.GetUserLogins() {
 				startedAny = true
 				br.Log.Info().Str("id", string(login.ID)).Msg("Starting user login")
-				err = login.Client.Connect(login.Log.WithContext(ctx))
-				if err != nil {
-					br.Log.Err(err).Msg("Failed to connect existing client")
-				}
+				login.Client.Connect(login.Log.WithContext(ctx))
 			}
 		}
 	}
@@ -145,25 +366,93 @@ func (br *Bridge) Start() error {
 		br.Log.Info().Msg("No user logins found")
 		br.SendGlobalBridgeState(status.BridgeState{StateEvent: status.StateUnconfigured})
 	}
+	if !br.Background {
+		go br.RunBackfillQueue()
+	}
 
 	br.Log.Info().Msg("Bridge started")
 	return nil
 }
 
-func (br *Bridge) Stop() {
-	br.Log.Info().Msg("Shutting down bridge")
-	br.Matrix.Stop()
-	br.cacheLock.Lock()
-	var wg sync.WaitGroup
-	wg.Add(len(br.userLoginsByID))
-	for _, login := range br.userLoginsByID {
-		go login.Disconnect(wg.Done)
+func (br *Bridge) ResetNetworkConnections() {
+	nrn, ok := br.Network.(NetworkResettingNetwork)
+	if ok {
+		br.Log.Info().Msg("Resetting network connections with NetworkConnector.ResetNetworkConnections")
+		nrn.ResetNetworkConnections()
+		return
 	}
-	wg.Wait()
-	br.cacheLock.Unlock()
-	err := br.DB.Close()
-	if err != nil {
-		br.Log.Warn().Err(err).Msg("Failed to close database")
+
+	br.Log.Info().Msg("Network connector doesn't support ResetNetworkConnections, recreating clients manually")
+	for _, login := range br.GetAllCachedUserLogins() {
+		login.Log.Debug().Msg("Disconnecting and recreating client for network reset")
+		ctx := login.Log.WithContext(br.BackgroundCtx)
+		login.Client.Disconnect()
+		err := login.recreateClient(ctx)
+		if err != nil {
+			login.Log.Err(err).Msg("Failed to recreate client during network reset")
+			login.BridgeState.Send(status.BridgeState{
+				StateEvent: status.StateUnknownError,
+				Error:      "bridgev2-network-reset-fail",
+				Info:       map[string]any{"go_error": err.Error()},
+			})
+		} else {
+			login.Client.Connect(ctx)
+		}
+	}
+	br.Log.Info().Msg("Finished resetting all user logins")
+}
+
+func (br *Bridge) GetHTTPClientSettings() exhttp.ClientSettings {
+	mchs, ok := br.Matrix.(MatrixConnectorWithHTTPSettings)
+	if ok {
+		return mchs.GetHTTPClientSettings()
+	}
+	return exhttp.SensibleClientSettings
+}
+
+func (br *Bridge) IsStopping() bool {
+	return br.stopping.Load()
+}
+
+func (br *Bridge) Stop() {
+	br.stop(false, 0)
+}
+
+func (br *Bridge) StopWithTimeout(timeout time.Duration) {
+	br.stop(false, timeout)
+}
+
+func (br *Bridge) stop(isRunOnce bool, timeout time.Duration) {
+	br.Log.Info().Msg("Shutting down bridge")
+	br.stopping.Store(true)
+	br.DisappearLoop.Stop()
+	br.stopBackfillQueue.Set()
+	br.Matrix.PreStop()
+	if !isRunOnce {
+		br.cacheLock.Lock()
+		var wg sync.WaitGroup
+		wg.Add(len(br.userLoginsByID))
+		for _, login := range br.userLoginsByID {
+			go func() {
+				login.DisconnectWithTimeout(timeout)
+				wg.Done()
+			}()
+		}
+		br.cacheLock.Unlock()
+		wg.Wait()
+	}
+	br.Matrix.Stop()
+	if br.cancelBackgroundCtx != nil {
+		br.cancelBackgroundCtx()
+	}
+	if stopNet, ok := br.Network.(StoppableNetwork); ok {
+		stopNet.Stop()
+	}
+	if !br.ExternallyManagedDB {
+		err := br.DB.Close()
+		if err != nil {
+			br.Log.Warn().Err(err).Msg("Failed to close database")
+		}
 	}
 	br.Log.Info().Msg("Shutdown complete")
 }

bridgev2/bridgeconfig/appservice.go

@@ -8,9 +8,9 @@ package bridgeconfig
 
 import (
 	"fmt"
-	"html/template"
 	"regexp"
 	"strings"
+	"text/template"
 
 	"go.mau.fi/util/exerrors"
 	"go.mau.fi/util/random"
@@ -79,12 +79,18 @@ func (asc *AppserviceConfig) copyToRegistration(registration *appservice.Registr
 	registration.SoruEphemeralEvents = asc.EphemeralEvents
 }
 
+func (ec *EncryptionConfig) applyUnstableFlags(registration *appservice.Registration) {
+	registration.MSC4190 = ec.MSC4190
+	registration.MSC3202 = ec.Appservice
+}
+
 // GenerateRegistration generates a registration file for the homeserver.
 func (config *Config) GenerateRegistration() *appservice.Registration {
 	registration := appservice.CreateRegistration()
 	config.AppService.HSToken = registration.ServerToken
 	config.AppService.ASToken = registration.AppToken
 	config.AppService.copyToRegistration(registration)
+	config.Encryption.applyUnstableFlags(registration)
 
 	registration.SenderLocalpart = random.String(32)
 	botRegex := regexp.MustCompile(fmt.Sprintf("^@%s:%s$",
@@ -103,6 +109,7 @@ func (config *Config) MakeAppService() *appservice.AppService {
 	as.Host.Hostname = config.AppService.Hostname
 	as.Host.Port = config.AppService.Port
 	as.Registration = config.AppService.GetRegistration()
+	config.Encryption.applyUnstableFlags(as.Registration)
 	return as
 }
 

bridgev2/bridgeconfig/backfill.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package bridgeconfig
+
+type BackfillConfig struct {
+	Enabled              bool `yaml:"enabled"`
+	MaxInitialMessages   int  `yaml:"max_initial_messages"`
+	MaxCatchupMessages   int  `yaml:"max_catchup_messages"`
+	UnreadHoursThreshold int  `yaml:"unread_hours_threshold"`
+
+	Threads BackfillThreadsConfig `yaml:"threads"`
+	Queue   BackfillQueueConfig   `yaml:"queue"`
+
+	// Flag to indicate that the creator will not run the backfill queue but will still paginate
+	// backfill by calling DoBackfillTask directly. Note that this is not used anywhere within
+	// mautrix-go and exists so bridges can use it to decide when to drop backfill data.
+	WillPaginateManually bool `yaml:"will_paginate_manually"`
+}
+
+type BackfillThreadsConfig struct {
+	MaxInitialMessages int `yaml:"max_initial_messages"`
+}
+
+type BackfillQueueConfig struct {
+	Enabled    bool `yaml:"enabled"`
+	BatchSize  int  `yaml:"batch_size"`
+	BatchDelay int  `yaml:"batch_delay"`
+	MaxBatches int  `yaml:"max_batches"`
+
+	MaxBatchesOverride map[string]int `yaml:"max_batches_override"`
+}
+
+func (bqc *BackfillQueueConfig) GetOverride(names ...string) int {
+	for _, name := range names {
+		override, ok := bqc.MaxBatchesOverride[name]
+		if ok {
+			return override
+		}
+	}
+	return bqc.MaxBatches
+}

bridgev2/bridgeconfig/config.go

@@ -7,10 +7,13 @@
 package bridgeconfig
 
 import (
+	"time"
+
 	"go.mau.fi/util/dbutil"
 	"go.mau.fi/zeroconfig"
 	"gopkg.in/yaml.v3"
 
+	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/mediaproxy"
 )
 
@@ -21,34 +24,90 @@ type Config struct {
 	Homeserver   HomeserverConfig   `yaml:"homeserver"`
 	AppService   AppserviceConfig   `yaml:"appservice"`
 	Matrix       MatrixConfig       `yaml:"matrix"`
+	Analytics    AnalyticsConfig    `yaml:"analytics"`
 	Provisioning ProvisioningConfig `yaml:"provisioning"`
+	PublicMedia  PublicMediaConfig  `yaml:"public_media"`
 	DirectMedia  DirectMediaConfig  `yaml:"direct_media"`
+	Backfill     BackfillConfig     `yaml:"backfill"`
 	DoublePuppet DoublePuppetConfig `yaml:"double_puppet"`
 	Encryption   EncryptionConfig   `yaml:"encryption"`
 	Logging      zeroconfig.Config  `yaml:"logging"`
 
+	EnvConfigPrefix string `yaml:"env_config_prefix"`
+
 	ManagementRoomTexts ManagementRoomTexts `yaml:"management_room_texts"`
 }
 
+type CleanupAction string
+
+const (
+	CleanupActionNull     CleanupAction = ""
+	CleanupActionNothing  CleanupAction = "nothing"
+	CleanupActionKick     CleanupAction = "kick"
+	CleanupActionUnbridge CleanupAction = "unbridge"
+	CleanupActionDelete   CleanupAction = "delete"
+)
+
+type CleanupOnLogout struct {
+	Private        CleanupAction `yaml:"private"`
+	Relayed        CleanupAction `yaml:"relayed"`
+	SharedNoUsers  CleanupAction `yaml:"shared_no_users"`
+	SharedHasUsers CleanupAction `yaml:"shared_has_users"`
+}
+
+type CleanupOnLogouts struct {
+	Enabled        bool            `yaml:"enabled"`
+	Manual         CleanupOnLogout `yaml:"manual"`
+	BadCredentials CleanupOnLogout `yaml:"bad_credentials"`
+}
+
 type BridgeConfig struct {
-	CommandPrefix           string           `yaml:"command_prefix"`
-	PersonalFilteringSpaces bool             `yaml:"personal_filtering_spaces"`
-	Relay                   RelayConfig      `yaml:"relay"`
-	Permissions             PermissionConfig `yaml:"permissions"`
+	CommandPrefix                 string           `yaml:"command_prefix"`
+	PersonalFilteringSpaces       bool             `yaml:"personal_filtering_spaces"`
+	PrivateChatPortalMeta         bool             `yaml:"private_chat_portal_meta"`
+	AsyncEvents                   bool             `yaml:"async_events"`
+	SplitPortals                  bool             `yaml:"split_portals"`
+	ResendBridgeInfo              bool             `yaml:"resend_bridge_info"`
+	NoBridgeInfoStateKey          bool             `yaml:"no_bridge_info_state_key"`
+	BridgeStatusNotices           string           `yaml:"bridge_status_notices"`
+	UnknownErrorAutoReconnect     time.Duration    `yaml:"unknown_error_auto_reconnect"`
+	UnknownErrorMaxAutoReconnects int              `yaml:"unknown_error_max_auto_reconnects"`
+	BridgeMatrixLeave             bool             `yaml:"bridge_matrix_leave"`
+	BridgeNotices                 bool             `yaml:"bridge_notices"`
+	TagOnlyOnCreate               bool             `yaml:"tag_only_on_create"`
+	OnlyBridgeTags                []event.RoomTag  `yaml:"only_bridge_tags"`
+	MuteOnlyOnCreate              bool             `yaml:"mute_only_on_create"`
+	DeduplicateMatrixMessages     bool             `yaml:"deduplicate_matrix_messages"`
+	CrossRoomReplies              bool             `yaml:"cross_room_replies"`
+	OutgoingMessageReID           bool             `yaml:"outgoing_message_re_id"`
+	RevertFailedStateChanges      bool             `yaml:"revert_failed_state_changes"`
+	KickMatrixUsers               bool             `yaml:"kick_matrix_users"`
+	CleanupOnLogout               CleanupOnLogouts `yaml:"cleanup_on_logout"`
+	Relay                         RelayConfig      `yaml:"relay"`
+	Permissions                   PermissionConfig `yaml:"permissions"`
+	Backfill                      BackfillConfig   `yaml:"backfill"`
 }
 
 type MatrixConfig struct {
-	MessageStatusEvents bool `yaml:"message_status_events"`
-	DeliveryReceipts    bool `yaml:"delivery_receipts"`
-	MessageErrorNotices bool `yaml:"message_error_notices"`
-	SyncDirectChatList  bool `yaml:"sync_direct_chat_list"`
-	FederateRooms       bool `yaml:"federate_rooms"`
+	MessageStatusEvents   bool  `yaml:"message_status_events"`
+	DeliveryReceipts      bool  `yaml:"delivery_receipts"`
+	MessageErrorNotices   bool  `yaml:"message_error_notices"`
+	SyncDirectChatList    bool  `yaml:"sync_direct_chat_list"`
+	FederateRooms         bool  `yaml:"federate_rooms"`
+	UploadFileThreshold   int64 `yaml:"upload_file_threshold"`
+	GhostExtraProfileInfo bool  `yaml:"ghost_extra_profile_info"`
+}
+
+type AnalyticsConfig struct {
+	Token  string `yaml:"token"`
+	URL    string `yaml:"url"`
+	UserID string `yaml:"user_id"`
 }
 
 type ProvisioningConfig struct {
-	Prefix         string `yaml:"prefix"`
-	SharedSecret   string `yaml:"shared_secret"`
-	DebugEndpoints bool   `yaml:"debug_endpoints"`
+	SharedSecret           string `yaml:"shared_secret"`
+	DebugEndpoints         bool   `yaml:"debug_endpoints"`
+	EnableSessionTransfers bool   `yaml:"enable_session_transfers"`
 }
 
 type DirectMediaConfig struct {
@@ -57,6 +116,15 @@ type DirectMediaConfig struct {
 	mediaproxy.BasicConfig `yaml:",inline"`
 }
 
+type PublicMediaConfig struct {
+	Enabled     bool   `yaml:"enabled"`
+	SigningKey  string `yaml:"signing_key"`
+	Expiry      int    `yaml:"expiry"`
+	HashLength  int    `yaml:"hash_length"`
+	PathPrefix  string `yaml:"path_prefix"`
+	UseDatabase bool   `yaml:"use_database"`
+}
+
 type DoublePuppetConfig struct {
 	Servers        map[string]string `yaml:"servers"`
 	AllowDiscovery bool              `yaml:"allow_discovery"`

bridgev2/bridgeconfig/encryption.go

@@ -15,6 +15,9 @@ type EncryptionConfig struct {
 	Default    bool `yaml:"default"`
 	Require    bool `yaml:"require"`
 	Appservice bool `yaml:"appservice"`
+	MSC4190    bool `yaml:"msc4190"`
+	MSC4392    bool `yaml:"msc4392"`
+	SelfSign   bool `yaml:"self_sign"`
 
 	PlaintextMentions bool `yaml:"plaintext_mentions"`
 

bridgev2/bridgeconfig/legacymigrate.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package bridgeconfig
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+
+	up "go.mau.fi/util/configupgrade"
+)
+
+var HackyMigrateLegacyNetworkConfig func(up.Helper)
+
+func CopyToOtherLocation(helper up.Helper, fieldType up.YAMLType, source, dest []string) {
+	val, ok := helper.Get(fieldType, source...)
+	if ok {
+		helper.Set(fieldType, val, dest...)
+	}
+}
+
+func CopyMapToOtherLocation(helper up.Helper, source, dest []string) {
+	val := helper.GetNode(source...)
+	if val != nil && val.Map != nil {
+		helper.SetMap(val.Map, dest...)
+	}
+}
+
+func doMigrateLegacy(helper up.Helper, python bool) {
+	if HackyMigrateLegacyNetworkConfig == nil {
+		_, _ = fmt.Fprintln(os.Stderr, "Legacy bridge config detected, but hacky network config migrator is not set")
+		os.Exit(1)
+	}
+	_, _ = fmt.Fprintln(os.Stderr, "Migrating legacy bridge config")
+
+	helper.Copy(up.Str, "homeserver", "address")
+	helper.Copy(up.Str, "homeserver", "domain")
+	helper.Copy(up.Str, "homeserver", "software")
+	helper.Copy(up.Str|up.Null, "homeserver", "status_endpoint")
+	helper.Copy(up.Str|up.Null, "homeserver", "message_send_checkpoint_endpoint")
+	helper.Copy(up.Bool, "homeserver", "async_media")
+	helper.Copy(up.Str|up.Null, "homeserver", "websocket_proxy")
+	helper.Copy(up.Bool, "homeserver", "websocket")
+	helper.Copy(up.Int, "homeserver", "ping_interval_seconds")
+
+	helper.Copy(up.Str|up.Null, "appservice", "address")
+	helper.Copy(up.Str|up.Null, "appservice", "hostname")
+	helper.Copy(up.Int|up.Null, "appservice", "port")
+	helper.Copy(up.Str, "appservice", "id")
+	if python {
+		CopyToOtherLocation(helper, up.Str, []string{"appservice", "bot_username"}, []string{"appservice", "bot", "username"})
+		CopyToOtherLocation(helper, up.Str, []string{"appservice", "bot_displayname"}, []string{"appservice", "bot", "displayname"})
+		CopyToOtherLocation(helper, up.Str, []string{"appservice", "bot_avatar"}, []string{"appservice", "bot", "avatar"})
+	} else {
+		helper.Copy(up.Str, "appservice", "bot", "username")
+		helper.Copy(up.Str, "appservice", "bot", "displayname")
+		helper.Copy(up.Str, "appservice", "bot", "avatar")
+	}
+	helper.Copy(up.Bool, "appservice", "ephemeral_events")
+	helper.Copy(up.Bool, "appservice", "async_transactions")
+	helper.Copy(up.Str, "appservice", "as_token")
+	helper.Copy(up.Str, "appservice", "hs_token")
+
+	helper.Copy(up.Str, "bridge", "command_prefix")
+	helper.Copy(up.Bool, "bridge", "personal_filtering_spaces")
+	if oldPM, ok := helper.Get(up.Str, "bridge", "private_chat_portal_meta"); ok && (oldPM == "default" || oldPM == "always") {
+		helper.Set(up.Bool, "true", "bridge", "private_chat_portal_meta")
+	} else {
+		helper.Set(up.Bool, "false", "bridge", "private_chat_portal_meta")
+	}
+	helper.Copy(up.Bool, "bridge", "relay", "enabled")
+	helper.Copy(up.Bool, "bridge", "relay", "admin_only")
+	helper.Copy(up.Map, "bridge", "permissions")
+
+	if python {
+		legacyDB, ok := helper.Get(up.Str, "appservice", "database")
+		if ok {
+			if strings.HasPrefix(legacyDB, "postgres") {
+				parsedDB, err := url.Parse(legacyDB)
+				if err != nil {
+					panic(err)
+				}
+				q := parsedDB.Query()
+				if parsedDB.Host == "" && !q.Has("host") {
+					q.Set("host", "/var/run/postgresql")
+				} else if !q.Has("sslmode") {
+					q.Set("sslmode", "disable")
+				}
+				parsedDB.RawQuery = q.Encode()
+				helper.Set(up.Str, parsedDB.String(), "database", "uri")
+				helper.Set(up.Str, "postgres", "database", "type")
+			} else {
+				dbPath := strings.TrimPrefix(strings.TrimPrefix(legacyDB, "sqlite:"), "///")
+				helper.Set(up.Str, fmt.Sprintf("file:%s?_txlock=immediate", dbPath), "database", "uri")
+				helper.Set(up.Str, "sqlite3-fk-wal", "database", "type")
+			}
+		}
+		if legacyDBMinSize, ok := helper.Get(up.Int, "appservice", "database_opts", "min_size"); ok {
+			helper.Set(up.Int, legacyDBMinSize, "database", "max_idle_conns")
+		}
+		if legacyDBMaxSize, ok := helper.Get(up.Int, "appservice", "database_opts", "max_size"); ok {
+			helper.Set(up.Int, legacyDBMaxSize, "database", "max_open_conns")
+		}
+	} else {
+		if dbType, ok := helper.Get(up.Str, "appservice", "database", "type"); ok && dbType == "sqlite3" {
+			helper.Set(up.Str, "sqlite3-fk-wal", "database", "type")
+		} else {
+			CopyToOtherLocation(helper, up.Str, []string{"appservice", "database", "type"}, []string{"database", "type"})
+		}
+		CopyToOtherLocation(helper, up.Str, []string{"appservice", "database", "uri"}, []string{"database", "uri"})
+		CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_open_conns"}, []string{"database", "max_open_conns"})
+		CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_idle_conns"}, []string{"database", "max_idle_conns"})
+		CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_conn_idle_time"}, []string{"database", "max_conn_idle_time"})
+		CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_conn_lifetime"}, []string{"database", "max_conn_lifetime"})
+	}
+
+	if python {
+		if usernameTemplate, ok := helper.Get(up.Str, "bridge", "username_template"); ok && strings.Contains(usernameTemplate, "{userid}") {
+			helper.Set(up.Str, strings.ReplaceAll(usernameTemplate, "{userid}", "{{.}}"), "appservice", "username_template")
+		}
+	} else {
+		CopyToOtherLocation(helper, up.Str, []string{"bridge", "username_template"}, []string{"appservice", "username_template"})
+	}
+
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "message_status_events"}, []string{"matrix", "message_status_events"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "delivery_receipts"}, []string{"matrix", "delivery_receipts"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "message_error_notices"}, []string{"matrix", "message_error_notices"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "sync_direct_chat_list"}, []string{"matrix", "sync_direct_chat_list"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "federate_rooms"}, []string{"matrix", "federate_rooms"})
+
+	CopyToOtherLocation(helper, up.Str, []string{"bridge", "provisioning", "shared_secret"}, []string{"provisioning", "shared_secret"})
+	CopyToOtherLocation(helper, up.Str, []string{"appservice", "provisioning", "shared_secret"}, []string{"provisioning", "shared_secret"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "provisioning", "debug_endpoints"}, []string{"provisioning", "debug_endpoints"})
+
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "double_puppet_allow_discovery"}, []string{"double_puppet", "allow_discovery"})
+	CopyMapToOtherLocation(helper, []string{"bridge", "double_puppet_server_map"}, []string{"double_puppet", "servers"})
+	CopyMapToOtherLocation(helper, []string{"bridge", "login_shared_secret_map"}, []string{"double_puppet", "secrets"})
+
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "allow"}, []string{"encryption", "allow"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "default"}, []string{"encryption", "default"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "require"}, []string{"encryption", "require"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "appservice"}, []string{"encryption", "appservice"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "allow_key_sharing"}, []string{"encryption", "allow_key_sharing"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_outbound_on_ack"}, []string{"encryption", "delete_keys", "delete_outbound_on_ack"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "dont_store_outbound"}, []string{"encryption", "delete_keys", "dont_store_outbound"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "ratchet_on_decrypt"}, []string{"encryption", "delete_keys", "ratchet_on_decrypt"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_fully_used_on_decrypt"}, []string{"encryption", "delete_keys", "delete_fully_used_on_decrypt"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_prev_on_new_session"}, []string{"encryption", "delete_keys", "delete_prev_on_new_session"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_on_device_delete"}, []string{"encryption", "delete_keys", "delete_on_device_delete"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "periodically_delete_expired"}, []string{"encryption", "delete_keys", "periodically_delete_expired"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_outdated_inbound"}, []string{"encryption", "delete_keys", "delete_outdated_inbound"})
+	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "receive"}, []string{"encryption", "verification_levels", "receive"})
+	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "send"}, []string{"encryption", "verification_levels", "send"})
+	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "share"}, []string{"encryption", "verification_levels", "share"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "rotation", "enable_custom"}, []string{"encryption", "rotation", "enable_custom"})
+	CopyToOtherLocation(helper, up.Int, []string{"bridge", "encryption", "rotation", "milliseconds"}, []string{"encryption", "rotation", "milliseconds"})
+	CopyToOtherLocation(helper, up.Int, []string{"bridge", "encryption", "rotation", "messages"}, []string{"encryption", "rotation", "messages"})
+	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "rotation", "disable_device_change_key_rotation"}, []string{"encryption", "rotation", "disable_device_change_key_rotation"})
+
+	if helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "print_level") != nil || helper.GetNode("logging", "file_name_format") != nil) {
+		_, _ = fmt.Fprintln(os.Stderr, "Migrating maulogger configs is not supported")
+	} else if (helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "handlers") != nil)) || python {
+		_, _ = fmt.Fprintln(os.Stderr, "Migrating Python log configs is not supported")
+	} else {
+		helper.Copy(up.Map, "logging")
+	}
+
+	HackyMigrateLegacyNetworkConfig(helper)
+}

bridgev2/bridgeconfig/permissions.go

@@ -8,6 +8,8 @@ package bridgeconfig
 
 import (
 	"fmt"
+	"os"
+	"strconv"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -21,6 +23,8 @@ type Permissions struct {
 	Login        bool `yaml:"login"`
 	DoublePuppet bool `yaml:"double_puppet"`
 	Admin        bool `yaml:"admin"`
+	ManageRelay  bool `yaml:"manage_relay"`
+	MaxLogins    int  `yaml:"max_logins"`
 }
 
 type PermissionConfig map[string]*Permissions
@@ -37,10 +41,7 @@ func (pc PermissionConfig) IsConfigured() bool {
 	_, hasExampleDomain := pc["example.com"]
 	_, hasExampleUser := pc["@admin:example.com"]
 	exampleLen := boolToInt(hasWildcard) + boolToInt(hasExampleUser) + boolToInt(hasExampleDomain)
-	if len(pc) <= exampleLen {
-		return false
-	}
-	return true
+	return len(pc) > exampleLen
 }
 
 func (pc PermissionConfig) Get(userID id.UserID) Permissions {
@@ -58,9 +59,9 @@ func (pc PermissionConfig) Get(userID id.UserID) Permissions {
 var (
 	PermissionLevelBlock    = Permissions{}
 	PermissionLevelRelay    = Permissions{SendEvents: true}
-	PermissionLevelCommands = Permissions{SendEvents: true, Commands: true}
-	PermissionLevelUser     = Permissions{SendEvents: true, Commands: true, Login: true, DoublePuppet: true}
-	PermissionLevelAdmin    = Permissions{SendEvents: true, Commands: true, Login: true, DoublePuppet: true, Admin: true}
+	PermissionLevelCommands = Permissions{SendEvents: true, Commands: true, ManageRelay: true}
+	PermissionLevelUser     = Permissions{SendEvents: true, Commands: true, ManageRelay: true, Login: true, DoublePuppet: true}
+	PermissionLevelAdmin    = Permissions{SendEvents: true, Commands: true, ManageRelay: true, Login: true, DoublePuppet: true, Admin: true}
 )
 
 var namesToLevels = map[string]Permissions{
@@ -93,6 +94,23 @@ func (p *Permissions) UnmarshalYAML(perm *yaml.Node) error {
 	case "!!map":
 		err := perm.Decode((*umPerm)(p))
 		return err
+	case "!!int":
+		val, err := strconv.Atoi(perm.Value)
+		if err != nil {
+			return fmt.Errorf("invalid permissions level %s", perm.Value)
+		}
+		_, _ = fmt.Fprintln(os.Stderr, "Warning: config contains deprecated integer permission values")
+		// Integer values are deprecated, so they're hardcoded
+		if val < 5 {
+			*p = PermissionLevelBlock
+		} else if val < 10 {
+			*p = PermissionLevelRelay
+		} else if val < 100 {
+			*p = PermissionLevelUser
+		} else {
+			*p = PermissionLevelAdmin
+		}
+		return nil
 	default:
 		return fmt.Errorf("invalid permissions type %s", perm.Tag)
 	}

bridgev2/bridgeconfig/relay.go

@@ -19,11 +19,13 @@ import (
 )
 
 type RelayConfig struct {
-	Enabled          bool                         `yaml:"enabled"`
-	AdminOnly        bool                         `yaml:"admin_only"`
-	DefaultRelays    []networkid.UserLoginID      `yaml:"default_relays"`
-	MessageFormats   map[event.MessageType]string `yaml:"message_formats"`
-	messageTemplates *template.Template           `yaml:"-"`
+	Enabled           bool                         `yaml:"enabled"`
+	AdminOnly         bool                         `yaml:"admin_only"`
+	DefaultRelays     []networkid.UserLoginID      `yaml:"default_relays"`
+	MessageFormats    map[event.MessageType]string `yaml:"message_formats"`
+	DisplaynameFormat string                       `yaml:"displayname_format"`
+	messageTemplates  *template.Template           `yaml:"-"`
+	nameTemplate      *template.Template           `yaml:"-"`
 }
 
 type umRelayConfig RelayConfig
@@ -42,6 +44,11 @@ func (rc *RelayConfig) UnmarshalYAML(node *yaml.Node) error {
 		}
 	}
 
+	rc.nameTemplate, err = template.New("nameTemplate").Parse(rc.DisplaynameFormat)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -94,3 +101,9 @@ func (rc *RelayConfig) FormatMessage(content *event.MessageEventContent, sender
 	content.Body = format.HTMLToText(content.FormattedBody)
 	return content, nil
 }
+
+func (rc *RelayConfig) FormatName(sender any) string {
+	var buf strings.Builder
+	_ = rc.nameTemplate.Execute(&buf, sender)
+	return strings.TrimSpace(buf.String())
+}

bridgev2/bridgeconfig/upgrade.go

@@ -8,7 +8,6 @@ package bridgeconfig
 
 import (
 	"fmt"
-	"os"
 
 	up "go.mau.fi/util/configupgrade"
 	"go.mau.fi/util/random"
@@ -18,19 +17,50 @@ import (
 
 func doUpgrade(helper up.Helper) {
 	if _, isLegacyConfig := helper.Get(up.Str, "appservice", "database", "uri"); isLegacyConfig {
-		doMigrateLegacy(helper)
+		doMigrateLegacy(helper, false)
+		return
+	} else if _, isLegacyPython := helper.Get(up.Str, "appservice", "database"); isLegacyPython {
+		doMigrateLegacy(helper, true)
 		return
 	}
 
 	helper.Copy(up.Str, "bridge", "command_prefix")
 	helper.Copy(up.Bool, "bridge", "personal_filtering_spaces")
+	helper.Copy(up.Bool, "bridge", "private_chat_portal_meta")
+	helper.Copy(up.Bool, "bridge", "async_events")
+	helper.Copy(up.Bool, "bridge", "split_portals")
+	helper.Copy(up.Bool, "bridge", "resend_bridge_info")
+	helper.Copy(up.Bool, "bridge", "no_bridge_info_state_key")
+	helper.Copy(up.Str|up.Null, "bridge", "bridge_status_notices")
+	helper.Copy(up.Str|up.Int|up.Null, "bridge", "unknown_error_auto_reconnect")
+	helper.Copy(up.Int, "bridge", "unknown_error_max_auto_reconnects")
+	helper.Copy(up.Bool, "bridge", "bridge_matrix_leave")
+	helper.Copy(up.Bool, "bridge", "bridge_notices")
+	helper.Copy(up.Bool, "bridge", "tag_only_on_create")
+	helper.Copy(up.List, "bridge", "only_bridge_tags")
+	helper.Copy(up.Bool, "bridge", "mute_only_on_create")
+	helper.Copy(up.Bool, "bridge", "deduplicate_matrix_messages")
+	helper.Copy(up.Bool, "bridge", "cross_room_replies")
+	helper.Copy(up.Bool, "bridge", "revert_failed_state_changes")
+	helper.Copy(up.Bool, "bridge", "kick_matrix_users")
+	helper.Copy(up.Bool, "bridge", "cleanup_on_logout", "enabled")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "manual", "private")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "manual", "relayed")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "manual", "shared_no_users")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "manual", "shared_has_users")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "bad_credentials", "private")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "bad_credentials", "relayed")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "bad_credentials", "shared_no_users")
+	helper.Copy(up.Str, "bridge", "cleanup_on_logout", "bad_credentials", "shared_has_users")
 	helper.Copy(up.Bool, "bridge", "relay", "enabled")
 	helper.Copy(up.Bool, "bridge", "relay", "admin_only")
 	helper.Copy(up.List, "bridge", "relay", "default_relays")
 	helper.Copy(up.Map, "bridge", "relay", "message_formats")
+	helper.Copy(up.Str, "bridge", "relay", "displayname_format")
 	helper.Copy(up.Map, "bridge", "permissions")
 
 	if dbType, ok := helper.Get(up.Str, "database", "type"); ok && dbType == "sqlite3" {
+		fmt.Println("Warning: invalid database type sqlite3 in config. Autocorrecting to sqlite3-fk-wal")
 		helper.Set(up.Str, "sqlite3-fk-wal", "database", "type")
 	} else {
 		helper.Copy(up.Str, "database", "type")
@@ -70,8 +100,13 @@ func doUpgrade(helper up.Helper) {
 	helper.Copy(up.Bool, "matrix", "message_error_notices")
 	helper.Copy(up.Bool, "matrix", "sync_direct_chat_list")
 	helper.Copy(up.Bool, "matrix", "federate_rooms")
+	helper.Copy(up.Int, "matrix", "upload_file_threshold")
+	helper.Copy(up.Bool, "matrix", "ghost_extra_profile_info")
+
+	helper.Copy(up.Str|up.Null, "analytics", "token")
+	helper.Copy(up.Str|up.Null, "analytics", "url")
+	helper.Copy(up.Str|up.Null, "analytics", "user_id")
 
-	helper.Copy(up.Str, "provisioning", "prefix")
 	if secret, ok := helper.Get(up.Str, "provisioning", "shared_secret"); !ok || secret == "generate" {
 		sharedSecret := random.String(64)
 		helper.Set(up.Str, sharedSecret, "provisioning", "shared_secret")
@@ -79,6 +114,7 @@ func doUpgrade(helper up.Helper) {
 		helper.Copy(up.Str, "provisioning", "shared_secret")
 	}
 	helper.Copy(up.Bool, "provisioning", "debug_endpoints")
+	helper.Copy(up.Bool, "provisioning", "enable_session_transfers")
 
 	helper.Copy(up.Bool, "direct_media", "enabled")
 	helper.Copy(up.Str|up.Null, "direct_media", "media_id_prefix")
@@ -92,6 +128,28 @@ func doUpgrade(helper up.Helper) {
 		helper.Copy(up.Str, "direct_media", "server_key")
 	}
 
+	helper.Copy(up.Bool, "public_media", "enabled")
+	if signingKey, ok := helper.Get(up.Str, "public_media", "signing_key"); !ok || signingKey == "generate" {
+		helper.Set(up.Str, random.String(64), "public_media", "signing_key")
+	} else {
+		helper.Copy(up.Str, "public_media", "signing_key")
+	}
+	helper.Copy(up.Int, "public_media", "expiry")
+	helper.Copy(up.Int, "public_media", "hash_length")
+	helper.Copy(up.Str|up.Null, "public_media", "path_prefix")
+	helper.Copy(up.Bool, "public_media", "use_database")
+
+	helper.Copy(up.Bool, "backfill", "enabled")
+	helper.Copy(up.Int, "backfill", "max_initial_messages")
+	helper.Copy(up.Int, "backfill", "max_catchup_messages")
+	helper.Copy(up.Int, "backfill", "unread_hours_threshold")
+	helper.Copy(up.Int, "backfill", "threads", "max_initial_messages")
+	helper.Copy(up.Bool, "backfill", "queue", "enabled")
+	helper.Copy(up.Int, "backfill", "queue", "batch_size")
+	helper.Copy(up.Int, "backfill", "queue", "batch_delay")
+	helper.Copy(up.Int, "backfill", "queue", "max_batches")
+	helper.Copy(up.Map, "backfill", "queue", "max_batches_override")
+
 	helper.Copy(up.Map, "double_puppet", "servers")
 	helper.Copy(up.Bool, "double_puppet", "allow_discovery")
 	helper.Copy(up.Map, "double_puppet", "secrets")
@@ -100,6 +158,13 @@ func doUpgrade(helper up.Helper) {
 	helper.Copy(up.Bool, "encryption", "default")
 	helper.Copy(up.Bool, "encryption", "require")
 	helper.Copy(up.Bool, "encryption", "appservice")
+	if val, ok := helper.Get(up.Bool, "appservice", "msc4190"); ok {
+		helper.Set(up.Bool, val, "encryption", "msc4190")
+	} else {
+		helper.Copy(up.Bool, "encryption", "msc4190")
+	}
+	helper.Copy(up.Bool, "encryption", "msc4392")
+	helper.Copy(up.Bool, "encryption", "self_sign")
 	helper.Copy(up.Bool, "encryption", "allow_key_sharing")
 	if secret, ok := helper.Get(up.Str, "encryption", "pickle_key"); !ok || secret == "generate" {
 		helper.Set(up.Str, random.String(64), "encryption", "pickle_key")
@@ -122,119 +187,15 @@ func doUpgrade(helper up.Helper) {
 	helper.Copy(up.Int, "encryption", "rotation", "messages")
 	helper.Copy(up.Bool, "encryption", "rotation", "disable_device_change_key_rotation")
 
+	helper.Copy(up.Str|up.Null, "env_config_prefix")
+
 	helper.Copy(up.Map, "logging")
 }
 
-func CopyToOtherLocation(helper up.Helper, fieldType up.YAMLType, source, dest []string) {
-	val, ok := helper.Get(fieldType, source...)
-	if ok {
-		helper.Set(fieldType, val, dest...)
-	}
-}
-
-func CopyMapToOtherLocation(helper up.Helper, source, dest []string) {
-	val := helper.GetNode(source...)
-	if val != nil && val.Map != nil {
-		helper.SetMap(val.Map, dest...)
-	}
-}
-
-var HackyMigrateLegacyNetworkConfig func(up.Helper)
-
-func doMigrateLegacy(helper up.Helper) {
-	if HackyMigrateLegacyNetworkConfig == nil {
-		_, _ = fmt.Fprintln(os.Stderr, "Legacy bridge config detected, but hacky network config migrator is not set")
-		os.Exit(1)
-	}
-	_, _ = fmt.Fprintln(os.Stderr, "Migrating legacy bridge config")
-
-	helper.Copy(up.Str, "homeserver", "address")
-	helper.Copy(up.Str, "homeserver", "domain")
-	helper.Copy(up.Str, "homeserver", "software")
-	helper.Copy(up.Str|up.Null, "homeserver", "status_endpoint")
-	helper.Copy(up.Str|up.Null, "homeserver", "message_send_checkpoint_endpoint")
-	helper.Copy(up.Bool, "homeserver", "async_media")
-	helper.Copy(up.Str|up.Null, "homeserver", "websocket_proxy")
-	helper.Copy(up.Bool, "homeserver", "websocket")
-	helper.Copy(up.Int, "homeserver", "ping_interval_seconds")
-
-	helper.Copy(up.Str|up.Null, "appservice", "address")
-	helper.Copy(up.Str|up.Null, "appservice", "hostname")
-	helper.Copy(up.Int|up.Null, "appservice", "port")
-	helper.Copy(up.Str, "appservice", "id")
-	helper.Copy(up.Str, "appservice", "bot", "username")
-	helper.Copy(up.Str, "appservice", "bot", "displayname")
-	helper.Copy(up.Str, "appservice", "bot", "avatar")
-	helper.Copy(up.Bool, "appservice", "ephemeral_events")
-	helper.Copy(up.Bool, "appservice", "async_transactions")
-	helper.Copy(up.Str, "appservice", "as_token")
-	helper.Copy(up.Str, "appservice", "hs_token")
-
-	helper.Copy(up.Str, "bridge", "command_prefix")
-	helper.Copy(up.Bool, "bridge", "personal_filtering_spaces")
-	helper.Copy(up.Bool, "bridge", "relay", "enabled")
-	helper.Copy(up.Bool, "bridge", "relay", "admin_only")
-	helper.Copy(up.Map, "bridge", "permissions")
-
-	CopyToOtherLocation(helper, up.Str, []string{"appservice", "database", "type"}, []string{"database", "type"})
-	CopyToOtherLocation(helper, up.Str, []string{"appservice", "database", "uri"}, []string{"database", "uri"})
-	CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_open_conns"}, []string{"database", "max_open_conns"})
-	CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_idle_conns"}, []string{"database", "max_idle_conns"})
-	CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_conn_idle_time"}, []string{"database", "max_conn_idle_time"})
-	CopyToOtherLocation(helper, up.Int, []string{"appservice", "database", "max_conn_lifetime"}, []string{"database", "max_conn_lifetime"})
-
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "username_template"}, []string{"appservice", "username_template"})
-
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "message_status_events"}, []string{"matrix", "message_status_events"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "delivery_receipts"}, []string{"matrix", "delivery_receipts"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "message_error_notices"}, []string{"matrix", "message_error_notices"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "sync_direct_chat_list"}, []string{"matrix", "sync_direct_chat_list"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "federate_rooms"}, []string{"matrix", "federate_rooms"})
-
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "provisioning", "prefix"}, []string{"provisioning", "prefix"})
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "provisioning", "shared_secret"}, []string{"provisioning", "shared_secret"})
-	CopyToOtherLocation(helper, up.Str, []string{"appservice", "provisioning", "prefix"}, []string{"provisioning", "prefix"})
-	CopyToOtherLocation(helper, up.Str, []string{"appservice", "provisioning", "shared_secret"}, []string{"provisioning", "shared_secret"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "provisioning", "debug_endpoints"}, []string{"provisioning", "debug_endpoints"})
-
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "double_puppet_allow_discovery"}, []string{"double_puppet", "allow_discovery"})
-	CopyMapToOtherLocation(helper, []string{"bridge", "double_puppet_server_map"}, []string{"double_puppet", "servers"})
-	CopyMapToOtherLocation(helper, []string{"bridge", "login_shared_secret_map"}, []string{"double_puppet", "secrets"})
-
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "allow"}, []string{"encryption", "allow"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "default"}, []string{"encryption", "default"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "require"}, []string{"encryption", "require"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "appservice"}, []string{"encryption", "appservice"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "allow_key_sharing"}, []string{"encryption", "allow_key_sharing"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_outbound_on_ack"}, []string{"encryption", "delete_keys", "delete_outbound_on_ack"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "dont_store_outbound"}, []string{"encryption", "delete_keys", "dont_store_outbound"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "ratchet_on_decrypt"}, []string{"encryption", "delete_keys", "ratchet_on_decrypt"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_fully_used_on_decrypt"}, []string{"encryption", "delete_keys", "delete_fully_used_on_decrypt"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_prev_on_new_session"}, []string{"encryption", "delete_keys", "delete_prev_on_new_session"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_on_device_delete"}, []string{"encryption", "delete_keys", "delete_on_device_delete"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "periodically_delete_expired"}, []string{"encryption", "delete_keys", "periodically_delete_expired"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "delete_keys", "delete_outdated_inbound"}, []string{"encryption", "delete_keys", "delete_outdated_inbound"})
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "receive"}, []string{"encryption", "verification_levels", "receive"})
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "send"}, []string{"encryption", "verification_levels", "send"})
-	CopyToOtherLocation(helper, up.Str, []string{"bridge", "encryption", "verification_levels", "share"}, []string{"encryption", "verification_levels", "share"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "rotation", "enable_custom"}, []string{"encryption", "rotation", "enable_custom"})
-	CopyToOtherLocation(helper, up.Int, []string{"bridge", "encryption", "rotation", "milliseconds"}, []string{"encryption", "rotation", "milliseconds"})
-	CopyToOtherLocation(helper, up.Int, []string{"bridge", "encryption", "rotation", "messages"}, []string{"encryption", "rotation", "messages"})
-	CopyToOtherLocation(helper, up.Bool, []string{"bridge", "encryption", "rotation", "disable_device_change_key_rotation"}, []string{"encryption", "rotation", "disable_device_change_key_rotation"})
-
-	if helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "print_level") != nil || helper.GetNode("logging", "file_name_format") != nil) {
-		_, _ = fmt.Fprintln(os.Stderr, "Migrating maulogger configs is not supported")
-	} else if helper.GetNode("logging", "writers") == nil && (helper.GetNode("logging", "handlers") != nil) {
-		_, _ = fmt.Fprintln(os.Stderr, "Migrating Python log configs is not supported")
-	} else {
-		helper.Copy(up.Map, "logging")
-	}
-
-	HackyMigrateLegacyNetworkConfig(helper)
-}
-
 var SpacedBlocks = [][]string{
 	{"bridge"},
+	{"bridge", "bridge_matrix_leave"},
+	{"bridge", "cleanup_on_logout"},
 	{"bridge", "relay"},
 	{"bridge", "permissions"},
 	{"database"},
@@ -248,10 +209,14 @@ var SpacedBlocks = [][]string{
 	{"appservice", "as_token"},
 	{"appservice", "username_template"},
 	{"matrix"},
+	{"analytics"},
 	{"provisioning"},
+	{"public_media"},
 	{"direct_media"},
+	{"backfill"},
 	{"double_puppet"},
 	{"encryption"},
+	{"env_config_prefix"},
 	{"logging"},
 }
 

bridgev2/bridgestate.go

@@ -8,19 +8,37 @@ package bridgev2
 
 import (
 	"context"
+	"fmt"
+	"math/rand/v2"
 	"runtime/debug"
+	"sync/atomic"
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.mau.fi/util/exfmt"
 
-	"maunium.net/go/mautrix/bridge/status"
+	"maunium.net/go/mautrix/bridgev2/status"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/format"
 )
 
+var CatchBridgeStateQueuePanics = true
+
 type BridgeStateQueue struct {
-	prev   *status.BridgeState
-	ch     chan status.BridgeState
-	bridge *Bridge
-	user   status.BridgeStateFiller
+	prevUnsent *status.BridgeState
+	prevSent   *status.BridgeState
+	errorSent  bool
+	ch         chan status.BridgeState
+	bridge     *Bridge
+	login      *UserLogin
+
+	firstTransientDisconnect time.Time
+	cancelScheduledNotice    atomic.Pointer[context.CancelFunc]
+
+	stopChan      chan struct{}
+	stopReconnect atomic.Pointer[context.CancelFunc]
+
+	unknownErrorReconnects int
 }
 
 func (br *Bridge) SendGlobalBridgeState(state status.BridgeState) {
@@ -40,51 +58,221 @@ func (br *Bridge) SendGlobalBridgeState(state status.BridgeState) {
 	}
 }
 
-func (br *Bridge) NewBridgeStateQueue(user status.BridgeStateFiller) *BridgeStateQueue {
+func (br *Bridge) NewBridgeStateQueue(login *UserLogin) *BridgeStateQueue {
 	bsq := &BridgeStateQueue{
-		ch:     make(chan status.BridgeState, 10),
-		bridge: br,
-		user:   user,
+		ch:       make(chan status.BridgeState, 10),
+		stopChan: make(chan struct{}),
+		bridge:   br,
+		login:    login,
 	}
 	go bsq.loop()
 	return bsq
 }
 
 func (bsq *BridgeStateQueue) Destroy() {
+	close(bsq.stopChan)
 	close(bsq.ch)
+	bsq.StopUnknownErrorReconnect()
+}
+
+func (bsq *BridgeStateQueue) StopUnknownErrorReconnect() {
+	if bsq == nil {
+		return
+	}
+	if cancelFn := bsq.stopReconnect.Swap(nil); cancelFn != nil {
+		(*cancelFn)()
+	}
+	if cancelFn := bsq.cancelScheduledNotice.Swap(nil); cancelFn != nil {
+		(*cancelFn)()
+	}
 }
 
 func (bsq *BridgeStateQueue) loop() {
-	defer func() {
-		err := recover()
-		if err != nil {
-			bsq.bridge.Log.Error().
-				Bytes(zerolog.ErrorStackFieldName, debug.Stack()).
-				Any(zerolog.ErrorFieldName, err).
-				Msg("Panic in bridge state loop")
-		}
-	}()
+	if CatchBridgeStateQueuePanics {
+		defer func() {
+			err := recover()
+			if err != nil {
+				bsq.login.Log.Error().
+					Bytes(zerolog.ErrorStackFieldName, debug.Stack()).
+					Any(zerolog.ErrorFieldName, err).
+					Msg("Panic in bridge state loop")
+			}
+		}()
+	}
 	for state := range bsq.ch {
 		bsq.immediateSendBridgeState(state)
 	}
 }
 
+func (bsq *BridgeStateQueue) scheduleNotice(triggeredBy status.BridgeState) {
+	log := bsq.login.Log.With().Str("action", "transient disconnect notice").Logger()
+	ctx := log.WithContext(bsq.bridge.BackgroundCtx)
+	if !bsq.waitForTransientDisconnectReconnect(ctx) {
+		return
+	}
+	prevUnsent := bsq.GetPrevUnsent()
+	prev := bsq.GetPrev()
+	if triggeredBy.Timestamp != prev.Timestamp || len(bsq.ch) > 0 || bsq.errorSent ||
+		prevUnsent.StateEvent != status.StateTransientDisconnect || prev.StateEvent != status.StateTransientDisconnect {
+		log.Trace().Any("triggered_by", triggeredBy).Msg("Not sending delayed transient disconnect notice")
+		return
+	}
+	log.Debug().Any("triggered_by", triggeredBy).Msg("Sending delayed transient disconnect notice")
+	bsq.sendNotice(ctx, triggeredBy, true)
+}
+
+func (bsq *BridgeStateQueue) sendNotice(ctx context.Context, state status.BridgeState, isDelayed bool) {
+	noticeConfig := bsq.bridge.Config.BridgeStatusNotices
+	isError := state.StateEvent == status.StateBadCredentials ||
+		state.StateEvent == status.StateUnknownError ||
+		state.UserAction == status.UserActionOpenNative ||
+		(isDelayed && state.StateEvent == status.StateTransientDisconnect)
+	sendNotice := noticeConfig == "all" || (noticeConfig == "errors" &&
+		(isError || (bsq.errorSent && state.StateEvent == status.StateConnected)))
+	if state.StateEvent != status.StateTransientDisconnect && state.StateEvent != status.StateUnknownError {
+		bsq.firstTransientDisconnect = time.Time{}
+	}
+	if !sendNotice {
+		if !bsq.errorSent && !isDelayed && noticeConfig == "errors" && state.StateEvent == status.StateTransientDisconnect {
+			if bsq.firstTransientDisconnect.IsZero() {
+				bsq.firstTransientDisconnect = time.Now()
+			}
+			go bsq.scheduleNotice(state)
+		}
+		return
+	}
+	managementRoom, err := bsq.login.User.GetManagementRoom(ctx)
+	if err != nil {
+		bsq.login.Log.Err(err).Msg("Failed to get management room")
+		return
+	}
+	name := bsq.login.RemoteName
+	if name == "" {
+		name = fmt.Sprintf("`%s`", bsq.login.ID)
+	}
+	message := fmt.Sprintf("State update for %s: `%s`", name, state.StateEvent)
+	if state.Error != "" {
+		message += fmt.Sprintf(" (`%s`)", state.Error)
+	}
+	if isDelayed {
+		message += fmt.Sprintf(" not resolved after waiting %s", exfmt.Duration(TransientDisconnectNoticeDelay))
+	}
+	if state.Message != "" {
+		message += fmt.Sprintf(": %s", state.Message)
+	}
+	content := format.RenderMarkdown(message, true, false)
+	if !isError {
+		content.MsgType = event.MsgNotice
+	}
+	_, err = bsq.bridge.Bot.SendMessage(ctx, managementRoom, event.EventMessage, &event.Content{
+		Parsed: content,
+		Raw: map[string]any{
+			"fi.mau.bridge_state": state,
+		},
+	}, nil)
+	if err != nil {
+		bsq.login.Log.Err(err).Msg("Failed to send bridge state notice")
+	} else {
+		bsq.errorSent = isError
+	}
+}
+
+func (bsq *BridgeStateQueue) unknownErrorReconnect(triggeredBy status.BridgeState) {
+	log := bsq.login.Log.With().Str("action", "unknown error reconnect").Logger()
+	ctx := log.WithContext(bsq.bridge.BackgroundCtx)
+	if !bsq.waitForUnknownErrorReconnect(ctx) {
+		return
+	}
+	prevUnsent := bsq.GetPrevUnsent()
+	prev := bsq.GetPrev()
+	if triggeredBy.Timestamp != prev.Timestamp {
+		log.Debug().Msg("Not reconnecting as a new bridge state was sent after the unknown error")
+		return
+	} else if len(bsq.ch) > 0 {
+		log.Warn().Msg("Not reconnecting as there are unsent bridge states")
+		return
+	} else if prevUnsent.StateEvent != status.StateUnknownError || prev.StateEvent != status.StateUnknownError {
+		log.Debug().Msg("Not reconnecting as the previous state was not an unknown error")
+		return
+	} else if bsq.unknownErrorReconnects > bsq.bridge.Config.UnknownErrorMaxAutoReconnects {
+		log.Warn().Msg("Not reconnecting as the maximum number of unknown error reconnects has been reached")
+		return
+	}
+	bsq.unknownErrorReconnects++
+	log.Info().
+		Int("reconnect_num", bsq.unknownErrorReconnects).
+		Msg("Disconnecting and reconnecting login due to unknown error")
+	bsq.login.Disconnect()
+	log.Debug().Msg("Disconnection finished, recreating client and reconnecting")
+	err := bsq.login.recreateClient(ctx)
+	if err != nil {
+		log.Err(err).Msg("Failed to recreate client after unknown error")
+		return
+	}
+	bsq.login.Client.Connect(ctx)
+	log.Debug().Msg("Reconnection finished")
+}
+
+func (bsq *BridgeStateQueue) waitForUnknownErrorReconnect(ctx context.Context) bool {
+	reconnectIn := bsq.bridge.Config.UnknownErrorAutoReconnect
+	// Don't allow too low values
+	if reconnectIn < 1*time.Minute {
+		return false
+	}
+	reconnectIn += time.Duration(rand.Int64N(int64(float64(reconnectIn)*0.4)) - int64(float64(reconnectIn)*0.2))
+	return bsq.waitForReconnect(ctx, reconnectIn, &bsq.stopReconnect)
+}
+
+const TransientDisconnectNoticeDelay = 3 * time.Minute
+
+func (bsq *BridgeStateQueue) waitForTransientDisconnectReconnect(ctx context.Context) bool {
+	timeUntilSchedule := time.Until(bsq.firstTransientDisconnect.Add(TransientDisconnectNoticeDelay))
+	zerolog.Ctx(ctx).Trace().
+		Stringer("duration", timeUntilSchedule).
+		Msg("Waiting before sending notice about transient disconnect")
+	return bsq.waitForReconnect(ctx, timeUntilSchedule, &bsq.cancelScheduledNotice)
+}
+
+func (bsq *BridgeStateQueue) waitForReconnect(
+	ctx context.Context, reconnectIn time.Duration, ptr *atomic.Pointer[context.CancelFunc],
+) bool {
+	cancelCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	if oldCancel := ptr.Swap(&cancel); oldCancel != nil {
+		(*oldCancel)()
+	}
+	select {
+	case <-time.After(reconnectIn):
+		return ptr.CompareAndSwap(&cancel, nil)
+	case <-cancelCtx.Done():
+		return false
+	case <-bsq.stopChan:
+		return false
+	}
+}
+
 func (bsq *BridgeStateQueue) immediateSendBridgeState(state status.BridgeState) {
+	if bsq.prevSent != nil && bsq.prevSent.ShouldDeduplicate(&state) {
+		bsq.login.Log.Debug().
+			Str("state_event", string(state.StateEvent)).
+			Msg("Not sending bridge state as it's a duplicate")
+		return
+	}
+	if state.StateEvent == status.StateUnknownError {
+		go bsq.unknownErrorReconnect(state)
+	}
+
+	ctx := bsq.login.Log.WithContext(context.Background())
+	bsq.sendNotice(ctx, state, false)
+
 	retryIn := 2
 	for {
-		if bsq.prev != nil && bsq.prev.ShouldDeduplicate(&state) {
-			bsq.bridge.Log.Debug().
-				Str("state_event", string(state.StateEvent)).
-				Msg("Not sending bridge state as it's a duplicate")
-			return
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 		err := bsq.bridge.Matrix.SendBridgeStatus(ctx, &state)
 		cancel()
 
 		if err != nil {
-			bsq.bridge.Log.Warn().Err(err).
+			bsq.login.Log.Warn().Err(err).
 				Int("retry_in_seconds", retryIn).
 				Msg("Failed to update bridge state")
 			time.Sleep(time.Duration(retryIn) * time.Second)
@@ -93,8 +281,8 @@ func (bsq *BridgeStateQueue) immediateSendBridgeState(state status.BridgeState)
 				retryIn = 64
 			}
 		} else {
-			bsq.prev = &state
-			bsq.bridge.Log.Debug().
+			bsq.prevSent = &state
+			bsq.login.Log.Debug().
 				Any("bridge_state", state).
 				Msg("Sent new bridge state")
 			return
@@ -107,10 +295,11 @@ func (bsq *BridgeStateQueue) Send(state status.BridgeState) {
 		return
 	}
 
-	state = state.Fill(bsq.user)
+	state = state.Fill(bsq.login)
+	bsq.prevUnsent = &state
 
 	if len(bsq.ch) >= 8 {
-		bsq.bridge.Log.Warn().Msg("Bridge state queue is nearly full, discarding an item")
+		bsq.login.Log.Warn().Msg("Bridge state queue is nearly full, discarding an item")
 		select {
 		case <-bsq.ch:
 		default:
@@ -119,19 +308,26 @@ func (bsq *BridgeStateQueue) Send(state status.BridgeState) {
 	select {
 	case bsq.ch <- state:
 	default:
-		bsq.bridge.Log.Error().Msg("Bridge state queue is full, dropped new state")
+		bsq.login.Log.Error().Msg("Bridge state queue is full, dropped new state")
 	}
 }
 
 func (bsq *BridgeStateQueue) GetPrev() status.BridgeState {
-	if bsq != nil && bsq.prev != nil {
-		return *bsq.prev
+	if bsq != nil && bsq.prevSent != nil {
+		return *bsq.prevSent
+	}
+	return status.BridgeState{}
+}
+
+func (bsq *BridgeStateQueue) GetPrevUnsent() status.BridgeState {
+	if bsq != nil && bsq.prevSent != nil {
+		return *bsq.prevUnsent
 	}
 	return status.BridgeState{}
 }
 
 func (bsq *BridgeStateQueue) SetPrev(prev status.BridgeState) {
 	if bsq != nil {
-		bsq.prev = &prev
+		bsq.prevSent = &prev
 	}
 }

bridgev2/commands/cleanup.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package commands
+
+import (
+	"maunium.net/go/mautrix/bridgev2"
+)
+
+var CommandDeletePortal = &FullHandler{
+	Func: func(ce *Event) {
+		// TODO clean up child portals?
+		err := ce.Portal.Delete(ce.Ctx)
+		if err != nil {
+			ce.Reply("Failed to delete portal: %v", err)
+			return
+		}
+		err = ce.Bot.DeleteRoom(ce.Ctx, ce.Portal.MXID, false)
+		if err != nil {
+			ce.Reply("Failed to clean up room: %v", err)
+		}
+		ce.MessageStatus.DisableMSS = true
+	},
+	Name: "delete-portal",
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Delete the current portal room",
+	},
+	RequiresAdmin:  true,
+	RequiresPortal: true,
+}
+
+var CommandDeleteAllPortals = &FullHandler{
+	Func: func(ce *Event) {
+		portals, err := ce.Bridge.GetAllPortals(ce.Ctx)
+		if err != nil {
+			ce.Reply("Failed to get portals: %v", err)
+			return
+		}
+		bridgev2.DeleteManyPortals(ce.Ctx, portals, func(portal *bridgev2.Portal, delete bool, err error) {
+			if !delete {
+				ce.Reply("Failed to delete portal %s: %v", portal.MXID, err)
+			} else {
+				ce.Reply("Failed to clean up room %s: %v", portal.MXID, err)
+			}
+		})
+	},
+	Name: "delete-all-portals",
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Delete all portals the bridge knows about",
+	},
+	RequiresAdmin: true,
+}
+
+var CommandSetManagementRoom = &FullHandler{
+	Func: func(ce *Event) {
+		if ce.User.ManagementRoom == ce.RoomID {
+			ce.Reply("This room is already your management room")
+			return
+		} else if ce.Portal != nil {
+			ce.Reply("This is a portal room: you can't set this as your management room")
+			return
+		}
+		members, err := ce.Bridge.Matrix.GetMembers(ce.Ctx, ce.RoomID)
+		if err != nil {
+			ce.Log.Err(err).Msg("Failed to get room members to check if room can be a management room")
+			ce.Reply("Failed to get room members")
+			return
+		}
+		_, hasBot := members[ce.Bot.GetMXID()]
+		if !hasBot {
+			// This reply will probably fail, but whatever
+			ce.Reply("The bridge bot must be in the room to set it as your management room")
+			return
+		} else if len(members) != 2 {
+			ce.Reply("Your management room must not have any members other than you and the bridge bot")
+			return
+		}
+		ce.User.ManagementRoom = ce.RoomID
+		err = ce.User.Save(ce.Ctx)
+		if err != nil {
+			ce.Log.Err(err).Msg("Failed to save management room")
+			ce.Reply("Failed to save management room")
+		} else {
+			ce.Reply("Management room updated")
+		}
+	},
+	Name: "set-management-room",
+	Help: HelpMeta{
+		Section:     HelpSectionGeneral,
+		Description: "Mark this room as your management room",
+	},
+}

bridgev2/commands/debug.go

@@ -7,10 +7,13 @@
 package commands
 
 import (
+	"encoding/json"
 	"strings"
+	"time"
 
 	"maunium.net/go/mautrix/bridgev2"
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/event"
 )
 
 var CommandRegisterPush = &FullHandler{
@@ -57,4 +60,66 @@ var CommandRegisterPush = &FullHandler{
 	},
 	RequiresAdmin: true,
 	RequiresLogin: true,
+	NetworkAPI:    NetworkAPIImplements[bridgev2.PushableNetworkAPI],
+}
+
+var CommandSendAccountData = &FullHandler{
+	Func: func(ce *Event) {
+		if len(ce.Args) < 2 {
+			ce.Reply("Usage: `$cmdprefix debug-account-data <type> <content>")
+			return
+		}
+		var content event.Content
+		evtType := event.Type{Type: ce.Args[0], Class: event.AccountDataEventType}
+		ce.RawArgs = strings.TrimSpace(strings.Trim(ce.RawArgs, ce.Args[0]))
+		err := json.Unmarshal([]byte(ce.RawArgs), &content)
+		if err != nil {
+			ce.Reply("Failed to parse JSON: %v", err)
+			return
+		}
+		err = content.ParseRaw(evtType)
+		if err != nil {
+			ce.Reply("Failed to deserialize content: %v", err)
+			return
+		}
+		res := ce.Bridge.QueueMatrixEvent(ce.Ctx, &event.Event{
+			Sender:    ce.User.MXID,
+			Type:      evtType,
+			Timestamp: time.Now().UnixMilli(),
+			RoomID:    ce.RoomID,
+			Content:   content,
+		})
+		ce.Reply("Result: %+v", res)
+	},
+	Name: "debug-account-data",
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Send a room account data event to the bridge",
+		Args:        "<_type_> <_content_>",
+	},
+	RequiresAdmin:  true,
+	RequiresPortal: true,
+	RequiresLogin:  true,
+}
+
+var CommandResetNetwork = &FullHandler{
+	Func: func(ce *Event) {
+		if strings.Contains(strings.ToLower(ce.RawArgs), "--reset-transport") {
+			nrn, ok := ce.Bridge.Network.(bridgev2.NetworkResettingNetwork)
+			if ok {
+				nrn.ResetHTTPTransport()
+			} else {
+				ce.Reply("Network connector does not support resetting HTTP transport")
+			}
+		}
+		ce.Bridge.ResetNetworkConnections()
+		ce.React("✅️")
+	},
+	Name: "debug-reset-network",
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Reset network connections to the remote network",
+		Args:        "[--reset-transport]",
+	},
+	RequiresAdmin: true,
 }

bridgev2/commands/event.go

@@ -24,20 +24,21 @@ import (
 
 // Event stores all data which might be used to handle commands
 type Event struct {
-	Bot       bridgev2.MatrixAPI
-	Bridge    *bridgev2.Bridge
-	Portal    *bridgev2.Portal
-	Processor *Processor
-	Handler   MinimalCommandHandler
-	RoomID    id.RoomID
-	EventID   id.EventID
-	User      *bridgev2.User
-	Command   string
-	Args      []string
-	RawArgs   string
-	ReplyTo   id.EventID
-	Ctx       context.Context
-	Log       *zerolog.Logger
+	Bot        bridgev2.MatrixAPI
+	Bridge     *bridgev2.Bridge
+	Portal     *bridgev2.Portal
+	Processor  *Processor
+	Handler    MinimalCommandHandler
+	RoomID     id.RoomID
+	OrigRoomID id.RoomID
+	EventID    id.EventID
+	User       *bridgev2.User
+	Command    string
+	Args       []string
+	RawArgs    string
+	ReplyTo    id.EventID
+	Ctx        context.Context
+	Log        *zerolog.Logger
 
 	MessageStatus *bridgev2.MessageStatus
 }
@@ -56,15 +57,15 @@ func (ce *Event) Reply(msg string, args ...any) {
 func (ce *Event) ReplyAdvanced(msg string, allowMarkdown, allowHTML bool) {
 	content := format.RenderMarkdown(msg, allowMarkdown, allowHTML)
 	content.MsgType = event.MsgNotice
-	_, err := ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventMessage, &event.Content{Parsed: content}, time.Now())
+	_, err := ce.Bot.SendMessage(ce.Ctx, ce.OrigRoomID, event.EventMessage, &event.Content{Parsed: &content}, nil)
 	if err != nil {
-		ce.Log.Err(err).Msgf("Failed to reply to command")
+		ce.Log.Err(err).Msg("Failed to reply to command")
 	}
 }
 
 // React sends a reaction to the command.
 func (ce *Event) React(key string) {
-	_, err := ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventReaction, &event.Content{
+	_, err := ce.Bot.SendMessage(ce.Ctx, ce.OrigRoomID, event.EventReaction, &event.Content{
 		Parsed: &event.ReactionEventContent{
 			RelatesTo: event.RelatesTo{
 				Type:    event.RelAnnotation,
@@ -72,29 +73,28 @@ func (ce *Event) React(key string) {
 				Key:     key,
 			},
 		},
-	}, time.Now())
+	}, nil)
 	if err != nil {
-		ce.Log.Err(err).Msgf("Failed to react to command")
+		ce.Log.Err(err).Msg("Failed to react to command")
 	}
 }
 
 // Redact redacts the command.
 func (ce *Event) Redact(req ...mautrix.ReqRedact) {
-	_, err := ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventRedaction, &event.Content{
+	_, err := ce.Bot.SendMessage(ce.Ctx, ce.OrigRoomID, event.EventRedaction, &event.Content{
 		Parsed: &event.RedactionEventContent{
 			Redacts: ce.EventID,
 		},
-	}, time.Now())
+	}, nil)
 	if err != nil {
-		ce.Log.Err(err).Msgf("Failed to redact command")
+		ce.Log.Err(err).Msg("Failed to redact command")
 	}
 }
 
 // MarkRead marks the command event as read.
 func (ce *Event) MarkRead() {
-	// TODO
-	//err := ce.Bot.SendReceipt(ce.Ctx, ce.RoomID, ce.EventID, event.ReceiptTypeRead, nil)
-	//if err != nil {
-	//	ce.Log.Err(err).Msgf("Failed to mark command as read")
-	//}
+	err := ce.Bot.MarkRead(ce.Ctx, ce.RoomID, ce.EventID, time.Now())
+	if err != nil {
+		ce.Log.Err(err).Msg("Failed to mark command as read")
+	}
 }

bridgev2/commands/handler.go

@@ -7,6 +7,7 @@
 package commands
 
 import (
+	"maunium.net/go/mautrix/bridgev2"
 	"maunium.net/go/mautrix/event"
 )
 
@@ -37,6 +38,18 @@ type AliasedCommandHandler interface {
 	GetAliases() []string
 }
 
+func NetworkAPIImplements[T bridgev2.NetworkAPI](val bridgev2.NetworkAPI) bool {
+	_, ok := val.(T)
+	return ok
+}
+
+func NetworkConnectorImplements[T bridgev2.NetworkConnector](val bridgev2.NetworkConnector) bool {
+	_, ok := val.(T)
+	return ok
+}
+
+type ImplementationChecker[T any] func(val T) bool
+
 type FullHandler struct {
 	Func func(*Event)
 
@@ -49,6 +62,9 @@ type FullHandler struct {
 	RequiresLogin           bool
 	RequiresEventLevel      event.Type
 	RequiresLoginPermission bool
+
+	NetworkAPI       ImplementationChecker[bridgev2.NetworkAPI]
+	NetworkConnector ImplementationChecker[bridgev2.NetworkConnector]
 }
 
 func (fh *FullHandler) GetHelp() HelpMeta {
@@ -64,9 +80,15 @@ func (fh *FullHandler) GetAliases() []string {
 	return fh.Aliases
 }
 
+func (fh *FullHandler) ImplementationsFulfilled(ce *Event) bool {
+	// TODO add dedicated method to get an empty NetworkAPI instead of getting default login
+	client := ce.User.GetDefaultLogin()
+	return (fh.NetworkAPI == nil || client == nil || fh.NetworkAPI(client.Client)) &&
+		(fh.NetworkConnector == nil || fh.NetworkConnector(ce.Bridge.Network))
+}
+
 func (fh *FullHandler) ShowInHelp(ce *Event) bool {
-	return true
-	//return !fh.RequiresAdmin || ce.User.GetPermissionLevel() >= bridgeconfig.PermissionLevelAdmin
+	return fh.ImplementationsFulfilled(ce) && (!fh.RequiresAdmin || ce.User.Permissions.Admin)
 }
 
 func (fh *FullHandler) userHasRoomPermission(ce *Event) bool {

bridgev2/commands/login.go

@@ -10,17 +10,18 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
+	"html"
+	"net/url"
 	"regexp"
+	"slices"
 	"strings"
-	"time"
 
 	"github.com/skip2/go-qrcode"
-	"golang.org/x/net/html"
+	"go.mau.fi/util/curl"
 
 	"maunium.net/go/mautrix/bridgev2"
-
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
@@ -36,6 +37,17 @@ var CommandLogin = &FullHandler{
 	RequiresLoginPermission: true,
 }
 
+var CommandRelogin = &FullHandler{
+	Func: fnLogin,
+	Name: "relogin",
+	Help: HelpMeta{
+		Section:     HelpSectionAuth,
+		Description: "Re-authenticate an existing login",
+		Args:        "<_login ID_> [_flow ID_]",
+	},
+	RequiresLoginPermission: true,
+}
+
 func formatFlowsReply(flows []bridgev2.LoginFlow) string {
 	var buf strings.Builder
 	for _, flow := range flows {
@@ -45,10 +57,33 @@ func formatFlowsReply(flows []bridgev2.LoginFlow) string {
 }
 
 func fnLogin(ce *Event) {
+	var reauth *bridgev2.UserLogin
+	if ce.Command == "relogin" {
+		if len(ce.Args) == 0 {
+			ce.Reply("Usage: `$cmdprefix relogin <login ID> [_flow ID_]`\n\nYour logins:\n\n%s", ce.User.GetFormattedUserLogins())
+			return
+		}
+		reauth = ce.Bridge.GetCachedUserLoginByID(networkid.UserLoginID(ce.Args[0]))
+		if reauth == nil {
+			ce.Reply("Login `%s` not found", ce.Args[0])
+			return
+		}
+		ce.Args = ce.Args[1:]
+	}
+	if reauth == nil && ce.User.HasTooManyLogins() {
+		ce.Reply(
+			"You have reached the maximum number of logins (%d). "+
+				"Please logout from an existing login before creating a new one. "+
+				"If you want to re-authenticate an existing login, use the `$cmdprefix relogin` command.",
+			ce.User.Permissions.MaxLogins,
+		)
+		return
+	}
 	flows := ce.Bridge.Network.GetLoginFlows()
 	var chosenFlowID string
 	if len(ce.Args) > 0 {
 		inputFlowID := strings.ToLower(ce.Args[0])
+		ce.Args = ce.Args[1:]
 		for _, flow := range flows {
 			if flow.ID == inputFlowID {
 				chosenFlowID = flow.ID
@@ -56,13 +91,17 @@ func fnLogin(ce *Event) {
 			}
 		}
 		if chosenFlowID == "" {
-			ce.Reply("Invalid login flow `%s`. Available options:\n\n%s", ce.Args[0], formatFlowsReply(flows))
+			ce.Reply("Invalid login flow `%s`. Available options:\n\n%s", inputFlowID, formatFlowsReply(flows))
 			return
 		}
 	} else if len(flows) == 1 {
 		chosenFlowID = flows[0].ID
 	} else {
-		ce.Reply("Please specify a login flow, e.g. `login %s`.\n\n%s", flows[0].ID, formatFlowsReply(flows))
+		if reauth != nil {
+			ce.Reply("Please specify a login flow, e.g. `relogin %s %s`.\n\n%s", reauth.ID, flows[0].ID, formatFlowsReply(flows))
+		} else {
+			ce.Reply("Please specify a login flow, e.g. `login %s`.\n\n%s", flows[0].ID, formatFlowsReply(flows))
+		}
 		return
 	}
 
@@ -71,27 +110,104 @@ func fnLogin(ce *Event) {
 		ce.Reply("Failed to prepare login process: %v", err)
 		return
 	}
-	nextStep, err := login.Start(ce.Ctx)
+	overridable, ok := login.(bridgev2.LoginProcessWithOverride)
+	var nextStep *bridgev2.LoginStep
+	if ok && reauth != nil {
+		nextStep, err = overridable.StartWithOverride(ce.Ctx, reauth)
+	} else {
+		nextStep, err = login.Start(ce.Ctx)
+	}
 	if err != nil {
 		ce.Reply("Failed to start login: %v", err)
 		return
 	}
-	doLoginStep(ce, login, nextStep)
+	ce.Log.Debug().Any("first_step", nextStep).Msg("Created login process")
+
+	nextStep = checkLoginCommandDirectParams(ce, login, nextStep)
+	if nextStep != nil {
+		doLoginStep(ce, login, nextStep, reauth)
+	}
+}
+
+func checkLoginCommandDirectParams(ce *Event, login bridgev2.LoginProcess, nextStep *bridgev2.LoginStep) *bridgev2.LoginStep {
+	if len(ce.Args) == 0 {
+		return nextStep
+	}
+	var ok bool
+	defer func() {
+		if !ok {
+			login.Cancel()
+		}
+	}()
+	var err error
+	switch nextStep.Type {
+	case bridgev2.LoginStepTypeDisplayAndWait:
+		ce.Reply("Invalid extra parameters for display and wait login step")
+		return nil
+	case bridgev2.LoginStepTypeUserInput:
+		if len(ce.Args) != len(nextStep.UserInputParams.Fields) {
+			ce.Reply("Invalid number of extra parameters (expected 0 or %d, got %d)", len(nextStep.UserInputParams.Fields), len(ce.Args))
+			return nil
+		}
+		input := make(map[string]string)
+		var shouldRedact bool
+		for i, param := range nextStep.UserInputParams.Fields {
+			param.FillDefaultValidate()
+			input[param.ID], err = param.Validate(ce.Args[i])
+			if err != nil {
+				ce.Reply("Invalid value for %s: %v", param.Name, err)
+				return nil
+			}
+			if param.Type == bridgev2.LoginInputFieldTypePassword || param.Type == bridgev2.LoginInputFieldTypeToken {
+				shouldRedact = true
+			}
+		}
+		if shouldRedact {
+			ce.Redact()
+		}
+		nextStep, err = login.(bridgev2.LoginProcessUserInput).SubmitUserInput(ce.Ctx, input)
+	case bridgev2.LoginStepTypeCookies:
+		if len(ce.Args) != len(nextStep.CookiesParams.Fields) {
+			ce.Reply("Invalid number of extra parameters (expected 0 or %d, got %d)", len(nextStep.CookiesParams.Fields), len(ce.Args))
+			return nil
+		}
+		input := make(map[string]string)
+		for i, param := range nextStep.CookiesParams.Fields {
+			val := maybeURLDecodeCookie(ce.Args[i], &param)
+			if match, _ := regexp.MatchString(param.Pattern, val); !match {
+				ce.Reply("Invalid value for %s: `%s` doesn't match regex `%s`", param.ID, val, param.Pattern)
+				return nil
+			}
+			input[param.ID] = val
+		}
+		ce.Redact()
+		nextStep, err = login.(bridgev2.LoginProcessCookies).SubmitCookies(ce.Ctx, input)
+	}
+	if err != nil {
+		ce.Reply("Failed to submit input: %v", err)
+		return nil
+	}
+	ok = true
+	return nextStep
 }
 
 type userInputLoginCommandState struct {
 	Login           bridgev2.LoginProcessUserInput
 	Data            map[string]string
 	RemainingFields []bridgev2.LoginInputDataField
+	Override        *bridgev2.UserLogin
 }
 
 func (uilcs *userInputLoginCommandState) promptNext(ce *Event) {
 	field := uilcs.RemainingFields[0]
+	parts := []string{fmt.Sprintf("Please enter your %s", field.Name)}
 	if field.Description != "" {
-		ce.Reply("Please enter your %s\n%s", field.Name, field.Description)
-	} else {
-		ce.Reply("Please enter your %s", field.Name)
+		parts = append(parts, field.Description)
 	}
+	if len(field.Options) > 0 {
+		parts = append(parts, fmt.Sprintf("Options: `%s`", strings.Join(field.Options, "`, `")))
+	}
+	ce.Reply(strings.Join(parts, "\n"))
 	StoreCommandState(ce.User, &CommandState{
 		Next:   MinimalCommandHandlerFunc(uilcs.submitNext),
 		Action: "Login",
@@ -103,7 +219,7 @@ func (uilcs *userInputLoginCommandState) promptNext(ce *Event) {
 func (uilcs *userInputLoginCommandState) submitNext(ce *Event) {
 	field := uilcs.RemainingFields[0]
 	field.FillDefaultValidate()
-	if field.Type == bridgev2.LoginInputFieldTypePassword {
+	if field.Type == bridgev2.LoginInputFieldTypePassword || field.Type == bridgev2.LoginInputFieldTypeToken {
 		ce.Redact()
 	}
 	var err error
@@ -120,7 +236,7 @@ func (uilcs *userInputLoginCommandState) submitNext(ce *Event) {
 	if nextStep, err := uilcs.Login.SubmitUserInput(ce.Ctx, uilcs.Data); err != nil {
 		ce.Reply("Failed to submit input: %v", err)
 	} else {
-		doLoginStep(ce, uilcs.Login, nextStep)
+		doLoginStep(ce, uilcs.Login, nextStep, uilcs.Override)
 	}
 }
 
@@ -136,19 +252,24 @@ func sendQR(ce *Event, qr string, prevEventID *id.EventID) error {
 		return fmt.Errorf("failed to upload image: %w", err)
 	}
 	content := &event.MessageEventContent{
-		MsgType:  event.MsgImage,
-		FileName: "qr.png",
-		URL:      qrMXC,
-		File:     qrFile,
-
+		MsgType:       event.MsgImage,
+		FileName:      "qr.png",
+		URL:           qrMXC,
+		File:          qrFile,
 		Body:          qr,
 		Format:        event.FormatHTML,
 		FormattedBody: fmt.Sprintf("<pre><code>%s</code></pre>", html.EscapeString(qr)),
+		Info: &event.FileInfo{
+			MimeType: "image/png",
+			Width:    qrSizePx,
+			Height:   qrSizePx,
+			Size:     len(qrData),
+		},
 	}
 	if *prevEventID != "" {
 		content.SetEdit(*prevEventID)
 	}
-	newEventID, err := ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventMessage, &event.Content{Parsed: content}, time.Now())
+	newEventID, err := ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventMessage, &event.Content{Parsed: content}, nil)
 	if err != nil {
 		return err
 	}
@@ -158,18 +279,55 @@ func sendQR(ce *Event, qr string, prevEventID *id.EventID) error {
 	return nil
 }
 
+func sendUserInputAttachments(ce *Event, atts []*bridgev2.LoginUserInputAttachment) error {
+	for _, att := range atts {
+		if att.FileName == "" {
+			return fmt.Errorf("missing attachment filename")
+		}
+		mxc, file, err := ce.Bot.UploadMedia(ce.Ctx, ce.RoomID, att.Content, att.FileName, att.Info.MimeType)
+		if err != nil {
+			return fmt.Errorf("failed to upload attachment %q: %w", att.FileName, err)
+		}
+		content := &event.MessageEventContent{
+			MsgType:  att.Type,
+			FileName: att.FileName,
+			URL:      mxc,
+			File:     file,
+			Info: &event.FileInfo{
+				MimeType: att.Info.MimeType,
+				Width:    att.Info.Width,
+				Height:   att.Info.Height,
+				Size:     att.Info.Size,
+			},
+			Body: att.FileName,
+		}
+		_, err = ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventMessage, &event.Content{Parsed: content}, nil)
+		if err != nil {
+			return nil
+		}
+	}
+	return nil
+}
+
 type contextKey int
 
 const (
 	contextKeyPrevEventID contextKey = iota
 )
 
-func doLoginDisplayAndWait(ce *Event, login bridgev2.LoginProcessDisplayAndWait, step *bridgev2.LoginStep) {
+func doLoginDisplayAndWait(ce *Event, login bridgev2.LoginProcessDisplayAndWait, step *bridgev2.LoginStep, override *bridgev2.UserLogin) {
 	prevEvent, ok := ce.Ctx.Value(contextKeyPrevEventID).(*id.EventID)
 	if !ok {
 		prevEvent = new(id.EventID)
 		ce.Ctx = context.WithValue(ce.Ctx, contextKeyPrevEventID, prevEvent)
 	}
+	cancelCtx, cancelFunc := context.WithCancel(ce.Ctx)
+	defer cancelFunc()
+	StoreCommandState(ce.User, &CommandState{
+		Action: "Login",
+		Cancel: cancelFunc,
+	})
+	defer StoreCommandState(ce.User, nil)
 	switch step.DisplayAndWaitParams.Type {
 	case bridgev2.LoginDisplayTypeQR:
 		err := sendQR(ce, step.DisplayAndWaitParams.Data, prevEvent)
@@ -189,29 +347,31 @@ func doLoginDisplayAndWait(ce *Event, login bridgev2.LoginProcessDisplayAndWait,
 		login.Cancel()
 		return
 	}
-	nextStep, err := login.Wait(ce.Ctx)
+	nextStep, err := login.Wait(cancelCtx)
 	// Redact the QR code, unless the next step is refreshing the code (in which case the event is just edited)
 	if *prevEvent != "" && (nextStep == nil || nextStep.StepID != step.StepID) {
 		_, _ = ce.Bot.SendMessage(ce.Ctx, ce.RoomID, event.EventRedaction, &event.Content{
 			Parsed: &event.RedactionEventContent{
 				Redacts: *prevEvent,
 			},
-		}, time.Now())
+		}, nil)
 		*prevEvent = ""
 	}
 	if err != nil {
 		ce.Reply("Login failed: %v", err)
 		return
 	}
-	doLoginStep(ce, login, nextStep)
+	doLoginStep(ce, login, nextStep, override)
 }
 
 type cookieLoginCommandState struct {
-	Login bridgev2.LoginProcessCookies
-	Data  *bridgev2.LoginCookiesParams
+	Login    bridgev2.LoginProcessCookies
+	Data     *bridgev2.LoginCookiesParams
+	Override *bridgev2.UserLogin
 }
 
 func (clcs *cookieLoginCommandState) prompt(ce *Event) {
+	ce.Reply("Login URL: <%s>", clcs.Data.URL)
 	StoreCommandState(ce.User, &CommandState{
 		Next:   MinimalCommandHandlerFunc(clcs.submit),
 		Action: "Login",
@@ -220,86 +380,162 @@ func (clcs *cookieLoginCommandState) prompt(ce *Event) {
 	})
 }
 
-var curlCookieRegex = regexp.MustCompile(`-H '[cC]ookie: ([^']*)'`)
-
-func missingKeys(required []string, data map[string]string) (missing []string) {
-	for _, requiredKey := range required {
-		if _, ok := data[requiredKey]; !ok {
-			missing = append(missing, requiredKey)
-		}
-	}
-	return
-}
-
 func (clcs *cookieLoginCommandState) submit(ce *Event) {
 	ce.Redact()
 
-	cookies := make(map[string]string)
+	cookiesInput := make(map[string]string)
 	if strings.HasPrefix(strings.TrimSpace(ce.RawArgs), "curl") {
-		if len(clcs.Data.LocalStorageKeys) > 0 || len(clcs.Data.SpecialKeys) > 0 {
-			ce.Reply("Special keys and localStorage keys can't be extracted from curl commands - please provide the data as JSON instead")
+		parsed, err := curl.Parse(ce.RawArgs)
+		if err != nil {
+			ce.Reply("Failed to parse curl: %v", err)
 			return
 		}
-		cookieHeader := curlCookieRegex.FindStringSubmatch(ce.RawArgs)
-		if len(cookieHeader) != 2 {
-			ce.Reply("Couldn't find `-H 'Cookie: ...'` in curl command")
-			return
+		reqCookies := make(map[string]string)
+		for _, cookie := range parsed.Cookies() {
+			reqCookies[cookie.Name], err = url.PathUnescape(cookie.Value)
+			if err != nil {
+				ce.Reply("Failed to parse cookie %s: %v", cookie.Name, err)
+				return
+			}
 		}
-		parsed := (&http.Request{Header: http.Header{"Cookie": {cookieHeader[1]}}}).Cookies()
-		for _, cookie := range parsed {
-			cookies[cookie.Name] = cookie.Value
+		var missingKeys, unsupportedKeys []string
+		for _, field := range clcs.Data.Fields {
+			var value string
+			var supported bool
+			for _, src := range field.Sources {
+				switch src.Type {
+				case bridgev2.LoginCookieTypeCookie:
+					supported = true
+					value = reqCookies[src.Name]
+				case bridgev2.LoginCookieTypeRequestHeader:
+					supported = true
+					value = parsed.Header.Get(src.Name)
+				case bridgev2.LoginCookieTypeRequestBody:
+					supported = true
+					switch {
+					case parsed.MultipartForm != nil:
+						values, ok := parsed.MultipartForm.Value[src.Name]
+						if ok && len(values) > 0 {
+							value = values[0]
+						}
+					case parsed.ParsedJSON != nil:
+						untypedValue, ok := parsed.ParsedJSON[src.Name]
+						if ok {
+							value = fmt.Sprintf("%v", untypedValue)
+						}
+					}
+				}
+				if value != "" {
+					cookiesInput[field.ID] = value
+					break
+				}
+			}
+			if value == "" && field.Required {
+				if supported {
+					missingKeys = append(missingKeys, field.ID)
+				} else {
+					unsupportedKeys = append(unsupportedKeys, field.ID)
+				}
+			}
+		}
+		if len(unsupportedKeys) > 0 {
+			ce.Reply("Some keys can't be extracted from a cURL request: %+v\n\nPlease provide a JSON object instead.", unsupportedKeys)
+			return
+		} else if len(missingKeys) > 0 {
+			ce.Reply("Missing some keys: %+v", missingKeys)
+			return
 		}
 	} else {
-		err := json.Unmarshal([]byte(ce.RawArgs), &cookies)
+		err := json.Unmarshal([]byte(ce.RawArgs), &cookiesInput)
 		if err != nil {
 			ce.Reply("Failed to parse input as JSON: %v", err)
 			return
 		}
+		for _, field := range clcs.Data.Fields {
+			val, ok := cookiesInput[field.ID]
+			if ok {
+				cookiesInput[field.ID] = maybeURLDecodeCookie(val, &field)
+			}
+		}
 	}
-	missingCookies := missingKeys(clcs.Data.CookieKeys, cookies)
-	if len(missingCookies) > 0 {
-		ce.Reply("Missing required cookies: %+v", missingCookies)
-		return
+	var missingKeys []string
+	for _, field := range clcs.Data.Fields {
+		val, ok := cookiesInput[field.ID]
+		if !ok && field.Required {
+			missingKeys = append(missingKeys, field.ID)
+		}
+		if match, _ := regexp.MatchString(field.Pattern, val); !match {
+			ce.Reply("Invalid value for %s: `%s` doesn't match regex `%s`", field.ID, val, field.Pattern)
+			return
+		}
 	}
-	missingLocalStorage := missingKeys(clcs.Data.LocalStorageKeys, cookies)
-	if len(missingLocalStorage) > 0 {
-		ce.Reply("Missing required localStorage keys: %+v", missingLocalStorage)
-		return
-	}
-	missingSpecial := missingKeys(clcs.Data.SpecialKeys, cookies)
-	if len(missingSpecial) > 0 {
-		ce.Reply("Missing required special keys: %+v", missingSpecial)
+	if len(missingKeys) > 0 {
+		ce.Reply("Missing some keys: %+v", missingKeys)
 		return
 	}
 	StoreCommandState(ce.User, nil)
-	nextStep, err := clcs.Login.SubmitCookies(ce.Ctx, cookies)
+	nextStep, err := clcs.Login.SubmitCookies(ce.Ctx, cookiesInput)
 	if err != nil {
 		ce.Reply("Login failed: %v", err)
+		return
 	}
-	doLoginStep(ce, clcs.Login, nextStep)
+	doLoginStep(ce, clcs.Login, nextStep, clcs.Override)
 }
 
-func doLoginStep(ce *Event, login bridgev2.LoginProcess, step *bridgev2.LoginStep) {
+func maybeURLDecodeCookie(val string, field *bridgev2.LoginCookieField) string {
+	if val == "" {
+		return val
+	}
+	isCookie := slices.ContainsFunc(field.Sources, func(src bridgev2.LoginCookieFieldSource) bool {
+		return src.Type == bridgev2.LoginCookieTypeCookie
+	})
+	if !isCookie {
+		return val
+	}
+	decoded, err := url.PathUnescape(val)
+	if err != nil {
+		return val
+	}
+	return decoded
+}
+
+func doLoginStep(ce *Event, login bridgev2.LoginProcess, step *bridgev2.LoginStep, override *bridgev2.UserLogin) {
+	ce.Log.Debug().Any("next_step", step).Msg("Got next login step")
 	if step.Instructions != "" {
 		ce.Reply(step.Instructions)
 	}
 
 	switch step.Type {
 	case bridgev2.LoginStepTypeDisplayAndWait:
-		doLoginDisplayAndWait(ce, login.(bridgev2.LoginProcessDisplayAndWait), step)
+		doLoginDisplayAndWait(ce, login.(bridgev2.LoginProcessDisplayAndWait), step, override)
 	case bridgev2.LoginStepTypeCookies:
 		(&cookieLoginCommandState{
-			Login: login.(bridgev2.LoginProcessCookies),
-			Data:  step.CookiesParams,
+			Login:    login.(bridgev2.LoginProcessCookies),
+			Data:     step.CookiesParams,
+			Override: override,
 		}).prompt(ce)
 	case bridgev2.LoginStepTypeUserInput:
+		err := sendUserInputAttachments(ce, step.UserInputParams.Attachments)
+		if err != nil {
+			ce.Reply("Failed to send attachments: %v", err)
+		}
 		(&userInputLoginCommandState{
 			Login:           login.(bridgev2.LoginProcessUserInput),
 			RemainingFields: step.UserInputParams.Fields,
 			Data:            make(map[string]string),
+			Override:        override,
 		}).promptNext(ce)
 	case bridgev2.LoginStepTypeComplete:
-		// Nothing to do other than instructions
+		if override != nil && override.ID != step.CompleteParams.UserLoginID {
+			ce.Log.Info().
+				Str("old_login_id", string(override.ID)).
+				Str("new_login_id", string(step.CompleteParams.UserLoginID)).
+				Msg("Login resulted in different remote ID than what was being overridden. Deleting previous login")
+			override.Delete(ce.Ctx, status.BridgeState{
+				StateEvent: status.StateLoggedOut,
+				Reason:     "LOGIN_OVERRIDDEN",
+			}, bridgev2.DeleteOpts{LogoutRemote: true})
+		}
 	default:
 		panic(fmt.Errorf("unknown login step type %q", step.Type))
 	}

bridgev2/commands/processor.go

@@ -17,8 +17,7 @@ import (
 	"github.com/rs/zerolog"
 
 	"maunium.net/go/mautrix/bridgev2"
-
-	"maunium.net/go/mautrix/bridge/status"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
@@ -42,10 +41,12 @@ func NewProcessor(bridge *bridgev2.Bridge) bridgev2.CommandProcessor {
 	}
 	proc.AddHandlers(
 		CommandHelp, CommandCancel,
-		CommandRegisterPush, CommandDeletePortal,
-		CommandLogin, CommandListLogins, CommandLogout, CommandSetPreferredLogin,
+		CommandRegisterPush, CommandSendAccountData, CommandResetNetwork,
+		CommandDeletePortal, CommandDeleteAllPortals, CommandSetManagementRoom,
+		CommandLogin, CommandRelogin, CommandListLogins, CommandLogout, CommandSetPreferredLogin,
 		CommandSetRelay, CommandUnsetRelay,
-		CommandResolveIdentifier, CommandStartChat,
+		CommandResolveIdentifier, CommandStartChat, CommandCreateGroup, CommandSearch, CommandSyncChat, CommandMute,
+		CommandSudo, CommandDoIn,
 	)
 	return proc
 }
@@ -72,16 +73,18 @@ func (proc *Processor) Handle(ctx context.Context, roomID id.RoomID, eventID id.
 		Step:   status.MsgStepCommand,
 		Status: event.MessageStatusSuccess,
 	}
+	logCopy := zerolog.Ctx(ctx).With().Logger()
+	log := &logCopy
 	defer func() {
 		statusInfo := &bridgev2.MessageStatusEventInfo{
-			RoomID:    roomID,
-			EventID:   eventID,
-			EventType: event.EventMessage,
-			Sender:    user.MXID,
+			RoomID:        roomID,
+			SourceEventID: eventID,
+			EventType:     event.EventMessage,
+			Sender:        user.MXID,
 		}
 		err := recover()
 		if err != nil {
-			logEvt := zerolog.Ctx(ctx).Error().
+			logEvt := log.Error().
 				Bytes(zerolog.ErrorStackFieldName, debug.Stack())
 			if realErr, ok := err.(error); ok {
 				logEvt = logEvt.Err(realErr)
@@ -108,29 +111,36 @@ func (proc *Processor) Handle(ctx context.Context, roomID id.RoomID, eventID id.
 	rawArgs := strings.TrimLeft(strings.TrimPrefix(message, command), " ")
 	portal, err := proc.bridge.GetPortalByMXID(ctx, roomID)
 	if err != nil {
+		log.Err(err).Msg("Failed to get portal")
 		// :(
 	}
 	ce := &Event{
-		Bot:       proc.bridge.Bot,
-		Bridge:    proc.bridge,
-		Portal:    portal,
-		Processor: proc,
-		RoomID:    roomID,
-		EventID:   eventID,
-		User:      user,
-		Command:   command,
-		Args:      args[1:],
-		RawArgs:   rawArgs,
-		ReplyTo:   replyTo,
-		Ctx:       ctx,
+		Bot:        proc.bridge.Bot,
+		Bridge:     proc.bridge,
+		Portal:     portal,
+		Processor:  proc,
+		RoomID:     roomID,
+		OrigRoomID: roomID,
+		EventID:    eventID,
+		User:       user,
+		Command:    command,
+		Args:       args[1:],
+		RawArgs:    rawArgs,
+		ReplyTo:    replyTo,
+		Ctx:        ctx,
+		Log:        log,
 
 		MessageStatus: ms,
 	}
+	proc.handleCommand(ctx, ce, message, args)
+}
 
+func (proc *Processor) handleCommand(ctx context.Context, ce *Event, origMessage string, origArgs []string) {
 	realCommand, ok := proc.aliases[ce.Command]
 	if !ok {
 		realCommand = ce.Command
 	}
+	log := zerolog.Ctx(ctx)
 
 	var handler MinimalCommandHandler
 	handler, ok = proc.handlers[realCommand]
@@ -138,23 +148,22 @@ func (proc *Processor) Handle(ctx context.Context, roomID id.RoomID, eventID id.
 		state := LoadCommandState(ce.User)
 		if state != nil && state.Next != nil {
 			ce.Command = ""
-			ce.RawArgs = message
-			ce.Args = args
+			ce.RawArgs = origMessage
+			ce.Args = origArgs
 			ce.Handler = state.Next
-			log := zerolog.Ctx(ctx).With().Str("action", state.Action).Logger()
-			ce.Log = &log
-			ce.Ctx = log.WithContext(ctx)
+			log.UpdateContext(func(c zerolog.Context) zerolog.Context {
+				return c.Str("action", state.Action)
+			})
 			log.Debug().Msg("Received reply to command state")
 			state.Next.Run(ce)
 		} else {
-			zerolog.Ctx(ctx).Debug().Str("mx_command", command).Msg("Received unknown command")
+			zerolog.Ctx(ctx).Debug().Str("mx_command", ce.Command).Msg("Received unknown command")
 			ce.Reply("Unknown command, use the `help` command for help.")
 		}
 	} else {
-		log := zerolog.Ctx(ctx).With().Str("mx_command", command).Logger()
-		ctx = log.WithContext(ctx)
-		ce.Log = &log
-		ce.Ctx = ctx
+		log.UpdateContext(func(c zerolog.Context) zerolog.Context {
+			return c.Str("mx_command", ce.Command)
+		})
 		log.Debug().Msg("Received command")
 		ce.Handler = handler
 		handler.Run(ce)

bridgev2/commands/relay.go

@@ -35,9 +35,14 @@ func fnSetRelay(ce *Event) {
 		ce.Reply("You don't have permission to manage the relay in this room")
 		return
 	}
+	onlySetDefaultRelays := !ce.User.Permissions.Admin && ce.Bridge.Config.Relay.AdminOnly
 	var relay *bridgev2.UserLogin
-	if len(ce.Args) == 0 {
+	if len(ce.Args) == 0 && ce.Portal.Receiver == "" {
 		relay = ce.User.GetDefaultLogin()
+		isLoggedIn := relay != nil
+		if onlySetDefaultRelays {
+			relay = nil
+		}
 		if relay == nil {
 			if len(ce.Bridge.Config.Relay.DefaultRelays) == 0 {
 				ce.Reply("You're not logged in and there are no default relay users configured")
@@ -59,18 +64,37 @@ func fnSetRelay(ce *Event) {
 				}
 			}
 			if relay == nil {
-				ce.Reply("You're not logged in and none of the default relay users are in the chat")
+				if isLoggedIn {
+					ce.Reply("You're not allowed to use yourself as relay and none of the default relay users are in the chat")
+				} else {
+					ce.Reply("You're not logged in and none of the default relay users are in the chat")
+				}
 				return
 			}
 		}
 	} else {
-		relay = ce.Bridge.GetCachedUserLoginByID(networkid.UserLoginID(ce.Args[0]))
+		var targetID networkid.UserLoginID
+		if ce.Portal.Receiver != "" {
+			targetID = ce.Portal.Receiver
+			if len(ce.Args) > 0 && ce.Args[0] != string(targetID) {
+				ce.Reply("In split portals, only the receiver (%s) can be set as relay", targetID)
+				return
+			}
+		} else {
+			targetID = networkid.UserLoginID(ce.Args[0])
+		}
+		relay = ce.Bridge.GetCachedUserLoginByID(targetID)
 		if relay == nil {
-			ce.Reply("User login with ID `%s` not found", ce.Args[0])
+			ce.Reply("User login with ID `%s` not found", targetID)
 			return
-		} else if !slices.Contains(ce.Bridge.Config.Relay.DefaultRelays, relay.ID) && relay.UserMXID != ce.User.MXID && !ce.User.Permissions.Admin {
+		} else if slices.Contains(ce.Bridge.Config.Relay.DefaultRelays, relay.ID) {
+			// All good
+		} else if relay.UserMXID != ce.User.MXID && !ce.User.Permissions.Admin {
 			ce.Reply("Only bridge admins can set another user's login as the relay")
 			return
+		} else if onlySetDefaultRelays {
+			ce.Reply("You're not allowed to use yourself as relay")
+			return
 		}
 	}
 	err := ce.Portal.SetRelay(ce.Ctx, relay)
@@ -116,12 +140,10 @@ func fnUnsetRelay(ce *Event) {
 }
 
 func canManageRelay(ce *Event) bool {
-	if ce.Bridge.Config.Relay.AdminOnly {
-		return ce.User.Permissions.Admin
-	}
-	return ce.User.Permissions.Admin ||
-		(ce.Portal.Relay != nil && ce.Portal.Relay.UserMXID == ce.User.MXID) ||
-		hasRelayRoomPermissions(ce)
+	return ce.User.Permissions.ManageRelay &&
+		(ce.User.Permissions.Admin ||
+			(ce.Portal.Relay != nil && ce.Portal.Relay.UserMXID == ce.User.MXID) ||
+			hasRelayRoomPermissions(ce))
 }
 
 func hasRelayRoomPermissions(ce *Event) bool {

bridgev2/commands/startchat.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024 Tulir Asokan
+// Copyright (c) 2025 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -7,15 +7,22 @@
 package commands
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"html"
+	"maps"
+	"slices"
 	"strings"
 	"time"
 
-	"golang.org/x/net/html"
+	"github.com/rs/zerolog"
 
 	"maunium.net/go/mautrix/bridgev2"
-
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/bridgev2/provisionutil"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/format"
 	"maunium.net/go/mautrix/id"
 )
 
@@ -28,22 +35,60 @@ var CommandResolveIdentifier = &FullHandler{
 		Args:        "[_login ID_] <_identifier_>",
 	},
 	RequiresLogin: true,
+	NetworkAPI:    NetworkAPIImplements[bridgev2.IdentifierResolvingNetworkAPI],
+}
+
+var CommandSyncChat = &FullHandler{
+	Func: func(ce *Event) {
+		login, _, err := ce.Portal.FindPreferredLogin(ce.Ctx, ce.User, false)
+		if err != nil {
+			ce.Log.Err(err).Msg("Failed to find login for sync")
+			ce.Reply("Failed to find login: %v", err)
+			return
+		} else if login == nil {
+			ce.Reply("No login found for sync")
+			return
+		}
+		info, err := login.Client.GetChatInfo(ce.Ctx, ce.Portal)
+		if err != nil {
+			ce.Log.Err(err).Msg("Failed to get chat info for sync")
+			ce.Reply("Failed to get chat info: %v", err)
+			return
+		}
+		ce.Portal.UpdateInfo(ce.Ctx, info, login, nil, time.Time{})
+		ce.React("✅️")
+	},
+	Name: "sync-portal",
+	Help: HelpMeta{
+		Section:     HelpSectionChats,
+		Description: "Sync the current portal room",
+	},
+	RequiresPortal: true,
+	RequiresLogin:  true,
 }
 
 var CommandStartChat = &FullHandler{
-	Func: fnResolveIdentifier,
-	Name: "start-chat",
+	Func:    fnResolveIdentifier,
+	Name:    "start-chat",
+	Aliases: []string{"pm"},
 	Help: HelpMeta{
 		Section:     HelpSectionChats,
 		Description: "Start a direct chat with the given user",
 		Args:        "[_login ID_] <_identifier_>",
 	},
 	RequiresLogin: true,
+	NetworkAPI:    NetworkAPIImplements[bridgev2.IdentifierResolvingNetworkAPI],
 }
 
-func getClientForStartingChat[T bridgev2.IdentifierResolvingNetworkAPI](ce *Event, thing string) (*bridgev2.UserLogin, T, []string) {
-	remainingArgs := ce.Args[1:]
-	login := ce.Bridge.GetCachedUserLoginByID(networkid.UserLoginID(ce.Args[0]))
+func getClientForStartingChat[T bridgev2.NetworkAPI](ce *Event, thing string) (*bridgev2.UserLogin, T, []string) {
+	var remainingArgs []string
+	if len(ce.Args) > 1 {
+		remainingArgs = ce.Args[1:]
+	}
+	var login *bridgev2.UserLogin
+	if len(ce.Args) > 0 {
+		login = ce.Bridge.GetCachedUserLoginByID(networkid.UserLoginID(ce.Args[0]))
+	}
 	if login == nil || login.UserMXID != ce.User.MXID {
 		remainingArgs = ce.Args
 		login = ce.User.GetDefaultLogin()
@@ -55,6 +100,16 @@ func getClientForStartingChat[T bridgev2.IdentifierResolvingNetworkAPI](ce *Even
 	return login, api, remainingArgs
 }
 
+func formatResolveIdentifierResult(resp *provisionutil.RespResolveIdentifier) string {
+	if resp.MXID != "" {
+		return fmt.Sprintf("`%s` / [%s](%s)", resp.ID, resp.Name, resp.MXID.URI().MatrixToURL())
+	} else if resp.Name != "" {
+		return fmt.Sprintf("`%s` / %s", resp.ID, resp.Name)
+	} else {
+		return fmt.Sprintf("`%s`", resp.ID)
+	}
+}
+
 func fnResolveIdentifier(ce *Event) {
 	if len(ce.Args) == 0 {
 		ce.Reply("Usage: `$cmdprefix %s <identifier>`", ce.Command)
@@ -64,100 +119,215 @@ func fnResolveIdentifier(ce *Event) {
 	if api == nil {
 		return
 	}
-	createChat := ce.Command == "start-chat"
+	allLogins := ce.User.GetUserLogins()
+	createChat := ce.Command == "start-chat" || ce.Command == "pm"
 	identifier := strings.Join(identifierParts, " ")
-	resp, err := api.ResolveIdentifier(ce.Ctx, identifier, createChat)
+	resp, err := provisionutil.ResolveIdentifier(ce.Ctx, login, identifier, createChat)
+	for i := 0; i < len(allLogins) && errors.Is(err, bridgev2.ErrResolveIdentifierTryNext); i++ {
+		resp, err = provisionutil.ResolveIdentifier(ce.Ctx, allLogins[i], identifier, createChat)
+	}
 	if err != nil {
-		ce.Log.Err(err).Msg("Failed to resolve identifier")
 		ce.Reply("Failed to resolve identifier: %v", err)
 		return
 	} else if resp == nil {
 		ce.ReplyAdvanced(fmt.Sprintf("Identifier <code>%s</code> not found", html.EscapeString(identifier)), false, true)
 		return
 	}
-	var targetName string
-	var targetMXID id.UserID
-	if resp.Ghost != nil {
-		if resp.UserInfo != nil {
-			resp.Ghost.UpdateInfo(ce.Ctx, resp.UserInfo)
-		}
-		targetName = resp.Ghost.Name
-		targetMXID = resp.Ghost.Intent.GetMXID()
-	} else if resp.UserInfo != nil && resp.UserInfo.Name != nil {
-		targetName = *resp.UserInfo.Name
-	}
-	var formattedName string
-	if targetMXID != "" {
-		formattedName = fmt.Sprintf("`%s` / [%s](%s)", resp.UserID, targetName, targetMXID.URI().MatrixToURL())
-	} else if targetName != "" {
-		formattedName = fmt.Sprintf("`%s` / %s", resp.UserID, targetName)
-	} else {
-		formattedName = fmt.Sprintf("`%s`", resp.UserID)
-	}
+	formattedName := formatResolveIdentifierResult(resp)
 	if createChat {
-		if resp.Chat == nil {
-			ce.Reply("Interface error: network connector did not return chat for create chat request")
-			return
+		name := resp.Portal.Name
+		if name == "" {
+			name = resp.Portal.MXID.String()
 		}
-		portal := resp.Chat.Portal
-		if portal == nil {
-			portal, err = ce.Bridge.GetPortalByKey(ce.Ctx, resp.Chat.PortalKey)
-			if err != nil {
-				ce.Log.Err(err).Msg("Failed to get portal")
-				ce.Reply("Failed to get portal: %v", err)
-				return
-			}
-		}
-		if resp.Chat.PortalInfo == nil {
-			resp.Chat.PortalInfo, err = api.GetChatInfo(ce.Ctx, portal)
-			if err != nil {
-				ce.Log.Err(err).Msg("Failed to get portal info")
-				ce.Reply("Failed to get portal info: %v", err)
-				return
-			}
-		}
-		if portal.MXID != "" {
-			name := portal.Name
-			if name == "" {
-				name = portal.MXID.String()
-			}
-			portal.UpdateInfo(ce.Ctx, resp.Chat.PortalInfo, login, nil, time.Time{})
-			ce.Reply("You already have a direct chat with %s at [%s](%s)", formattedName, name, portal.MXID.URI().MatrixToURL())
+		if !resp.JustCreated {
+			ce.Reply("You already have a direct chat with %s at [%s](%s)", formattedName, name, resp.Portal.MXID.URI().MatrixToURL())
 		} else {
-			err = portal.CreateMatrixRoom(ce.Ctx, login, resp.Chat.PortalInfo)
-			if err != nil {
-				ce.Log.Err(err).Msg("Failed to create room")
-				ce.Reply("Failed to create room: %v", err)
-				return
-			}
-			name := portal.Name
-			if name == "" {
-				name = portal.MXID.String()
-			}
-			ce.Reply("Created chat with %s: [%s](%s)", formattedName, name, portal.MXID.URI().MatrixToURL())
+			ce.Reply("Created chat with %s: [%s](%s)", formattedName, name, resp.Portal.MXID.URI().MatrixToURL())
 		}
 	} else {
 		ce.Reply("Found %s", formattedName)
 	}
 }
 
-var CommandDeletePortal = &FullHandler{
-	Func: func(ce *Event) {
-		err := ce.Portal.Delete(ce.Ctx)
-		if err != nil {
-			ce.Reply("Failed to delete portal: %v", err)
-		}
-		err = ce.Bot.DeleteRoom(ce.Ctx, ce.Portal.MXID, false)
-		if err != nil {
-			ce.Reply("Failed to clean up room: %v", err)
-		}
-		ce.MessageStatus.DisableMSS = true
-	},
-	Name: "delete-portal",
+var CommandCreateGroup = &FullHandler{
+	Func:    fnCreateGroup,
+	Name:    "create-group",
+	Aliases: []string{"create"},
 	Help: HelpMeta{
-		Section:     HelpSectionAdmin,
-		Description: "Delete the current portal room",
+		Section:     HelpSectionChats,
+		Description: "Create a new group chat for the current Matrix room",
+		Args:        "[_group type_]",
 	},
-	RequiresAdmin:  true,
-	RequiresPortal: true,
+	RequiresLogin: true,
+	NetworkAPI:    NetworkAPIImplements[bridgev2.GroupCreatingNetworkAPI],
+}
+
+func getState[T any](ctx context.Context, roomID id.RoomID, evtType event.Type, provider bridgev2.MatrixConnectorWithArbitraryRoomState) (content T) {
+	evt, err := provider.GetStateEvent(ctx, roomID, evtType, "")
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Stringer("event_type", evtType).Msg("Failed to get state event for group creation")
+	} else if evt != nil {
+		content, _ = evt.Content.Parsed.(T)
+	}
+	return
+}
+
+func fnCreateGroup(ce *Event) {
+	ce.Bridge.Matrix.GetCapabilities()
+	login, api, remainingArgs := getClientForStartingChat[bridgev2.GroupCreatingNetworkAPI](ce, "creating group")
+	if api == nil {
+		return
+	}
+	stateProvider, ok := ce.Bridge.Matrix.(bridgev2.MatrixConnectorWithArbitraryRoomState)
+	if !ok {
+		ce.Reply("Matrix connector doesn't support fetching room state")
+		return
+	}
+	members, err := ce.Bridge.Matrix.GetMembers(ce.Ctx, ce.RoomID)
+	if err != nil {
+		ce.Log.Err(err).Msg("Failed to get room members for group creation")
+		ce.Reply("Failed to get room members: %v", err)
+		return
+	}
+	caps := ce.Bridge.Network.GetCapabilities()
+	params := &bridgev2.GroupCreateParams{
+		Username:     "",
+		Participants: make([]networkid.UserID, 0, len(members)-2),
+		Parent:       nil, // TODO check space parent event
+		Name:         getState[*event.RoomNameEventContent](ce.Ctx, ce.RoomID, event.StateRoomName, stateProvider),
+		Avatar:       getState[*event.RoomAvatarEventContent](ce.Ctx, ce.RoomID, event.StateRoomAvatar, stateProvider),
+		Topic:        getState[*event.TopicEventContent](ce.Ctx, ce.RoomID, event.StateTopic, stateProvider),
+		Disappear:    getState[*event.BeeperDisappearingTimer](ce.Ctx, ce.RoomID, event.StateBeeperDisappearingTimer, stateProvider),
+		RoomID:       ce.RoomID,
+	}
+	for userID, member := range members {
+		if userID == ce.User.MXID || userID == ce.Bot.GetMXID() || !member.Membership.IsInviteOrJoin() {
+			continue
+		}
+		if parsedUserID, ok := ce.Bridge.Matrix.ParseGhostMXID(userID); ok {
+			params.Participants = append(params.Participants, parsedUserID)
+		} else if !ce.Bridge.Config.SplitPortals {
+			if user, err := ce.Bridge.GetExistingUserByMXID(ce.Ctx, userID); err != nil {
+				ce.Log.Err(err).Stringer("user_id", userID).Msg("Failed to get user for room member")
+			} else if user != nil {
+				// TODO add user logins to participants
+				//for _, login := range user.GetUserLogins() {
+				//	params.Participants = append(params.Participants, login.GetUserID())
+				//}
+			}
+		}
+	}
+
+	if len(caps.Provisioning.GroupCreation) == 0 {
+		ce.Reply("No group creation types defined in network capabilities")
+		return
+	} else if len(remainingArgs) > 0 {
+		params.Type = remainingArgs[0]
+	} else if len(caps.Provisioning.GroupCreation) == 1 {
+		for params.Type = range caps.Provisioning.GroupCreation {
+			// The loop assigns the variable we want
+		}
+	} else {
+		types := strings.Join(slices.Collect(maps.Keys(caps.Provisioning.GroupCreation)), "`, `")
+		ce.Reply("Please specify type of group to create: `%s`", types)
+		return
+	}
+	resp, err := provisionutil.CreateGroup(ce.Ctx, login, params)
+	if err != nil {
+		ce.Reply("Failed to create group: %v", err)
+		return
+	}
+	var postfix string
+	if len(resp.FailedParticipants) > 0 {
+		failedParticipantsStrings := make([]string, len(resp.FailedParticipants))
+		i := 0
+		for participantID, meta := range resp.FailedParticipants {
+			failedParticipantsStrings[i] = fmt.Sprintf("* %s: %s", format.SafeMarkdownCode(participantID), meta.Reason)
+			i++
+		}
+		postfix += "\n\nFailed to add some participants:\n" + strings.Join(failedParticipantsStrings, "\n")
+	}
+	ce.Reply("Successfully created group `%s`%s", resp.ID, postfix)
+}
+
+var CommandSearch = &FullHandler{
+	Func: fnSearch,
+	Name: "search",
+	Help: HelpMeta{
+		Section:     HelpSectionChats,
+		Description: "Search for users on the remote network",
+		Args:        "<_query_>",
+	},
+	RequiresLogin: true,
+	NetworkAPI:    NetworkAPIImplements[bridgev2.UserSearchingNetworkAPI],
+}
+
+func fnSearch(ce *Event) {
+	if len(ce.Args) == 0 {
+		ce.Reply("Usage: `$cmdprefix search <query>`")
+		return
+	}
+	login, api, queryParts := getClientForStartingChat[bridgev2.UserSearchingNetworkAPI](ce, "searching users")
+	if api == nil {
+		return
+	}
+	resp, err := provisionutil.SearchUsers(ce.Ctx, login, strings.Join(queryParts, " "))
+	if err != nil {
+		ce.Reply("Failed to search for users: %v", err)
+		return
+	}
+	resultsString := make([]string, len(resp.Results))
+	for i, res := range resp.Results {
+		formattedName := formatResolveIdentifierResult(res)
+		resultsString[i] = fmt.Sprintf("* %s", formattedName)
+		if res.Portal != nil && res.Portal.MXID != "" {
+			portalName := res.Portal.Name
+			if portalName == "" {
+				portalName = res.Portal.MXID.String()
+			}
+			resultsString[i] = fmt.Sprintf("%s - DM portal: [%s](%s)", resultsString[i], portalName, res.Portal.MXID.URI().MatrixToURL())
+		}
+	}
+	ce.Reply("Search results:\n\n%s", strings.Join(resultsString, "\n"))
+}
+
+var CommandMute = &FullHandler{
+	Func:    fnMute,
+	Name:    "mute",
+	Aliases: []string{"unmute"},
+	Help: HelpMeta{
+		Section:     HelpSectionChats,
+		Description: "Mute or unmute a chat on the remote network",
+		Args:        "[duration]",
+	},
+	RequiresPortal: true,
+	RequiresLogin:  true,
+	NetworkAPI:     NetworkAPIImplements[bridgev2.MuteHandlingNetworkAPI],
+}
+
+func fnMute(ce *Event) {
+	_, api, _ := getClientForStartingChat[bridgev2.MuteHandlingNetworkAPI](ce, "muting chats")
+	var mutedUntil int64
+	if ce.Command == "mute" {
+		mutedUntil = -1
+		if len(ce.Args) > 0 {
+			duration, err := time.ParseDuration(ce.Args[0])
+			if err != nil {
+				ce.Reply("Invalid duration: %v", err)
+				return
+			}
+			mutedUntil = time.Now().Add(duration).UnixMilli()
+		}
+	}
+	err := api.HandleMute(ce.Ctx, &bridgev2.MatrixMute{
+		MatrixEventBase: bridgev2.MatrixEventBase[*event.BeeperMuteEventContent]{
+			Content: &event.BeeperMuteEventContent{MutedUntil: mutedUntil},
+			Portal:  ce.Portal,
+		},
+	})
+	if err != nil {
+		ce.Reply("Failed to %s chat: %v", ce.Command, err)
+	} else {
+		ce.React("✅️")
+	}
 }

bridgev2/commands/sudo.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package commands
+
+import (
+	"strings"
+
+	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+)
+
+var CommandSudo = &FullHandler{
+	Func:    fnSudo,
+	Name:    "sudo",
+	Aliases: []string{"doas", "do-as", "runas", "run-as"},
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Run a command as a different user.",
+		Args:        "[--create] <_user ID_> <_command_> [_args..._]",
+	},
+	RequiresAdmin: true,
+}
+
+func fnSudo(ce *Event) {
+	forceNonexistentUser := len(ce.Args) > 0 && strings.ToLower(ce.Args[0]) == "--create"
+	if forceNonexistentUser {
+		ce.Args = ce.Args[1:]
+	}
+	if len(ce.Args) < 2 {
+		ce.Reply("Usage: `$cmdprefix sudo [--create] <user ID> <command> [args...]`")
+		return
+	}
+	targetUserID := id.UserID(ce.Args[0])
+	if _, _, err := targetUserID.Parse(); err != nil || len(targetUserID) > id.UserIDMaxLength {
+		ce.Reply("Invalid user ID `%s`", targetUserID)
+		return
+	}
+	var targetUser *bridgev2.User
+	var err error
+	if forceNonexistentUser {
+		targetUser, err = ce.Bridge.GetUserByMXID(ce.Ctx, targetUserID)
+	} else {
+		targetUser, err = ce.Bridge.GetExistingUserByMXID(ce.Ctx, targetUserID)
+	}
+	if err != nil {
+		ce.Log.Err(err).Msg("Failed to get user from database")
+		ce.Reply("Failed to get user")
+		return
+	} else if targetUser == nil {
+		ce.Reply("User not found. Use `--create` if you want to run commands as a user who has never used the bridge.")
+		return
+	}
+	ce.User = targetUser
+	origArgs := ce.Args[1:]
+	ce.Command = strings.ToLower(ce.Args[1])
+	ce.Args = ce.Args[2:]
+	ce.RawArgs = strings.Join(ce.Args, " ")
+	ce.Processor.handleCommand(ce.Ctx, ce, strings.Join(origArgs, " "), origArgs)
+}
+
+var CommandDoIn = &FullHandler{
+	Func:    fnDoIn,
+	Name:    "doin",
+	Aliases: []string{"do-in", "runin", "run-in"},
+	Help: HelpMeta{
+		Section:     HelpSectionAdmin,
+		Description: "Run a command in a different room.",
+		Args:        "<_room ID_> <_command_> [_args..._]",
+	},
+}
+
+func fnDoIn(ce *Event) {
+	if len(ce.Args) < 2 {
+		ce.Reply("Usage: `$cmdprefix doin <room ID> <command> [args...]`")
+		return
+	}
+	targetRoomID := id.RoomID(ce.Args[0])
+	if !ce.User.Permissions.Admin {
+		memberInfo, err := ce.Bridge.Matrix.GetMemberInfo(ce.Ctx, targetRoomID, ce.User.MXID)
+		if err != nil {
+			ce.Log.Err(err).Msg("Failed to check if user is in doin target room")
+			ce.Reply("Failed to check if you're in the target room")
+			return
+		} else if memberInfo == nil || memberInfo.Membership != event.MembershipJoin {
+			ce.Reply("You must be in the target room to run commands there")
+			return
+		}
+	}
+	ce.RoomID = targetRoomID
+	var err error
+	ce.Portal, err = ce.Bridge.GetPortalByMXID(ce.Ctx, targetRoomID)
+	if err != nil {
+		ce.Log.Err(err).Msg("Failed to get target portal")
+		ce.Reply("Failed to get portal")
+		return
+	}
+	origArgs := ce.Args[1:]
+	ce.Command = strings.ToLower(ce.Args[1])
+	ce.Args = ce.Args[2:]
+	ce.RawArgs = strings.Join(ce.Args, " ")
+	ce.Processor.handleCommand(ce.Ctx, ce, strings.Join(origArgs, " "), origArgs)
+}

bridgev2/database/backfillqueue.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"go.mau.fi/util/dbutil"
+
+	"maunium.net/go/mautrix/bridgev2/networkid"
+)
+
+type BackfillTaskQuery struct {
+	BridgeID networkid.BridgeID
+	*dbutil.QueryHelper[*BackfillTask]
+}
+
+type BackfillTask struct {
+	BridgeID    networkid.BridgeID
+	PortalKey   networkid.PortalKey
+	UserLoginID networkid.UserLoginID
+
+	BatchCount        int
+	IsDone            bool
+	Cursor            networkid.PaginationCursor
+	OldestMessageID   networkid.MessageID
+	DispatchedAt      time.Time
+	CompletedAt       time.Time
+	NextDispatchMinTS time.Time
+}
+
+var BackfillNextDispatchNever = time.Unix(0, (1<<63)-1)
+
+const (
+	ensureBackfillExistsQuery = `
+		INSERT INTO backfill_task (bridge_id, portal_id, portal_receiver, user_login_id, batch_count, is_done, next_dispatch_min_ts)
+		VALUES ($1, $2, $3, $4, -1, false, $5)
+		ON CONFLICT (bridge_id, portal_id, portal_receiver) DO UPDATE
+			SET user_login_id=CASE
+					WHEN backfill_task.user_login_id=''
+						THEN excluded.user_login_id
+					ELSE backfill_task.user_login_id
+				END,
+			    next_dispatch_min_ts=CASE
+			        WHEN backfill_task.next_dispatch_min_ts=9223372036854775807
+			            THEN excluded.next_dispatch_min_ts
+			        ELSE backfill_task.next_dispatch_min_ts
+				END
+	`
+	upsertBackfillQueueQuery = `
+		INSERT INTO backfill_task (
+			bridge_id, portal_id, portal_receiver, user_login_id, batch_count, is_done, cursor,
+			oldest_message_id, dispatched_at, completed_at, next_dispatch_min_ts
+		) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		ON CONFLICT (bridge_id, portal_id, portal_receiver) DO UPDATE
+			SET user_login_id=excluded.user_login_id,
+				batch_count=excluded.batch_count,
+				is_done=excluded.is_done,
+				cursor=excluded.cursor,
+				oldest_message_id=excluded.oldest_message_id,
+				dispatched_at=excluded.dispatched_at,
+				completed_at=excluded.completed_at,
+				next_dispatch_min_ts=excluded.next_dispatch_min_ts
+	`
+	markBackfillDispatchedQuery = `
+		UPDATE backfill_task SET dispatched_at=$4, completed_at=NULL, next_dispatch_min_ts=$5
+		WHERE bridge_id = $1 AND portal_id = $2 AND portal_receiver = $3
+	`
+	updateBackfillQueueQuery = `
+		UPDATE backfill_task
+		SET user_login_id=$4, batch_count=$5, is_done=$6, cursor=$7, oldest_message_id=$8,
+			dispatched_at=$9, completed_at=$10, next_dispatch_min_ts=$11
+		WHERE bridge_id = $1 AND portal_id = $2 AND portal_receiver = $3
+	`
+	markBackfillTaskNotDoneQuery = `
+		UPDATE backfill_task
+		SET is_done = false
+		WHERE bridge_id = $1 AND portal_id = $2 AND portal_receiver = $3 AND user_login_id = $4
+	`
+	getNextBackfillQuery = `
+		SELECT
+			bridge_id, portal_id, portal_receiver, user_login_id, batch_count, is_done,
+			cursor, oldest_message_id, dispatched_at, completed_at, next_dispatch_min_ts
+		FROM backfill_task
+		WHERE bridge_id = $1 AND next_dispatch_min_ts < $2 AND is_done = false AND user_login_id <> ''
+		ORDER BY next_dispatch_min_ts LIMIT 1
+	`
+	getNextBackfillQueryForPortal = `
+		SELECT
+			bridge_id, portal_id, portal_receiver, user_login_id, batch_count, is_done,
+			cursor, oldest_message_id, dispatched_at, completed_at, next_dispatch_min_ts
+		FROM backfill_task
+		WHERE bridge_id = $1 AND portal_id = $2 AND portal_receiver = $3 AND is_done = false AND user_login_id <> ''
+	`
+	deleteBackfillQueueQuery = `
+		DELETE FROM backfill_task
+		WHERE bridge_id = $1 AND portal_id = $2 AND portal_receiver = $3
+	`
+)
+
+func (btq *BackfillTaskQuery) EnsureExists(ctx context.Context, portal networkid.PortalKey, loginID networkid.UserLoginID) error {
+	return btq.Exec(ctx, ensureBackfillExistsQuery, btq.BridgeID, portal.ID, portal.Receiver, loginID, time.Now().UnixNano())
+}
+
+func (btq *BackfillTaskQuery) Upsert(ctx context.Context, bq *BackfillTask) error {
+	ensureBridgeIDMatches(&bq.BridgeID, btq.BridgeID)
+	return btq.Exec(ctx, upsertBackfillQueueQuery, bq.sqlVariables()...)
+}
+
+const UnfinishedBackfillBackoff = 1 * time.Hour
+
+func (btq *BackfillTaskQuery) MarkDispatched(ctx context.Context, bq *BackfillTask) error {
+	ensureBridgeIDMatches(&bq.BridgeID, btq.BridgeID)
+	bq.DispatchedAt = time.Now()
+	bq.CompletedAt = time.Time{}
+	bq.NextDispatchMinTS = bq.DispatchedAt.Add(UnfinishedBackfillBackoff)
+	return btq.Exec(
+		ctx, markBackfillDispatchedQuery,
+		bq.BridgeID, bq.PortalKey.ID, bq.PortalKey.Receiver,
+		bq.DispatchedAt.UnixNano(), bq.NextDispatchMinTS.UnixNano(),
+	)
+}
+
+func (btq *BackfillTaskQuery) Update(ctx context.Context, bq *BackfillTask) error {
+	ensureBridgeIDMatches(&bq.BridgeID, btq.BridgeID)
+	return btq.Exec(ctx, updateBackfillQueueQuery, bq.sqlVariables()...)
+}
+
+func (btq *BackfillTaskQuery) MarkNotDone(ctx context.Context, portalKey networkid.PortalKey, userLoginID networkid.UserLoginID) error {
+	return btq.Exec(ctx, markBackfillTaskNotDoneQuery, btq.BridgeID, portalKey.ID, portalKey.Receiver, userLoginID)
+}
+
+func (btq *BackfillTaskQuery) GetNext(ctx context.Context) (*BackfillTask, error) {
+	return btq.QueryOne(ctx, getNextBackfillQuery, btq.BridgeID, time.Now().UnixNano())
+}
+
+func (btq *BackfillTaskQuery) GetNextForPortal(ctx context.Context, portalKey networkid.PortalKey) (*BackfillTask, error) {
+	return btq.QueryOne(ctx, getNextBackfillQueryForPortal, btq.BridgeID, portalKey.ID, portalKey.Receiver)
+}
+
+func (btq *BackfillTaskQuery) Delete(ctx context.Context, portalKey networkid.PortalKey) error {
+	return btq.Exec(ctx, deleteBackfillQueueQuery, btq.BridgeID, portalKey.ID, portalKey.Receiver)
+}
+
+func (bt *BackfillTask) Scan(row dbutil.Scannable) (*BackfillTask, error) {
+	var cursor, oldestMessageID sql.NullString
+	var dispatchedAt, completedAt, nextDispatchMinTS sql.NullInt64
+	err := row.Scan(
+		&bt.BridgeID, &bt.PortalKey.ID, &bt.PortalKey.Receiver, &bt.UserLoginID, &bt.BatchCount, &bt.IsDone,
+		&cursor, &oldestMessageID, &dispatchedAt, &completedAt, &nextDispatchMinTS)
+	if err != nil {
+		return nil, err
+	}
+	bt.Cursor = networkid.PaginationCursor(cursor.String)
+	bt.OldestMessageID = networkid.MessageID(oldestMessageID.String)
+	if dispatchedAt.Valid {
+		bt.DispatchedAt = time.Unix(0, dispatchedAt.Int64)
+	}
+	if completedAt.Valid {
+		bt.CompletedAt = time.Unix(0, completedAt.Int64)
+	}
+	if nextDispatchMinTS.Valid {
+		bt.NextDispatchMinTS = time.Unix(0, nextDispatchMinTS.Int64)
+	}
+	return bt, nil
+}
+
+func (bt *BackfillTask) sqlVariables() []any {
+	return []any{
+		bt.BridgeID, bt.PortalKey.ID, bt.PortalKey.Receiver, bt.UserLoginID, bt.BatchCount, bt.IsDone,
+		dbutil.StrPtr(bt.Cursor), dbutil.StrPtr(bt.OldestMessageID),
+		dbutil.ConvertedPtr(bt.DispatchedAt, time.Time.UnixNano),
+		dbutil.ConvertedPtr(bt.CompletedAt, time.Time.UnixNano),
+		bt.NextDispatchMinTS.UnixNano(),
+	}
+}

bridgev2/database/database.go

@@ -7,13 +7,7 @@
 package database
 
 import (
-	"encoding/json"
-	"reflect"
-	"strings"
-
 	"go.mau.fi/util/dbutil"
-	"golang.org/x/exp/constraints"
-	"golang.org/x/exp/maps"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
 
@@ -32,6 +26,9 @@ type Database struct {
 	User                *UserQuery
 	UserLogin           *UserLoginQuery
 	UserPortal          *UserPortalQuery
+	BackfillTask        *BackfillTaskQuery
+	KV                  *KVQuery
+	PublicMedia         *PublicMediaQuery
 }
 
 type MetaMerger interface {
@@ -129,6 +126,22 @@ func New(bridgeID networkid.BridgeID, mt MetaTypes, db *dbutil.Database) *Databa
 				return &UserPortal{}
 			}),
 		},
+		BackfillTask: &BackfillTaskQuery{
+			BridgeID: bridgeID,
+			QueryHelper: dbutil.MakeQueryHelper(db, func(_ *dbutil.QueryHelper[*BackfillTask]) *BackfillTask {
+				return &BackfillTask{}
+			}),
+		},
+		KV: &KVQuery{
+			BridgeID: bridgeID,
+			Database: db,
+		},
+		PublicMedia: &PublicMediaQuery{
+			BridgeID: bridgeID,
+			QueryHelper: dbutil.MakeQueryHelper(db, func(_ *dbutil.QueryHelper[*PublicMedia]) *PublicMedia {
+				return &PublicMedia{}
+			}),
+		},
 	}
 }
 
@@ -139,55 +152,3 @@ func ensureBridgeIDMatches(ptr *networkid.BridgeID, expected networkid.BridgeID)
 		panic("bridge ID mismatch")
 	}
 }
-
-func GetNumberFromMap[T constraints.Integer | constraints.Float](m map[string]any, key string) (T, bool) {
-	if val, found := m[key]; found {
-		floatVal, ok := val.(float64)
-		if ok {
-			return T(floatVal), true
-		}
-		tVal, ok := val.(T)
-		if ok {
-			return tVal, true
-		}
-	}
-	return 0, false
-}
-
-func unmarshalMerge(input []byte, data any, extra *map[string]any) error {
-	err := json.Unmarshal(input, data)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(input, extra)
-	if err != nil {
-		return err
-	}
-	if *extra == nil {
-		*extra = make(map[string]any)
-	}
-	return nil
-}
-
-func marshalMerge(data any, extra map[string]any) ([]byte, error) {
-	if extra == nil {
-		return json.Marshal(data)
-	}
-	merged := make(map[string]any)
-	maps.Copy(merged, extra)
-	dataRef := reflect.ValueOf(data).Elem()
-	dataType := dataRef.Type()
-	for _, field := range reflect.VisibleFields(dataType) {
-		parts := strings.Split(field.Tag.Get("json"), ",")
-		if len(parts) == 0 || len(parts[0]) == 0 || parts[0] == "-" {
-			continue
-		}
-		fieldVal := dataRef.FieldByIndex(field.Index)
-		if fieldVal.IsZero() {
-			delete(merged, parts[0])
-		} else {
-			merged[parts[0]] = fieldVal.Interface()
-		}
-	}
-	return json.Marshal(merged)
-}

bridgev2/database/disappear.go

@@ -12,56 +12,94 @@ import (
 	"time"
 
 	"go.mau.fi/util/dbutil"
+	"go.mau.fi/util/jsontime"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
 
-// DisappearingType represents the type of a disappearing message timer.
-type DisappearingType string
+// Deprecated: use [event.DisappearingType]
+type DisappearingType = event.DisappearingType
 
+// Deprecated: use constants in event package
 const (
-	DisappearingTypeNone      DisappearingType = ""
-	DisappearingTypeAfterRead DisappearingType = "after_read"
-	DisappearingTypeAfterSend DisappearingType = "after_send"
+	DisappearingTypeNone      = event.DisappearingTypeNone
+	DisappearingTypeAfterRead = event.DisappearingTypeAfterRead
+	DisappearingTypeAfterSend = event.DisappearingTypeAfterSend
 )
 
 // DisappearingSetting represents a disappearing message timer setting
 // by combining a type with a timer and an optional start timestamp.
 type DisappearingSetting struct {
-	Type        DisappearingType
+	Type        event.DisappearingType
 	Timer       time.Duration
 	DisappearAt time.Time
 }
 
+func DisappearingSettingFromEvent(evt *event.BeeperDisappearingTimer) DisappearingSetting {
+	if evt == nil || evt.Type == event.DisappearingTypeNone {
+		return DisappearingSetting{}
+	}
+	return DisappearingSetting{
+		Type:  evt.Type,
+		Timer: evt.Timer.Duration,
+	}
+}
+
+func (ds DisappearingSetting) Normalize() DisappearingSetting {
+	if ds.Type == event.DisappearingTypeNone {
+		ds.Timer = 0
+	} else if ds.Timer == 0 {
+		ds.Type = event.DisappearingTypeNone
+	}
+	return ds
+}
+
+func (ds DisappearingSetting) StartingAt(start time.Time) DisappearingSetting {
+	ds.DisappearAt = start.Add(ds.Timer)
+	return ds
+}
+
+func (ds DisappearingSetting) ToEventContent() *event.BeeperDisappearingTimer {
+	if ds.Type == event.DisappearingTypeNone || ds.Timer == 0 {
+		return &event.BeeperDisappearingTimer{}
+	}
+	return &event.BeeperDisappearingTimer{
+		Type:  ds.Type,
+		Timer: jsontime.MS(ds.Timer),
+	}
+}
+
 type DisappearingMessageQuery struct {
 	BridgeID networkid.BridgeID
 	*dbutil.QueryHelper[*DisappearingMessage]
 }
 
 type DisappearingMessage struct {
-	BridgeID networkid.BridgeID
-	RoomID   id.RoomID
-	EventID  id.EventID
+	BridgeID  networkid.BridgeID
+	RoomID    id.RoomID
+	EventID   id.EventID
+	Timestamp time.Time
 	DisappearingSetting
 }
 
 const (
 	upsertDisappearingMessageQuery = `
-		INSERT INTO disappearing_message (bridge_id, mx_room, mxid, type, timer, disappear_at)
-		VALUES ($1, $2, $3, $4, $5, $6)
+		INSERT INTO disappearing_message (bridge_id, mx_room, mxid, timestamp, type, timer, disappear_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7)
 		ON CONFLICT (bridge_id, mxid) DO UPDATE SET timer=excluded.timer, disappear_at=excluded.disappear_at
 	`
 	startDisappearingMessagesQuery = `
 		UPDATE disappearing_message
 		SET disappear_at=$1 + timer
-		WHERE bridge_id=$2 AND mx_room=$3 AND disappear_at IS NULL AND type='after_read'
-		RETURNING bridge_id, mx_room, mxid, type, timer, disappear_at
+		WHERE bridge_id=$2 AND mx_room=$3 AND disappear_at IS NULL AND type='after_read' AND timestamp<=$4
+		RETURNING bridge_id, mx_room, mxid, timestamp, type, timer, disappear_at
 	`
 	getUpcomingDisappearingMessagesQuery = `
-		SELECT bridge_id, mx_room, mxid, type, timer, disappear_at
+		SELECT bridge_id, mx_room, mxid, timestamp, type, timer, disappear_at
 		FROM disappearing_message WHERE bridge_id = $1 AND disappear_at IS NOT NULL AND disappear_at < $2
-		ORDER BY disappear_at
+		ORDER BY disappear_at LIMIT $3
 	`
 	deleteDisappearingMessageQuery = `
 		DELETE FROM disappearing_message WHERE bridge_id=$1 AND mxid=$2
@@ -73,12 +111,12 @@ func (dmq *DisappearingMessageQuery) Put(ctx context.Context, dm *DisappearingMe
 	return dmq.Exec(ctx, upsertDisappearingMessageQuery, dm.sqlVariables()...)
 }
 
-func (dmq *DisappearingMessageQuery) StartAll(ctx context.Context, roomID id.RoomID) ([]*DisappearingMessage, error) {
-	return dmq.QueryMany(ctx, startDisappearingMessagesQuery, time.Now().UnixNano(), dmq.BridgeID, roomID)
+func (dmq *DisappearingMessageQuery) StartAllBefore(ctx context.Context, roomID id.RoomID, beforeTS time.Time) ([]*DisappearingMessage, error) {
+	return dmq.QueryMany(ctx, startDisappearingMessagesQuery, time.Now().UnixNano(), dmq.BridgeID, roomID, beforeTS.UnixNano())
 }
 
-func (dmq *DisappearingMessageQuery) GetUpcoming(ctx context.Context, duration time.Duration) ([]*DisappearingMessage, error) {
-	return dmq.QueryMany(ctx, getUpcomingDisappearingMessagesQuery, dmq.BridgeID, time.Now().Add(duration).UnixNano())
+func (dmq *DisappearingMessageQuery) GetUpcoming(ctx context.Context, duration time.Duration, limit int) ([]*DisappearingMessage, error) {
+	return dmq.QueryMany(ctx, getUpcomingDisappearingMessagesQuery, dmq.BridgeID, time.Now().Add(duration).UnixNano(), limit)
 }
 
 func (dmq *DisappearingMessageQuery) Delete(ctx context.Context, eventID id.EventID) error {
@@ -86,17 +124,19 @@ func (dmq *DisappearingMessageQuery) Delete(ctx context.Context, eventID id.Even
 }
 
 func (d *DisappearingMessage) Scan(row dbutil.Scannable) (*DisappearingMessage, error) {
+	var timestamp int64
 	var disappearAt sql.NullInt64
-	err := row.Scan(&d.BridgeID, &d.RoomID, &d.EventID, &d.Type, &d.Timer, &disappearAt)
+	err := row.Scan(&d.BridgeID, &d.RoomID, &d.EventID, &timestamp, &d.Type, &d.Timer, &disappearAt)
 	if err != nil {
 		return nil, err
 	}
 	if disappearAt.Valid {
 		d.DisappearAt = time.Unix(0, disappearAt.Int64)
 	}
+	d.Timestamp = time.Unix(0, timestamp)
 	return d, nil
 }
 
 func (d *DisappearingMessage) sqlVariables() []any {
-	return []any{d.BridgeID, d.RoomID, d.EventID, d.Type, d.Timer, dbutil.ConvertedPtr(d.DisappearAt, time.Time.UnixNano)}
+	return []any{d.BridgeID, d.RoomID, d.EventID, d.Timestamp.UnixNano(), d.Type, d.Timer, dbutil.ConvertedPtr(d.DisappearAt, time.Time.UnixNano)}
 }

bridgev2/database/ghost.go

@@ -7,12 +7,17 @@
 package database
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
+	"encoding/json"
+	"fmt"
 
 	"go.mau.fi/util/dbutil"
+	"go.mau.fi/util/exerrors"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/crypto/canonicaljson"
 	"maunium.net/go/mautrix/id"
 )
 
@@ -22,6 +27,55 @@ type GhostQuery struct {
 	*dbutil.QueryHelper[*Ghost]
 }
 
+type ExtraProfile map[string]json.RawMessage
+
+func (ep *ExtraProfile) Set(key string, value any) error {
+	if key == "displayname" || key == "avatar_url" {
+		return fmt.Errorf("cannot set reserved profile key %q", key)
+	}
+	marshaled, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	if *ep == nil {
+		*ep = make(ExtraProfile)
+	}
+	(*ep)[key] = canonicaljson.CanonicalJSONAssumeValid(marshaled)
+	return nil
+}
+
+func (ep *ExtraProfile) With(key string, value any) *ExtraProfile {
+	exerrors.PanicIfNotNil(ep.Set(key, value))
+	return ep
+}
+
+func canonicalizeIfObject(data json.RawMessage) json.RawMessage {
+	if len(data) > 0 && (data[0] == '{' || data[0] == '[') {
+		return canonicaljson.CanonicalJSONAssumeValid(data)
+	}
+	return data
+}
+
+func (ep *ExtraProfile) CopyTo(dest *ExtraProfile) (changed bool) {
+	if len(*ep) == 0 {
+		return
+	}
+	if *dest == nil {
+		*dest = make(ExtraProfile)
+	}
+	for key, val := range *ep {
+		if key == "displayname" || key == "avatar_url" {
+			continue
+		}
+		existing, exists := (*dest)[key]
+		if !exists || !bytes.Equal(canonicalizeIfObject(existing), val) {
+			(*dest)[key] = val
+			changed = true
+		}
+	}
+	return
+}
+
 type Ghost struct {
 	BridgeID networkid.BridgeID
 	ID       networkid.UserID
@@ -35,26 +89,29 @@ type Ghost struct {
 	ContactInfoSet bool
 	IsBot          bool
 	Identifiers    []string
+	ExtraProfile   ExtraProfile
 	Metadata       any
 }
 
 const (
 	getGhostBaseQuery = `
 		SELECT bridge_id, id, name, avatar_id, avatar_hash, avatar_mxc,
-		       name_set, avatar_set, contact_info_set, is_bot, identifiers, metadata
+		       name_set, avatar_set, contact_info_set, is_bot, identifiers, extra_profile, metadata
 		FROM ghost
 	`
-	getGhostByIDQuery = getGhostBaseQuery + `WHERE bridge_id=$1 AND id=$2`
-	insertGhostQuery  = `
+	getGhostByIDQuery       = getGhostBaseQuery + `WHERE bridge_id=$1 AND id=$2`
+	getGhostByMetadataQuery = getGhostBaseQuery + `WHERE bridge_id=$1 AND metadata->>$2=$3`
+	insertGhostQuery        = `
 		INSERT INTO ghost (
 			bridge_id, id, name, avatar_id, avatar_hash, avatar_mxc,
-			name_set, avatar_set, contact_info_set, is_bot, identifiers, metadata
+			name_set, avatar_set, contact_info_set, is_bot, identifiers, extra_profile, metadata
 		)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 	`
 	updateGhostQuery = `
 		UPDATE ghost SET name=$3, avatar_id=$4, avatar_hash=$5, avatar_mxc=$6,
-		                 name_set=$7, avatar_set=$8, contact_info_set=$9, is_bot=$10, identifiers=$11, metadata=$12
+		                 name_set=$7, avatar_set=$8, contact_info_set=$9, is_bot=$10,
+		                 identifiers=$11, extra_profile=$12, metadata=$13
 		WHERE bridge_id=$1 AND id=$2
 	`
 )
@@ -63,6 +120,12 @@ func (gq *GhostQuery) GetByID(ctx context.Context, id networkid.UserID) (*Ghost,
 	return gq.QueryOne(ctx, getGhostByIDQuery, gq.BridgeID, id)
 }
 
+// GetByMetadata returns the ghosts whose metadata field at the given JSON key
+// matches the given value.
+func (gq *GhostQuery) GetByMetadata(ctx context.Context, key string, value any) ([]*Ghost, error) {
+	return gq.QueryMany(ctx, getGhostByMetadataQuery, gq.BridgeID, key, value)
+}
+
 func (gq *GhostQuery) Insert(ctx context.Context, ghost *Ghost) error {
 	ensureBridgeIDMatches(&ghost.BridgeID, gq.BridgeID)
 	return gq.Exec(ctx, insertGhostQuery, ghost.ensureHasMetadata(gq.MetaType).sqlVariables()...)
@@ -79,7 +142,7 @@ func (g *Ghost) Scan(row dbutil.Scannable) (*Ghost, error) {
 		&g.BridgeID, &g.ID,
 		&g.Name, &g.AvatarID, &avatarHash, &g.AvatarMXC,
 		&g.NameSet, &g.AvatarSet, &g.ContactInfoSet, &g.IsBot,
-		dbutil.JSON{Data: &g.Identifiers}, dbutil.JSON{Data: g.Metadata},
+		dbutil.JSON{Data: &g.Identifiers}, dbutil.JSON{Data: &g.ExtraProfile}, dbutil.JSON{Data: g.Metadata},
 	)
 	if err != nil {
 		return nil, err
@@ -109,6 +172,6 @@ func (g *Ghost) sqlVariables() []any {
 		g.BridgeID, g.ID,
 		g.Name, g.AvatarID, avatarHash, g.AvatarMXC,
 		g.NameSet, g.AvatarSet, g.ContactInfoSet, g.IsBot,
-		dbutil.JSON{Data: &g.Identifiers}, dbutil.JSON{Data: g.Metadata},
+		dbutil.JSON{Data: &g.Identifiers}, dbutil.JSON{Data: g.ExtraProfile}, dbutil.JSON{Data: g.Metadata},
 	}
 }

bridgev2/database/kvstore.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"github.com/rs/zerolog"
+	"go.mau.fi/util/dbutil"
+
+	"maunium.net/go/mautrix/bridgev2/networkid"
+)
+
+type Key string
+
+const (
+	KeySplitPortalsEnabled     Key = "split_portals_enabled"
+	KeyBridgeInfoVersion       Key = "bridge_info_version"
+	KeyEncryptionStateResynced Key = "encryption_state_resynced"
+	KeyRecoveryKey             Key = "recovery_key"
+)
+
+type KVQuery struct {
+	BridgeID networkid.BridgeID
+	*dbutil.Database
+}
+
+const (
+	getKVQuery = `SELECT value FROM kv_store WHERE bridge_id = $1 AND key = $2`
+	setKVQuery = `
+		INSERT INTO kv_store (bridge_id, key, value) VALUES ($1, $2, $3)
+		ON CONFLICT (bridge_id, key) DO UPDATE SET value = $3
+	`
+)
+
+func (kvq *KVQuery) Get(ctx context.Context, key Key) string {
+	var value string
+	err := kvq.QueryRow(ctx, getKVQuery, kvq.BridgeID, key).Scan(&value)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		zerolog.Ctx(ctx).Err(err).Str("key", string(key)).Msg("Failed to get key from kvstore")
+	}
+	return value
+}
+
+func (kvq *KVQuery) Set(ctx context.Context, key Key, value string) {
+	_, err := kvq.Exec(ctx, setKVQuery, kvq.BridgeID, key, value)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).
+			Str("key", string(key)).
+			Str("value", value).
+			Msg("Failed to set key in kvstore")
+	}
+}

bridgev2/database/message.go

@@ -8,9 +8,15 @@ package database
 
 import (
 	"context"
+	"crypto/sha256"
 	"database/sql"
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"sync"
 	"time"
 
+	"github.com/rs/zerolog"
 	"go.mau.fi/util/dbutil"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
@@ -21,6 +27,7 @@ type MessageQuery struct {
 	BridgeID networkid.BridgeID
 	MetaType MetaTypeCreator
 	*dbutil.QueryHelper[*Message]
+	chunkDeleteLock sync.Mutex
 }
 
 type Message struct {
@@ -30,48 +37,62 @@ type Message struct {
 	PartID   networkid.PartID
 	MXID     id.EventID
 
-	Room       networkid.PortalKey
-	SenderID   networkid.UserID
-	SenderMXID id.UserID
-	Timestamp  time.Time
-	EditCount  int
+	Room             networkid.PortalKey
+	SenderID         networkid.UserID
+	SenderMXID       id.UserID
+	Timestamp        time.Time
+	EditCount        int
+	IsDoublePuppeted bool
 
 	ThreadRoot networkid.MessageID
 	ReplyTo    networkid.MessageOptionalPartID
 
+	SendTxnID networkid.RawTransactionID
+
 	Metadata any
 }
 
 const (
 	getMessageBaseQuery = `
 		SELECT rowid, bridge_id, id, part_id, mxid, room_id, room_receiver, sender_id, sender_mxid,
-		       timestamp, edit_count, thread_root_id, reply_to_id, reply_to_part_id, metadata
+		       timestamp, edit_count, double_puppeted, thread_root_id, reply_to_id, reply_to_part_id,
+			   send_txn_id, metadata
 		FROM message
 	`
 	getAllMessagePartsByIDQuery  = getMessageBaseQuery + `WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND id=$3`
 	getMessagePartByIDQuery      = getMessageBaseQuery + `WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND id=$3 AND part_id=$4`
 	getMessagePartByRowIDQuery   = getMessageBaseQuery + `WHERE bridge_id=$1 AND rowid=$2`
 	getMessageByMXIDQuery        = getMessageBaseQuery + `WHERE bridge_id=$1 AND mxid=$2`
+	getMessageByTxnIDQuery       = getMessageBaseQuery + `WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND (mxid=$3 OR send_txn_id=$4)`
 	getLastMessagePartByIDQuery  = getMessageBaseQuery + `WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND id=$3 ORDER BY part_id DESC LIMIT 1`
 	getFirstMessagePartByIDQuery = getMessageBaseQuery + `WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND id=$3 ORDER BY part_id ASC LIMIT 1`
 	getMessagesBetweenTimeQuery  = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND timestamp>$4 AND timestamp<=$5`
-	getFirstMessageInThread      = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND (id=$4 OR thread_root_id=$4) ORDER BY timestamp ASC, part_id ASC LIMIT 1`
-	getLastMessageInThread       = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND (id=$4 OR thread_root_id=$4) ORDER BY timestamp DESC, part_id DESC LIMIT 1`
+	getOldestMessageInPortal     = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 ORDER BY timestamp ASC, part_id ASC LIMIT 1`
+	getFirstMessageInThread      = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND (id=$4 OR thread_root_id=$4) ORDER BY thread_root_id NULLS FIRST, timestamp ASC, part_id ASC LIMIT 1`
+	getLastMessageInThread       = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND (id=$4 OR thread_root_id=$4) ORDER BY thread_root_id NULLS LAST, timestamp DESC, part_id DESC LIMIT 1`
+	getLastNInPortal             = getMessageBaseQuery + `WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 ORDER BY timestamp DESC, part_id DESC LIMIT $4`
 
-	getLastMessagePartAtOrBeforeTimeQuery = getMessageBaseQuery + `WHERE bridge_id = $1 AND room_id=$2 AND room_receiver=$3 AND timestamp<=$4 ORDER BY timestamp DESC, part_id DESC LIMIT 1`
+	getLastMessagePartAtOrBeforeTimeQuery        = getMessageBaseQuery + `WHERE bridge_id = $1 AND room_id=$2 AND room_receiver=$3 AND timestamp<=$4 ORDER BY timestamp DESC, part_id DESC LIMIT 1`
+	getLastNonFakeMessagePartAtOrBeforeTimeQuery = getMessageBaseQuery + `WHERE bridge_id = $1 AND room_id=$2 AND room_receiver=$3 AND timestamp<=$4 AND mxid NOT LIKE '~fake:%' ORDER BY timestamp DESC, part_id DESC LIMIT 1`
+
+	countMessagesInPortalQuery = `
+		SELECT COUNT(*) FROM message WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3
+	`
 
 	insertMessageQuery = `
 		INSERT INTO message (
 			bridge_id, id, part_id, mxid, room_id, room_receiver, sender_id, sender_mxid,
-			timestamp, edit_count, thread_root_id, reply_to_id, reply_to_part_id, metadata
+			timestamp, edit_count, double_puppeted, thread_root_id, reply_to_id, reply_to_part_id,
+			send_txn_id, metadata
 		)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
 		RETURNING rowid
 	`
 	updateMessageQuery = `
 		UPDATE message SET id=$2, part_id=$3, mxid=$4, room_id=$5, room_receiver=$6, sender_id=$7, sender_mxid=$8,
-		                   timestamp=$9, edit_count=$10, thread_root_id=$11, reply_to_id=$12, reply_to_part_id=$13, metadata=$14
-		WHERE bridge_id=$1 AND rowid=$15
+		                   timestamp=$9, edit_count=$10, double_puppeted=$11, thread_root_id=$12, reply_to_id=$13,
+                           reply_to_part_id=$14, send_txn_id=$15, metadata=$16
+		WHERE bridge_id=$1 AND rowid=$17
 	`
 	deleteAllMessagePartsByIDQuery = `
 		DELETE FROM message WHERE bridge_id=$1 AND (room_receiver=$2 OR room_receiver='') AND id=$3
@@ -79,6 +100,10 @@ const (
 	deleteMessagePartByRowIDQuery = `
 		DELETE FROM message WHERE bridge_id=$1 AND rowid=$2
 	`
+	deleteMessageChunkQuery = `
+		DELETE FROM message WHERE bridge_id=$1 AND room_id=$2 AND room_receiver=$3 AND rowid > $4 AND rowid <= $5
+	`
+	getMaxMessageRowIDQuery = `SELECT MAX(rowid) FROM message WHERE bridge_id=$1`
 )
 
 func (mq *MessageQuery) GetAllPartsByID(ctx context.Context, receiver networkid.UserLoginID, id networkid.MessageID) ([]*Message, error) {
@@ -93,6 +118,10 @@ func (mq *MessageQuery) GetPartByMXID(ctx context.Context, mxid id.EventID) (*Me
 	return mq.QueryOne(ctx, getMessageByMXIDQuery, mq.BridgeID, mxid)
 }
 
+func (mq *MessageQuery) GetPartByTxnID(ctx context.Context, receiver networkid.UserLoginID, mxid id.EventID, txnID networkid.RawTransactionID) (*Message, error) {
+	return mq.QueryOne(ctx, getMessageByTxnIDQuery, mq.BridgeID, receiver, mxid, txnID)
+}
+
 func (mq *MessageQuery) GetLastPartByID(ctx context.Context, receiver networkid.UserLoginID, id networkid.MessageID) (*Message, error) {
 	return mq.QueryOne(ctx, getLastMessagePartByIDQuery, mq.BridgeID, receiver, id)
 }
@@ -117,10 +146,18 @@ func (mq *MessageQuery) GetLastPartAtOrBeforeTime(ctx context.Context, portal ne
 	return mq.QueryOne(ctx, getLastMessagePartAtOrBeforeTimeQuery, mq.BridgeID, portal.ID, portal.Receiver, maxTS.UnixNano())
 }
 
+func (mq *MessageQuery) GetLastNonFakePartAtOrBeforeTime(ctx context.Context, portal networkid.PortalKey, maxTS time.Time) (*Message, error) {
+	return mq.QueryOne(ctx, getLastNonFakeMessagePartAtOrBeforeTimeQuery, mq.BridgeID, portal.ID, portal.Receiver, maxTS.UnixNano())
+}
+
 func (mq *MessageQuery) GetMessagesBetweenTimeQuery(ctx context.Context, portal networkid.PortalKey, start, end time.Time) ([]*Message, error) {
 	return mq.QueryMany(ctx, getMessagesBetweenTimeQuery, mq.BridgeID, portal.ID, portal.Receiver, start.UnixNano(), end.UnixNano())
 }
 
+func (mq *MessageQuery) GetFirstPortalMessage(ctx context.Context, portal networkid.PortalKey) (*Message, error) {
+	return mq.QueryOne(ctx, getOldestMessageInPortal, mq.BridgeID, portal.ID, portal.Receiver)
+}
+
 func (mq *MessageQuery) GetFirstThreadMessage(ctx context.Context, portal networkid.PortalKey, threadRoot networkid.MessageID) (*Message, error) {
 	return mq.QueryOne(ctx, getFirstMessageInThread, mq.BridgeID, portal.ID, portal.Receiver, threadRoot)
 }
@@ -129,6 +166,10 @@ func (mq *MessageQuery) GetLastThreadMessage(ctx context.Context, portal network
 	return mq.QueryOne(ctx, getLastMessageInThread, mq.BridgeID, portal.ID, portal.Receiver, threadRoot)
 }
 
+func (mq *MessageQuery) GetLastNInPortal(ctx context.Context, portal networkid.PortalKey, n int) ([]*Message, error) {
+	return mq.QueryMany(ctx, getLastNInPortal, mq.BridgeID, portal.ID, portal.Receiver, n)
+}
+
 func (mq *MessageQuery) Insert(ctx context.Context, msg *Message) error {
 	ensureBridgeIDMatches(&msg.BridgeID, mq.BridgeID)
 	return mq.GetDB().QueryRow(ctx, insertMessageQuery, msg.ensureHasMetadata(mq.MetaType).sqlVariables()...).Scan(&msg.RowID)
@@ -147,24 +188,114 @@ func (mq *MessageQuery) Delete(ctx context.Context, rowID int64) error {
 	return mq.Exec(ctx, deleteMessagePartByRowIDQuery, mq.BridgeID, rowID)
 }
 
+func (mq *MessageQuery) deleteChunk(ctx context.Context, portal networkid.PortalKey, minRowID, maxRowID int64) (int64, error) {
+	res, err := mq.GetDB().Exec(ctx, deleteMessageChunkQuery, mq.BridgeID, portal.ID, portal.Receiver, minRowID, maxRowID)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+func (mq *MessageQuery) getMaxRowID(ctx context.Context) (maxRowID int64, err error) {
+	err = mq.GetDB().QueryRow(ctx, getMaxMessageRowIDQuery, mq.BridgeID).Scan(&maxRowID)
+	return
+}
+
+const deleteChunkSize = 100_000
+
+func (mq *MessageQuery) DeleteInChunks(ctx context.Context, portal networkid.PortalKey) error {
+	if mq.GetDB().Dialect != dbutil.SQLite {
+		return nil
+	}
+	log := zerolog.Ctx(ctx).With().
+		Str("action", "delete messages in chunks").
+		Stringer("portal_key", portal).
+		Logger()
+	if !mq.chunkDeleteLock.TryLock() {
+		log.Warn().Msg("Portal deletion lock is being held, waiting...")
+		mq.chunkDeleteLock.Lock()
+		log.Debug().Msg("Acquired portal deletion lock after waiting")
+	}
+	defer mq.chunkDeleteLock.Unlock()
+	total, err := mq.CountMessagesInPortal(ctx, portal)
+	if err != nil {
+		return fmt.Errorf("failed to count messages in portal: %w", err)
+	} else if total < deleteChunkSize/3 {
+		return nil
+	}
+	globalMaxRowID, err := mq.getMaxRowID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get max row ID: %w", err)
+	}
+	log.Debug().
+		Int("total_count", total).
+		Int64("global_max_row_id", globalMaxRowID).
+		Msg("Portal has lots of messages, deleting in chunks to avoid database locks")
+	maxRowID := int64(deleteChunkSize)
+	globalMaxRowID += deleteChunkSize * 1.2
+	var dbTimeUsed time.Duration
+	globalStart := time.Now()
+	for total > 500 && maxRowID < globalMaxRowID {
+		start := time.Now()
+		count, err := mq.deleteChunk(ctx, portal, maxRowID-deleteChunkSize, maxRowID)
+		duration := time.Since(start)
+		dbTimeUsed += duration
+		if err != nil {
+			return fmt.Errorf("failed to delete chunk of messages before %d: %w", maxRowID, err)
+		}
+		total -= int(count)
+		maxRowID += deleteChunkSize
+		sleepTime := max(10*time.Millisecond, min(250*time.Millisecond, time.Duration(count/100)*time.Millisecond))
+		log.Debug().
+			Int64("max_row_id", maxRowID).
+			Int64("deleted_count", count).
+			Int("remaining_count", total).
+			Dur("duration", duration).
+			Dur("sleep_time", sleepTime).
+			Msg("Deleted chunk of messages")
+		select {
+		case <-time.After(sleepTime):
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+	log.Debug().
+		Int("remaining_count", total).
+		Dur("db_time_used", dbTimeUsed).
+		Dur("total_duration", time.Since(globalStart)).
+		Msg("Finished chunked delete of messages in portal")
+	return nil
+}
+
+func (mq *MessageQuery) CountMessagesInPortal(ctx context.Context, key networkid.PortalKey) (count int, err error) {
+	err = mq.GetDB().QueryRow(ctx, countMessagesInPortalQuery, mq.BridgeID, key.ID, key.Receiver).Scan(&count)
+	return
+}
+
 func (m *Message) Scan(row dbutil.Scannable) (*Message, error) {
 	var timestamp int64
-	var threadRootID, replyToID, replyToPartID sql.NullString
+	var threadRootID, replyToID, replyToPartID, sendTxnID sql.NullString
+	var doublePuppeted sql.NullBool
 	err := row.Scan(
 		&m.RowID, &m.BridgeID, &m.ID, &m.PartID, &m.MXID, &m.Room.ID, &m.Room.Receiver, &m.SenderID, &m.SenderMXID,
-		&m.EditCount, &timestamp, &threadRootID, &replyToID, &replyToPartID, dbutil.JSON{Data: m.Metadata},
+		&timestamp, &m.EditCount, &doublePuppeted, &threadRootID, &replyToID, &replyToPartID, &sendTxnID,
+		dbutil.JSON{Data: m.Metadata},
 	)
 	if err != nil {
 		return nil, err
 	}
 	m.Timestamp = time.Unix(0, timestamp)
 	m.ThreadRoot = networkid.MessageID(threadRootID.String)
+	m.IsDoublePuppeted = doublePuppeted.Valid
 	if replyToID.Valid {
 		m.ReplyTo.MessageID = networkid.MessageID(replyToID.String)
 		if replyToPartID.Valid {
 			m.ReplyTo.PartID = (*networkid.PartID)(&replyToPartID.String)
 		}
 	}
+	if sendTxnID.Valid {
+		m.SendTxnID = networkid.RawTransactionID(sendTxnID.String)
+	}
 	return m, nil
 }
 
@@ -178,7 +309,8 @@ func (m *Message) ensureHasMetadata(metaType MetaTypeCreator) *Message {
 func (m *Message) sqlVariables() []any {
 	return []any{
 		m.BridgeID, m.ID, m.PartID, m.MXID, m.Room.ID, m.Room.Receiver, m.SenderID, m.SenderMXID,
-		m.Timestamp.UnixNano(), m.EditCount, dbutil.StrPtr(m.ThreadRoot), dbutil.StrPtr(m.ReplyTo.MessageID), m.ReplyTo.PartID,
+		m.Timestamp.UnixNano(), m.EditCount, m.IsDoublePuppeted, dbutil.StrPtr(m.ThreadRoot),
+		dbutil.StrPtr(m.ReplyTo.MessageID), m.ReplyTo.PartID, dbutil.StrPtr(m.SendTxnID),
 		dbutil.JSON{Data: m.Metadata},
 	}
 }
@@ -186,3 +318,17 @@ func (m *Message) sqlVariables() []any {
 func (m *Message) updateSQLVariables() []any {
 	return append(m.sqlVariables(), m.RowID)
 }
+
+const FakeMXIDPrefix = "~fake:"
+const TxnMXIDPrefix = "~txn:"
+const NetworkTxnMXIDPrefix = TxnMXIDPrefix + "network:"
+const RandomTxnMXIDPrefix = TxnMXIDPrefix + "random:"
+
+func (m *Message) SetFakeMXID() {
+	hash := sha256.Sum256([]byte(m.ID))
+	m.MXID = id.EventID(FakeMXIDPrefix + base64.RawURLEncoding.EncodeToString(hash[:]))
+}
+
+func (m *Message) HasFakeMXID() bool {
+	return strings.HasPrefix(m.MXID.String(), FakeMXIDPrefix)
+}

bridgev2/database/portal.go

@@ -16,6 +16,7 @@ import (
 	"go.mau.fi/util/dbutil"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
 
@@ -34,33 +35,53 @@ type PortalQuery struct {
 	*dbutil.QueryHelper[*Portal]
 }
 
+type CapStateFlags uint32
+
+func (csf CapStateFlags) Has(flag CapStateFlags) bool {
+	return csf&flag != 0
+}
+
+const (
+	CapStateFlagDisappearingTimerSet CapStateFlags = 1 << iota
+)
+
+type CapabilityState struct {
+	Source networkid.UserLoginID `json:"source"`
+	ID     string                `json:"id"`
+	Flags  CapStateFlags         `json:"flags"`
+}
+
 type Portal struct {
 	BridgeID networkid.BridgeID
 	networkid.PortalKey
 	MXID id.RoomID
 
-	ParentID     networkid.PortalID
-	RelayLoginID networkid.UserLoginID
-	Name         string
-	Topic        string
-	AvatarID     networkid.AvatarID
-	AvatarHash   [32]byte
-	AvatarMXC    id.ContentURIString
-	NameSet      bool
-	TopicSet     bool
-	AvatarSet    bool
-	InSpace      bool
-	RoomType     RoomType
-	Disappear    DisappearingSetting
-	Metadata     any
+	ParentKey      networkid.PortalKey
+	RelayLoginID   networkid.UserLoginID
+	OtherUserID    networkid.UserID
+	Name           string
+	Topic          string
+	AvatarID       networkid.AvatarID
+	AvatarHash     [32]byte
+	AvatarMXC      id.ContentURIString
+	NameSet        bool
+	TopicSet       bool
+	AvatarSet      bool
+	NameIsCustom   bool
+	InSpace        bool
+	MessageRequest bool
+	RoomType       RoomType
+	Disappear      DisappearingSetting
+	CapState       CapabilityState
+	Metadata       any
 }
 
 const (
 	getPortalBaseQuery = `
-		SELECT bridge_id, id, receiver, mxid, parent_id, relay_login_id,
+		SELECT bridge_id, id, receiver, mxid, parent_id, parent_receiver, relay_login_id, other_user_id,
 		       name, topic, avatar_id, avatar_hash, avatar_mxc,
-		       name_set, topic_set, avatar_set, in_space,
-		       room_type, disappear_type, disappear_timer,
+		       name_set, topic_set, avatar_set, name_is_custom, in_space, message_request,
+		       room_type, disappear_type, disappear_timer, cap_state,
 		       metadata
 		FROM portal
 	`
@@ -68,36 +89,71 @@ const (
 	getPortalByIDWithUncertainReceiverQuery = getPortalBaseQuery + `WHERE bridge_id=$1 AND id=$2 AND (receiver=$3 OR receiver='')`
 	getPortalByMXIDQuery                    = getPortalBaseQuery + `WHERE bridge_id=$1 AND mxid=$2`
 	getAllPortalsWithMXIDQuery              = getPortalBaseQuery + `WHERE bridge_id=$1 AND mxid IS NOT NULL`
-	getChildPortalsQuery                    = getPortalBaseQuery + `WHERE bridge_id=$1 AND parent_id=$2`
+	getAllPortalsWithoutReceiver            = getPortalBaseQuery + `WHERE bridge_id=$1 AND (receiver='' OR (parent_id<>'' AND parent_receiver='')) ORDER BY parent_id DESC`
+	getAllDMPortalsQuery                    = getPortalBaseQuery + `WHERE bridge_id=$1 AND room_type='dm' AND other_user_id=$2`
+	getDMPortalQuery                        = getPortalBaseQuery + `WHERE bridge_id=$1 AND room_type='dm' AND receiver=$2 AND other_user_id=$3`
+	getAllPortalsQuery                      = getPortalBaseQuery + `WHERE bridge_id=$1`
+	getChildPortalsQuery                    = getPortalBaseQuery + `WHERE bridge_id=$1 AND parent_id=$2 AND parent_receiver=$3`
 
 	findPortalReceiverQuery = `SELECT id, receiver FROM portal WHERE bridge_id=$1 AND id=$2 AND (receiver=$3 OR receiver='') LIMIT 1`
 
 	insertPortalQuery = `
 		INSERT INTO portal (
 			bridge_id, id, receiver, mxid,
-			parent_id, relay_login_id,
+			parent_id, parent_receiver, relay_login_id, other_user_id,
 			name, topic, avatar_id, avatar_hash, avatar_mxc,
-			name_set, avatar_set, topic_set, in_space,
-			room_type, disappear_type, disappear_timer,
+			name_set, avatar_set, topic_set, name_is_custom, in_space, message_request,
+			room_type, disappear_type, disappear_timer, cap_state,
 			metadata, relay_bridge_id
 		) VALUES (
-			$1, $2, $3, $4, $5, cast($6 AS TEXT), $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19,
-			CASE WHEN cast($6 AS TEXT) IS NULL THEN NULL ELSE $1 END
+			$1, $2, $3, $4, $5, $6, cast($7 AS TEXT), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24,
+			CASE WHEN cast($7 AS TEXT) IS NULL THEN NULL ELSE $1 END
 		)
 	`
 	updatePortalQuery = `
 		UPDATE portal
-		SET mxid=$4, parent_id=$5, relay_login_id=cast($6 AS TEXT), relay_bridge_id=CASE WHEN cast($6 AS TEXT) IS NULL THEN NULL ELSE bridge_id END,
-		    name=$7, topic=$8, avatar_id=$9, avatar_hash=$10, avatar_mxc=$11,
-		    name_set=$12, avatar_set=$13, topic_set=$14, in_space=$15,
-		    room_type=$16, disappear_type=$17, disappear_timer=$18, metadata=$19
+		SET mxid=$4, parent_id=$5, parent_receiver=$6,
+		    relay_login_id=cast($7 AS TEXT), relay_bridge_id=CASE WHEN cast($7 AS TEXT) IS NULL THEN NULL ELSE bridge_id END,
+		    other_user_id=$8, name=$9, topic=$10, avatar_id=$11, avatar_hash=$12, avatar_mxc=$13,
+		    name_set=$14, avatar_set=$15, topic_set=$16, name_is_custom=$17, in_space=$18, message_request=$19,
+		    room_type=$20, disappear_type=$21, disappear_timer=$22, cap_state=$23, metadata=$24
 		WHERE bridge_id=$1 AND id=$2 AND receiver=$3
 	`
 	deletePortalQuery = `
 		DELETE FROM portal
 		WHERE bridge_id=$1 AND id=$2 AND receiver=$3
 	`
-	reIDPortalQuery = `UPDATE portal SET id=$4, receiver=$5 WHERE bridge_id=$1 AND id=$2 AND receiver=$3`
+	reIDPortalQuery            = `UPDATE portal SET id=$4, receiver=$5 WHERE bridge_id=$1 AND id=$2 AND receiver=$3`
+	migrateToSplitPortalsQuery = `
+		UPDATE portal
+		SET receiver=new_receiver
+		FROM (
+			SELECT bridge_id, id, COALESCE((
+				SELECT login_id
+				FROM user_portal
+				WHERE bridge_id=portal.bridge_id AND portal_id=portal.id AND portal_receiver=''
+				LIMIT 1
+			), (
+				SELECT login_id
+				FROM user_portal
+				WHERE portal.parent_id<>'' AND bridge_id=portal.bridge_id AND portal_id=portal.parent_id
+				LIMIT 1
+			), (
+				SELECT id FROM user_login WHERE bridge_id=portal.bridge_id LIMIT 1
+			), '') AS new_receiver
+			FROM portal
+			WHERE receiver='' AND bridge_id=$1
+		) updates
+		WHERE portal.bridge_id=updates.bridge_id AND portal.id=updates.id AND portal.receiver='' AND NOT EXISTS (
+			SELECT 1 FROM portal p2 WHERE p2.bridge_id=updates.bridge_id AND p2.id=updates.id AND p2.receiver=updates.new_receiver
+		)
+	`
+	fixParentsAfterSplitPortalMigrationQuery = `
+		UPDATE portal
+		SET parent_receiver=receiver
+		WHERE bridge_id=$1 AND parent_receiver='' AND receiver<>'' AND parent_id<>''
+		  AND EXISTS(SELECT 1 FROM portal pp WHERE pp.bridge_id=$1 AND pp.id=portal.parent_id AND pp.receiver=portal.receiver);
+	`
 )
 
 func (pq *PortalQuery) GetByKey(ctx context.Context, key networkid.PortalKey) (*Portal, error) {
@@ -124,8 +180,24 @@ func (pq *PortalQuery) GetAllWithMXID(ctx context.Context) ([]*Portal, error) {
 	return pq.QueryMany(ctx, getAllPortalsWithMXIDQuery, pq.BridgeID)
 }
 
-func (pq *PortalQuery) GetChildren(ctx context.Context, parentID networkid.PortalID) ([]*Portal, error) {
-	return pq.QueryMany(ctx, getChildPortalsQuery, pq.BridgeID, parentID)
+func (pq *PortalQuery) GetAllWithoutReceiver(ctx context.Context) ([]*Portal, error) {
+	return pq.QueryMany(ctx, getAllPortalsWithoutReceiver, pq.BridgeID)
+}
+
+func (pq *PortalQuery) GetAll(ctx context.Context) ([]*Portal, error) {
+	return pq.QueryMany(ctx, getAllPortalsQuery, pq.BridgeID)
+}
+
+func (pq *PortalQuery) GetAllDMsWith(ctx context.Context, otherUserID networkid.UserID) ([]*Portal, error) {
+	return pq.QueryMany(ctx, getAllDMPortalsQuery, pq.BridgeID, otherUserID)
+}
+
+func (pq *PortalQuery) GetDM(ctx context.Context, receiver networkid.UserLoginID, otherUserID networkid.UserID) (*Portal, error) {
+	return pq.QueryOne(ctx, getDMPortalQuery, pq.BridgeID, receiver, otherUserID)
+}
+
+func (pq *PortalQuery) GetChildren(ctx context.Context, parentKey networkid.PortalKey) ([]*Portal, error) {
+	return pq.QueryMany(ctx, getChildPortalsQuery, pq.BridgeID, parentKey.ID, parentKey.Receiver)
 }
 
 func (pq *PortalQuery) ReID(ctx context.Context, oldID, newID networkid.PortalKey) error {
@@ -146,16 +218,33 @@ func (pq *PortalQuery) Delete(ctx context.Context, key networkid.PortalKey) erro
 	return pq.Exec(ctx, deletePortalQuery, pq.BridgeID, key.ID, key.Receiver)
 }
 
+func (pq *PortalQuery) MigrateToSplitPortals(ctx context.Context) (int64, error) {
+	res, err := pq.GetDB().Exec(ctx, migrateToSplitPortalsQuery, pq.BridgeID)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+func (pq *PortalQuery) FixParentsAfterSplitPortalMigration(ctx context.Context) (int64, error) {
+	res, err := pq.GetDB().Exec(ctx, fixParentsAfterSplitPortalMigrationQuery, pq.BridgeID)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
 func (p *Portal) Scan(row dbutil.Scannable) (*Portal, error) {
-	var mxid, parentID, relayLoginID, disappearType sql.NullString
+	var mxid, parentID, parentReceiver, relayLoginID, otherUserID, disappearType sql.NullString
 	var disappearTimer sql.NullInt64
 	var avatarHash string
 	err := row.Scan(
 		&p.BridgeID, &p.ID, &p.Receiver, &mxid,
-		&parentID, &relayLoginID, &p.Name, &p.Topic, &p.AvatarID, &avatarHash, &p.AvatarMXC,
-		&p.NameSet, &p.TopicSet, &p.AvatarSet, &p.InSpace,
+		&parentID, &parentReceiver, &relayLoginID, &otherUserID,
+		&p.Name, &p.Topic, &p.AvatarID, &avatarHash, &p.AvatarMXC,
+		&p.NameSet, &p.TopicSet, &p.AvatarSet, &p.NameIsCustom, &p.InSpace, &p.MessageRequest,
 		&p.RoomType, &disappearType, &disappearTimer,
-		dbutil.JSON{Data: p.Metadata},
+		dbutil.JSON{Data: &p.CapState}, dbutil.JSON{Data: p.Metadata},
 	)
 	if err != nil {
 		return nil, err
@@ -168,12 +257,18 @@ func (p *Portal) Scan(row dbutil.Scannable) (*Portal, error) {
 	}
 	if disappearType.Valid {
 		p.Disappear = DisappearingSetting{
-			Type:  DisappearingType(disappearType.String),
+			Type:  event.DisappearingType(disappearType.String),
 			Timer: time.Duration(disappearTimer.Int64),
 		}
 	}
 	p.MXID = id.RoomID(mxid.String)
-	p.ParentID = networkid.PortalID(parentID.String)
+	p.OtherUserID = networkid.UserID(otherUserID.String)
+	if parentID.Valid {
+		p.ParentKey = networkid.PortalKey{
+			ID:       networkid.PortalID(parentID.String),
+			Receiver: networkid.UserLoginID(parentReceiver.String),
+		}
+	}
 	p.RelayLoginID = networkid.UserLoginID(relayLoginID.String)
 	return p, nil
 }
@@ -192,10 +287,10 @@ func (p *Portal) sqlVariables() []any {
 	}
 	return []any{
 		p.BridgeID, p.ID, p.Receiver, dbutil.StrPtr(p.MXID),
-		dbutil.StrPtr(p.ParentID), dbutil.StrPtr(p.RelayLoginID),
+		dbutil.StrPtr(p.ParentKey.ID), p.ParentKey.Receiver, dbutil.StrPtr(p.RelayLoginID), dbutil.StrPtr(p.OtherUserID),
 		p.Name, p.Topic, p.AvatarID, avatarHash, p.AvatarMXC,
-		p.NameSet, p.TopicSet, p.AvatarSet, p.InSpace,
+		p.NameSet, p.TopicSet, p.AvatarSet, p.NameIsCustom, p.InSpace, p.MessageRequest,
 		p.RoomType, dbutil.StrPtr(p.Disappear.Type), dbutil.NumPtr(p.Disappear.Timer),
-		dbutil.JSON{Data: p.Metadata},
+		dbutil.JSON{Data: p.CapState}, dbutil.JSON{Data: p.Metadata},
 	}
 }

bridgev2/database/publicmedia.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2025 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"go.mau.fi/util/dbutil"
+
+	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/crypto/attachment"
+	"maunium.net/go/mautrix/id"
+)
+
+type PublicMediaQuery struct {
+	BridgeID networkid.BridgeID
+	*dbutil.QueryHelper[*PublicMedia]
+}
+
+type PublicMedia struct {
+	BridgeID networkid.BridgeID
+	PublicID string
+	MXC      id.ContentURI
+	Keys     *attachment.EncryptedFile
+	MimeType string
+	Expiry   time.Time
+}
+
+const (
+	upsertPublicMediaQuery = `
+		INSERT INTO public_media (bridge_id, public_id, mxc, keys, mimetype, expiry)
+		VALUES ($1, $2, $3, $4, $5, $6)
+		ON CONFLICT (bridge_id, public_id) DO UPDATE SET expiry=EXCLUDED.expiry
+	`
+	getPublicMediaQuery = `
+		SELECT bridge_id, public_id, mxc, keys, mimetype, expiry
+		FROM public_media WHERE bridge_id=$1 AND public_id=$2
+	`
+)
+
+func (pmq *PublicMediaQuery) Put(ctx context.Context, pm *PublicMedia) error {
+	ensureBridgeIDMatches(&pm.BridgeID, pmq.BridgeID)
+	return pmq.Exec(ctx, upsertPublicMediaQuery, pm.sqlVariables()...)
+}
+
+func (pmq *PublicMediaQuery) Get(ctx context.Context, publicID string) (*PublicMedia, error) {
+	return pmq.QueryOne(ctx, getPublicMediaQuery, pmq.BridgeID, publicID)
+}
+
+func (pm *PublicMedia) Scan(row dbutil.Scannable) (*PublicMedia, error) {
+	var expiry sql.NullInt64
+	var mimetype sql.NullString
+	err := row.Scan(&pm.BridgeID, &pm.PublicID, &pm.MXC, dbutil.JSON{Data: &pm.Keys}, &mimetype, &expiry)
+	if err != nil {
+		return nil, err
+	}
+	if expiry.Valid {
+		pm.Expiry = time.Unix(0, expiry.Int64)
+	}
+	pm.MimeType = mimetype.String
+	return pm, nil
+}
+
+func (pm *PublicMedia) sqlVariables() []any {
+	return []any{pm.BridgeID, pm.PublicID, &pm.MXC, dbutil.JSONPtr(pm.Keys), dbutil.StrPtr(pm.MimeType), dbutil.ConvertedPtr(pm.Expiry, time.Time.UnixNano)}
+}

bridgev2/database/reaction.go

@@ -28,6 +28,7 @@ type Reaction struct {
 	MessageID     networkid.MessageID
 	MessagePartID networkid.PartID
 	SenderID      networkid.UserID
+	SenderMXID    id.UserID
 	EmojiID       networkid.EmojiID
 	MXID          id.EventID
 
@@ -38,38 +39,43 @@ type Reaction struct {
 
 const (
 	getReactionBaseQuery = `
-		SELECT bridge_id, message_id, message_part_id, sender_id, emoji_id, emoji, room_id, room_receiver, mxid, timestamp, metadata FROM reaction
+		SELECT bridge_id, message_id, message_part_id, sender_id, sender_mxid, emoji_id, emoji, room_id, room_receiver, mxid, timestamp, metadata FROM reaction
 	`
-	getReactionByIDQuery                   = getReactionBaseQuery + `WHERE bridge_id=$1 AND message_id=$2 AND message_part_id=$3 AND sender_id=$4 AND emoji_id=$5`
-	getReactionByIDWithoutMessagePartQuery = getReactionBaseQuery + `WHERE bridge_id=$1 AND message_id=$2 AND sender_id=$3 AND emoji_id=$4 ORDER BY message_part_id ASC LIMIT 1`
-	getAllReactionsToMessageBySenderQuery  = getReactionBaseQuery + `WHERE bridge_id=$1 AND message_id=$2 AND sender_id=$3 ORDER BY timestamp DESC`
-	getAllReactionsToMessageQuery          = getReactionBaseQuery + `WHERE bridge_id=$1 AND message_id=$2`
+	getReactionByIDQuery                   = getReactionBaseQuery + `WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3 AND message_part_id=$4 AND sender_id=$5 AND emoji_id=$6`
+	getReactionByIDWithoutMessagePartQuery = getReactionBaseQuery + `WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3 AND sender_id=$4 AND emoji_id=$5 ORDER BY message_part_id ASC LIMIT 1`
+	getAllReactionsToMessageBySenderQuery  = getReactionBaseQuery + `WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3 AND sender_id=$4 ORDER BY timestamp DESC`
+	getAllReactionsToMessageQuery          = getReactionBaseQuery + `WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3`
+	getAllReactionsToMessagePartQuery      = getReactionBaseQuery + `WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3 AND message_part_id=$4`
 	getReactionByMXIDQuery                 = getReactionBaseQuery + `WHERE bridge_id=$1 AND mxid=$2`
 	upsertReactionQuery                    = `
-		INSERT INTO reaction (bridge_id, message_id, message_part_id, sender_id, emoji_id, emoji, room_id, room_receiver, mxid, timestamp, metadata)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		INSERT INTO reaction (bridge_id, message_id, message_part_id, sender_id, sender_mxid, emoji_id, emoji, room_id, room_receiver, mxid, timestamp, metadata)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
 		ON CONFLICT (bridge_id, room_receiver, message_id, message_part_id, sender_id, emoji_id)
-		DO UPDATE SET mxid=excluded.mxid, timestamp=excluded.timestamp, emoji=excluded.emoji, metadata=excluded.metadata
+		DO UPDATE SET sender_mxid=excluded.sender_mxid, mxid=excluded.mxid, timestamp=excluded.timestamp, emoji=excluded.emoji, metadata=excluded.metadata
 	`
 	deleteReactionQuery = `
-		DELETE FROM reaction WHERE bridge_id=$1 AND message_id=$2 AND message_part_id=$3 AND sender_id=$4 AND emoji_id=$5
+		DELETE FROM reaction WHERE bridge_id=$1 AND room_receiver=$2 AND message_id=$3 AND message_part_id=$4 AND sender_id=$5 AND emoji_id=$6
 	`
 )
 
-func (rq *ReactionQuery) GetByID(ctx context.Context, messageID networkid.MessageID, messagePartID networkid.PartID, senderID networkid.UserID, emojiID networkid.EmojiID) (*Reaction, error) {
-	return rq.QueryOne(ctx, getReactionByIDQuery, rq.BridgeID, messageID, messagePartID, senderID, emojiID)
+func (rq *ReactionQuery) GetByID(ctx context.Context, receiver networkid.UserLoginID, messageID networkid.MessageID, messagePartID networkid.PartID, senderID networkid.UserID, emojiID networkid.EmojiID) (*Reaction, error) {
+	return rq.QueryOne(ctx, getReactionByIDQuery, rq.BridgeID, receiver, messageID, messagePartID, senderID, emojiID)
 }
 
-func (rq *ReactionQuery) GetByIDWithoutMessagePart(ctx context.Context, messageID networkid.MessageID, senderID networkid.UserID, emojiID networkid.EmojiID) (*Reaction, error) {
-	return rq.QueryOne(ctx, getReactionByIDWithoutMessagePartQuery, rq.BridgeID, messageID, senderID, emojiID)
+func (rq *ReactionQuery) GetByIDWithoutMessagePart(ctx context.Context, receiver networkid.UserLoginID, messageID networkid.MessageID, senderID networkid.UserID, emojiID networkid.EmojiID) (*Reaction, error) {
+	return rq.QueryOne(ctx, getReactionByIDWithoutMessagePartQuery, rq.BridgeID, receiver, messageID, senderID, emojiID)
 }
 
-func (rq *ReactionQuery) GetAllToMessageBySender(ctx context.Context, messageID networkid.MessageID, senderID networkid.UserID) ([]*Reaction, error) {
-	return rq.QueryMany(ctx, getAllReactionsToMessageBySenderQuery, rq.BridgeID, messageID, senderID)
+func (rq *ReactionQuery) GetAllToMessageBySender(ctx context.Context, receiver networkid.UserLoginID, messageID networkid.MessageID, senderID networkid.UserID) ([]*Reaction, error) {
+	return rq.QueryMany(ctx, getAllReactionsToMessageBySenderQuery, rq.BridgeID, receiver, messageID, senderID)
 }
 
-func (rq *ReactionQuery) GetAllToMessage(ctx context.Context, messageID networkid.MessageID) ([]*Reaction, error) {
-	return rq.QueryMany(ctx, getAllReactionsToMessageQuery, rq.BridgeID, messageID)
+func (rq *ReactionQuery) GetAllToMessage(ctx context.Context, receiver networkid.UserLoginID, messageID networkid.MessageID) ([]*Reaction, error) {
+	return rq.QueryMany(ctx, getAllReactionsToMessageQuery, rq.BridgeID, receiver, messageID)
+}
+
+func (rq *ReactionQuery) GetAllToMessagePart(ctx context.Context, receiver networkid.UserLoginID, messageID networkid.MessageID, partID networkid.PartID) ([]*Reaction, error) {
+	return rq.QueryMany(ctx, getAllReactionsToMessagePartQuery, rq.BridgeID, receiver, messageID, partID)
 }
 
 func (rq *ReactionQuery) GetByMXID(ctx context.Context, mxid id.EventID) (*Reaction, error) {
@@ -83,13 +89,13 @@ func (rq *ReactionQuery) Upsert(ctx context.Context, reaction *Reaction) error {
 
 func (rq *ReactionQuery) Delete(ctx context.Context, reaction *Reaction) error {
 	ensureBridgeIDMatches(&reaction.BridgeID, rq.BridgeID)
-	return rq.Exec(ctx, deleteReactionQuery, reaction.BridgeID, reaction.MessageID, reaction.MessagePartID, reaction.SenderID, reaction.EmojiID)
+	return rq.Exec(ctx, deleteReactionQuery, reaction.BridgeID, reaction.Room.Receiver, reaction.MessageID, reaction.MessagePartID, reaction.SenderID, reaction.EmojiID)
 }
 
 func (r *Reaction) Scan(row dbutil.Scannable) (*Reaction, error) {
 	var timestamp int64
 	err := row.Scan(
-		&r.BridgeID, &r.MessageID, &r.MessagePartID, &r.SenderID, &r.EmojiID, &r.Emoji,
+		&r.BridgeID, &r.MessageID, &r.MessagePartID, &r.SenderID, &r.SenderMXID, &r.EmojiID, &r.Emoji,
 		&r.Room.ID, &r.Room.Receiver, &r.MXID, &timestamp, dbutil.JSON{Data: r.Metadata},
 	)
 	if err != nil {
@@ -108,7 +114,7 @@ func (r *Reaction) ensureHasMetadata(metaType MetaTypeCreator) *Reaction {
 
 func (r *Reaction) sqlVariables() []any {
 	return []any{
-		r.BridgeID, r.MessageID, r.MessagePartID, r.SenderID, r.EmojiID, r.Emoji,
+		r.BridgeID, r.MessageID, r.MessagePartID, r.SenderID, r.SenderMXID, r.EmojiID, r.Emoji,
 		r.Room.ID, r.Room.Receiver, r.MXID, r.Timestamp.UnixNano(), dbutil.JSON{Data: r.Metadata},
 	}
 }

bridgev2/database/upgrades/00-latest.sql

@@ -1,4 +1,4 @@
--- v0 -> v11 (compatible with v9+): Latest revision
+-- v0 -> v27 (compatible with v9+): Latest revision
 CREATE TABLE "user" (
 	bridge_id       TEXT NOT NULL,
 	mxid            TEXT NOT NULL,
@@ -10,12 +10,13 @@ CREATE TABLE "user" (
 );
 
 CREATE TABLE user_login (
-	bridge_id   TEXT  NOT NULL,
-	user_mxid   TEXT  NOT NULL,
-	id          TEXT  NOT NULL,
-	remote_name TEXT  NOT NULL,
-	space_room  TEXT,
-	metadata    jsonb NOT NULL,
+	bridge_id      TEXT  NOT NULL,
+	user_mxid      TEXT  NOT NULL,
+	id             TEXT  NOT NULL,
+	remote_name    TEXT  NOT NULL,
+	remote_profile jsonb,
+	space_room     TEXT,
+	metadata       jsonb NOT NULL,
 
 	PRIMARY KEY (bridge_id, id),
 	CONSTRAINT user_login_user_fkey FOREIGN KEY (bridge_id, user_mxid)
@@ -30,13 +31,13 @@ CREATE TABLE portal (
 	mxid            TEXT,
 
 	parent_id       TEXT,
-	-- This is not accessed by the bridge, it's only used for the portal parent foreign key.
-	-- Parent groups are probably never DMs, so they don't need a receiver.
 	parent_receiver TEXT    NOT NULL DEFAULT '',
 
 	relay_bridge_id TEXT,
 	relay_login_id  TEXT,
 
+	other_user_id   TEXT,
+
 	name            TEXT    NOT NULL,
 	topic           TEXT    NOT NULL,
 	avatar_id       TEXT    NOT NULL,
@@ -45,10 +46,13 @@ CREATE TABLE portal (
 	name_set        BOOLEAN NOT NULL,
 	avatar_set      BOOLEAN NOT NULL,
 	topic_set       BOOLEAN NOT NULL,
+	name_is_custom  BOOLEAN NOT NULL DEFAULT false,
 	in_space        BOOLEAN NOT NULL,
+	message_request BOOLEAN NOT NULL DEFAULT false,
 	room_type       TEXT    NOT NULL,
 	disappear_type  TEXT,
 	disappear_timer BIGINT,
+	cap_state       jsonb,
 	metadata        jsonb   NOT NULL,
 
 	PRIMARY KEY (bridge_id, id, receiver),
@@ -60,6 +64,8 @@ CREATE TABLE portal (
 		REFERENCES user_login (bridge_id, id)
 		ON DELETE SET NULL ON UPDATE CASCADE
 );
+CREATE UNIQUE INDEX portal_bridge_mxid_idx ON portal (bridge_id, mxid);
+CREATE INDEX portal_parent_idx ON portal (bridge_id, parent_id, parent_receiver);
 
 CREATE TABLE ghost (
 	bridge_id        TEXT    NOT NULL,
@@ -74,6 +80,7 @@ CREATE TABLE ghost (
 	contact_info_set BOOLEAN NOT NULL,
 	is_bot           BOOLEAN NOT NULL,
 	identifiers      jsonb   NOT NULL,
+	extra_profile    jsonb,
 	metadata         jsonb   NOT NULL,
 
 	PRIMARY KEY (bridge_id, id)
@@ -85,7 +92,7 @@ CREATE TABLE message (
 	-- would try to set bridge_id to null as well.
 
 	-- only: sqlite (line commented)
---	rowid      INTEGER PRIMARY KEY,
+--	rowid              INTEGER PRIMARY KEY,
 	-- only: postgres
 	rowid            BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
 
@@ -100,9 +107,11 @@ CREATE TABLE message (
 	sender_mxid      TEXT    NOT NULL,
 	timestamp        BIGINT  NOT NULL,
 	edit_count       INTEGER NOT NULL,
+	double_puppeted  BOOLEAN,
 	thread_root_id   TEXT,
 	reply_to_id      TEXT,
 	reply_to_part_id TEXT,
+	send_txn_id      TEXT,
 	metadata         jsonb   NOT NULL,
 
 	CONSTRAINT message_room_fkey FOREIGN KEY (bridge_id, room_id, room_receiver)
@@ -111,7 +120,9 @@ CREATE TABLE message (
 	CONSTRAINT message_sender_fkey FOREIGN KEY (bridge_id, sender_id)
 		REFERENCES ghost (bridge_id, id)
 		ON DELETE CASCADE ON UPDATE CASCADE,
-	CONSTRAINT message_real_pkey UNIQUE (bridge_id, room_receiver, id, part_id)
+	CONSTRAINT message_real_pkey UNIQUE (bridge_id, room_receiver, id, part_id),
+	CONSTRAINT message_mxid_unique UNIQUE (bridge_id, mxid),
+	CONSTRAINT message_txn_id_unique UNIQUE (bridge_id, room_receiver, send_txn_id)
 );
 CREATE INDEX message_room_idx ON message (bridge_id, room_id, room_receiver);
 
@@ -119,18 +130,25 @@ CREATE TABLE disappearing_message (
 	bridge_id    TEXT   NOT NULL,
 	mx_room      TEXT   NOT NULL,
 	mxid         TEXT   NOT NULL,
+	timestamp    BIGINT NOT NULL DEFAULT 0,
 	type         TEXT   NOT NULL,
 	timer        BIGINT NOT NULL,
 	disappear_at BIGINT,
 
-	PRIMARY KEY (bridge_id, mxid)
+	PRIMARY KEY (bridge_id, mxid),
+    CONSTRAINT disappearing_message_portal_fkey
+        FOREIGN KEY (bridge_id, mx_room)
+        REFERENCES portal (bridge_id, mxid)
+        ON DELETE CASCADE
 );
+CREATE INDEX disappearing_message_portal_idx ON disappearing_message (bridge_id, mx_room);
 
 CREATE TABLE reaction (
 	bridge_id       TEXT   NOT NULL,
 	message_id      TEXT   NOT NULL,
 	message_part_id TEXT   NOT NULL,
 	sender_id       TEXT   NOT NULL,
+	sender_mxid     TEXT   NOT NULL DEFAULT '',
 	emoji_id        TEXT   NOT NULL,
 	room_id         TEXT   NOT NULL,
 	room_receiver   TEXT   NOT NULL,
@@ -149,7 +167,8 @@ CREATE TABLE reaction (
 		ON DELETE CASCADE ON UPDATE CASCADE,
 	CONSTRAINT reaction_sender_fkey FOREIGN KEY (bridge_id, sender_id)
 		REFERENCES ghost (bridge_id, id)
-		ON DELETE CASCADE ON UPDATE CASCADE
+		ON DELETE CASCADE ON UPDATE CASCADE,
+	CONSTRAINT reaction_mxid_unique UNIQUE (bridge_id, mxid)
 );
 CREATE INDEX reaction_room_idx ON reaction (bridge_id, room_id, room_receiver);
 
@@ -173,3 +192,42 @@ CREATE TABLE user_portal (
 );
 CREATE INDEX user_portal_login_idx ON user_portal (bridge_id, login_id);
 CREATE INDEX user_portal_portal_idx ON user_portal (bridge_id, portal_id, portal_receiver);
+
+CREATE TABLE backfill_task (
+	bridge_id            TEXT    NOT NULL,
+	portal_id            TEXT    NOT NULL,
+	portal_receiver      TEXT    NOT NULL,
+	user_login_id        TEXT    NOT NULL,
+
+	batch_count          INTEGER NOT NULL,
+	is_done              BOOLEAN NOT NULL,
+	cursor               TEXT,
+	oldest_message_id    TEXT,
+	dispatched_at        BIGINT,
+	completed_at         BIGINT,
+	next_dispatch_min_ts BIGINT  NOT NULL,
+
+	PRIMARY KEY (bridge_id, portal_id, portal_receiver),
+	CONSTRAINT backfill_queue_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver)
+		REFERENCES portal (bridge_id, id, receiver)
+		ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE kv_store (
+	bridge_id TEXT NOT NULL,
+	key       TEXT NOT NULL,
+	value     TEXT NOT NULL,
+
+	PRIMARY KEY (bridge_id, key)
+);
+
+CREATE TABLE public_media (
+	bridge_id TEXT   NOT NULL,
+	public_id TEXT   NOT NULL,
+	mxc       TEXT   NOT NULL,
+	keys      jsonb,
+	mimetype  TEXT,
+	expiry    BIGINT,
+
+	PRIMARY KEY (bridge_id, public_id)
+);

bridgev2/database/upgrades/12-dm-portal-other-user.sql

@@ -0,0 +1,2 @@
+-- v12 (compatible with v9+): Save other user ID in DM portals
+ALTER TABLE portal ADD COLUMN other_user_id TEXT;

bridgev2/database/upgrades/13-backfill-queue.sql

@@ -0,0 +1,20 @@
+-- v13 (compatible with v9+): Add backfill queue
+CREATE TABLE backfill_task (
+	bridge_id            TEXT    NOT NULL,
+	portal_id            TEXT    NOT NULL,
+	portal_receiver      TEXT    NOT NULL,
+	user_login_id        TEXT    NOT NULL,
+
+	batch_count          INTEGER NOT NULL,
+	is_done              BOOLEAN NOT NULL,
+	cursor               TEXT,
+	oldest_message_id    TEXT,
+	dispatched_at        BIGINT,
+	completed_at         BIGINT,
+	next_dispatch_min_ts BIGINT  NOT NULL,
+
+	PRIMARY KEY (bridge_id, portal_id, portal_receiver),
+	CONSTRAINT backfill_queue_portal_fkey FOREIGN KEY (bridge_id, portal_id, portal_receiver)
+		REFERENCES portal (bridge_id, id, receiver)
+		ON DELETE CASCADE ON UPDATE CASCADE
+);

bridgev2/database/upgrades/14-portal-name-custom.sql

@@ -0,0 +1,2 @@
+-- v14 (compatible with v9+): Save whether name is custom in portals
+ALTER TABLE portal ADD COLUMN name_is_custom BOOLEAN NOT NULL DEFAULT false;

bridgev2/database/upgrades/15-reaction-sender-mxid.sql

@@ -0,0 +1,2 @@
+-- v15 (compatible with v9+): Save sender MXID for reactions
+ALTER TABLE reaction ADD COLUMN sender_mxid TEXT NOT NULL DEFAULT '';

bridgev2/database/upgrades/16-user-login-profile.sql

@@ -0,0 +1,2 @@
+-- v16 (compatible with v9+): Save remote profile in user logins
+ALTER TABLE user_login ADD COLUMN remote_profile jsonb;

bridgev2/database/upgrades/17-message-mxid-unique.sql

@@ -0,0 +1,8 @@
+-- v17 (compatible with v9+): Add unique constraint for message and reaction mxids
+DELETE FROM message WHERE mxid IN (SELECT mxid FROM message GROUP BY mxid HAVING COUNT(*) > 1);
+-- only: postgres for next 2 lines
+ALTER TABLE message ADD CONSTRAINT message_mxid_unique UNIQUE (bridge_id, mxid);
+ALTER TABLE reaction ADD CONSTRAINT reaction_mxid_unique UNIQUE (bridge_id, mxid);
+-- only: sqlite for next 2 lines
+CREATE UNIQUE INDEX message_mxid_unique ON message (bridge_id, mxid);
+CREATE UNIQUE INDEX reaction_mxid_unique ON reaction (bridge_id, mxid);

bridgev2/database/upgrades/18-kv-store.sql

@@ -0,0 +1,8 @@
+-- v18 (compatible with v9+): Add generic key-value store
+CREATE TABLE kv_store (
+	bridge_id TEXT NOT NULL,
+	key       TEXT NOT NULL,
+	value     TEXT NOT NULL,
+
+	PRIMARY KEY (bridge_id, key)
+);

bridgev2/database/upgrades/19-add-double-puppeted-to-message.sql

@@ -0,0 +1,2 @@
+-- v19 (compatible with v9+): Add double puppeted state to messages
+ALTER TABLE message ADD COLUMN double_puppeted BOOLEAN;

bridgev2/database/upgrades/20-portal-capabilities.sql

@@ -0,0 +1,2 @@
+-- v20 (compatible with v9+): Add portal capability state
+ALTER TABLE portal ADD COLUMN cap_state jsonb;

bridgev2/database/upgrades/21-disappearing-message-fkey.postgres.sql

@@ -0,0 +1,8 @@
+-- v21 (compatible with v9+): Add foreign key constraint from disappearing_message.mx_room to portals.mxid
+CREATE UNIQUE INDEX portal_bridge_mxid_idx ON portal (bridge_id, mxid);
+DELETE FROM disappearing_message WHERE mx_room NOT IN (SELECT mxid FROM portal WHERE mxid IS NOT NULL);
+ALTER TABLE disappearing_message
+    ADD CONSTRAINT disappearing_message_portal_fkey
+    FOREIGN KEY (bridge_id, mx_room)
+    REFERENCES portal (bridge_id, mxid)
+    ON DELETE CASCADE;

bridgev2/database/upgrades/21-disappearing-message-fkey.sqlite.sql

@@ -0,0 +1,24 @@
+-- v21 (compatible with v9+): Add foreign key constraint from disappearing_message.mx_room to portals.mxid
+CREATE UNIQUE INDEX portal_bridge_mxid_idx ON portal (bridge_id, mxid);
+CREATE TABLE disappearing_message_new (
+    bridge_id    TEXT   NOT NULL,
+    mx_room      TEXT   NOT NULL,
+    mxid         TEXT   NOT NULL,
+    type         TEXT   NOT NULL,
+    timer        BIGINT NOT NULL,
+    disappear_at BIGINT,
+
+    PRIMARY KEY (bridge_id, mxid),
+    CONSTRAINT disappearing_message_portal_fkey
+        FOREIGN KEY (bridge_id, mx_room)
+        REFERENCES portal (bridge_id, mxid)
+        ON DELETE CASCADE
+);
+
+WITH portal_mxids AS (SELECT mxid FROM portal WHERE mxid IS NOT NULL)
+INSERT INTO disappearing_message_new (bridge_id, mx_room, mxid, type, timer, disappear_at)
+SELECT bridge_id, mx_room, mxid, type, timer, disappear_at
+FROM disappearing_message WHERE mx_room IN portal_mxids;
+
+DROP TABLE disappearing_message;
+ALTER TABLE disappearing_message_new RENAME TO disappearing_message;

bridgev2/database/upgrades/22-message-send-txn-id.sql

@@ -0,0 +1,6 @@
+-- v22 (compatible with v9+): Add message send transaction ID column
+ALTER TABLE message ADD COLUMN send_txn_id TEXT;
+-- only: postgres
+ALTER TABLE message ADD CONSTRAINT message_txn_id_unique UNIQUE (bridge_id, room_receiver, send_txn_id);
+-- only: sqlite
+CREATE UNIQUE INDEX message_txn_id_unique ON message (bridge_id, room_receiver, send_txn_id);

bridgev2/database/upgrades/23-disappearing-timer-ts.sql

@@ -0,0 +1,2 @@
+-- v23 (compatible with v9+): Add event timestamp for disappearing messages
+ALTER TABLE disappearing_message ADD COLUMN timestamp BIGINT NOT NULL DEFAULT 0;

bridgev2/database/upgrades/24-public-media.sql

@@ -0,0 +1,11 @@
+-- v24 (compatible with v9+): Custom URLs for public media
+CREATE TABLE public_media (
+	bridge_id TEXT   NOT NULL,
+	public_id TEXT   NOT NULL,
+	mxc       TEXT   NOT NULL,
+	keys      jsonb,
+	mimetype  TEXT,
+	expiry    BIGINT,
+
+	PRIMARY KEY (bridge_id, public_id)
+);

bridgev2/database/upgrades/25-message-requests.sql

@@ -0,0 +1,2 @@
+-- v25 (compatible with v9+): Flag for message request portals
+ALTER TABLE portal ADD COLUMN message_request BOOLEAN NOT NULL DEFAULT false;

bridgev2/database/upgrades/26-disappearing-message-portal-index.sql

@@ -0,0 +1,3 @@
+-- v26 (compatible with v9+): Add room index for disappearing message table and portal parents
+CREATE INDEX disappearing_message_portal_idx ON disappearing_message (bridge_id, mx_room);
+CREATE INDEX portal_parent_idx ON portal (bridge_id, parent_id, parent_receiver);

bridgev2/database/upgrades/27-ghost-extra-profile.sql

@@ -0,0 +1,2 @@
+-- v27 (compatible with v9+): Add column for extra ghost profile metadata
+ALTER TABLE ghost ADD COLUMN extra_profile jsonb;

bridgev2/database/userlogin.go

@@ -13,6 +13,7 @@ import (
 	"go.mau.fi/util/dbutil"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/id"
 )
 
@@ -23,32 +24,33 @@ type UserLoginQuery struct {
 }
 
 type UserLogin struct {
-	BridgeID   networkid.BridgeID
-	UserMXID   id.UserID
-	ID         networkid.UserLoginID
-	RemoteName string
-	SpaceRoom  id.RoomID
-	Metadata   any
+	BridgeID      networkid.BridgeID
+	UserMXID      id.UserID
+	ID            networkid.UserLoginID
+	RemoteName    string
+	RemoteProfile status.RemoteProfile
+	SpaceRoom     id.RoomID
+	Metadata      any
 }
 
 const (
 	getUserLoginBaseQuery = `
-		SELECT bridge_id, user_mxid, id, remote_name, space_room, metadata FROM user_login
+		SELECT bridge_id, user_mxid, id, remote_name, remote_profile, space_room, metadata FROM user_login
 	`
 	getLoginByIDQuery          = getUserLoginBaseQuery + `WHERE bridge_id=$1 AND id=$2`
 	getAllUsersWithLoginsQuery = `SELECT DISTINCT user_mxid FROM user_login WHERE bridge_id=$1`
 	getAllLoginsForUserQuery   = getUserLoginBaseQuery + `WHERE bridge_id=$1 AND user_mxid=$2`
 	getAllLoginsInPortalQuery  = `
-		SELECT ul.bridge_id, ul.user_mxid, ul.id, ul.remote_name, ul.space_room, ul.metadata FROM user_portal
+		SELECT ul.bridge_id, ul.user_mxid, ul.id, ul.remote_name, ul.remote_profile, ul.space_room, ul.metadata FROM user_portal
 		LEFT JOIN user_login ul ON user_portal.bridge_id=ul.bridge_id AND user_portal.user_mxid=ul.user_mxid AND user_portal.login_id=ul.id
 		WHERE user_portal.bridge_id=$1 AND user_portal.portal_id=$2 AND user_portal.portal_receiver=$3
 	`
 	insertUserLoginQuery = `
-		INSERT INTO user_login (bridge_id, user_mxid, id, remote_name, space_room, metadata)
-		VALUES ($1, $2, $3, $4, $5, $6)
+		INSERT INTO user_login (bridge_id, user_mxid, id, remote_name, remote_profile, space_room, metadata)
+		VALUES ($1, $2, $3, $4, $5, $6, $7)
 	`
 	updateUserLoginQuery = `
-		UPDATE user_login SET remote_name=$4, space_room=$5, metadata=$6
+		UPDATE user_login SET remote_name=$4, remote_profile=$5, space_room=$6, metadata=$7
 		WHERE bridge_id=$1 AND user_mxid=$2 AND id=$3
 	`
 	deleteUserLoginQuery = `
@@ -89,7 +91,15 @@ func (uq *UserLoginQuery) Delete(ctx context.Context, loginID networkid.UserLogi
 
 func (u *UserLogin) Scan(row dbutil.Scannable) (*UserLogin, error) {
 	var spaceRoom sql.NullString
-	err := row.Scan(&u.BridgeID, &u.UserMXID, &u.ID, &u.RemoteName, &spaceRoom, dbutil.JSON{Data: u.Metadata})
+	err := row.Scan(
+		&u.BridgeID,
+		&u.UserMXID,
+		&u.ID,
+		&u.RemoteName,
+		dbutil.JSON{Data: &u.RemoteProfile},
+		&spaceRoom,
+		dbutil.JSON{Data: u.Metadata},
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -105,5 +115,9 @@ func (u *UserLogin) ensureHasMetadata(metaType MetaTypeCreator) *UserLogin {
 }
 
 func (u *UserLogin) sqlVariables() []any {
-	return []any{u.BridgeID, u.UserMXID, u.ID, u.RemoteName, dbutil.StrPtr(u.SpaceRoom), dbutil.JSON{Data: u.Metadata}}
+	var remoteProfile dbutil.JSON
+	if !u.RemoteProfile.IsZero() {
+		remoteProfile.Data = &u.RemoteProfile
+	}
+	return []any{u.BridgeID, u.UserMXID, u.ID, u.RemoteName, remoteProfile, dbutil.StrPtr(u.SpaceRoom), dbutil.JSON{Data: u.Metadata}}
 }

bridgev2/database/userportal.go

@@ -67,6 +67,12 @@ const (
 	markLoginAsPreferredQuery = `
 		UPDATE user_portal SET preferred=(login_id=$3) WHERE bridge_id=$1 AND user_mxid=$2 AND portal_id=$4 AND portal_receiver=$5
 	`
+	markAllNotInSpaceQuery = `
+		UPDATE user_portal SET in_space=false WHERE bridge_id=$1 AND portal_id=$2 AND portal_receiver=$3
+	`
+	deleteUserPortalQuery = `
+		DELETE FROM user_portal WHERE bridge_id=$1 AND user_mxid=$2 AND login_id=$3 AND portal_id=$4 AND portal_receiver=$5
+	`
 )
 
 func UserPortalFor(ul *UserLogin, portal networkid.PortalKey) *UserPortal {
@@ -107,6 +113,14 @@ func (upq *UserPortalQuery) MarkAsPreferred(ctx context.Context, login *UserLogi
 	return upq.Exec(ctx, markLoginAsPreferredQuery, upq.BridgeID, login.UserMXID, login.ID, portal.ID, portal.Receiver)
 }
 
+func (upq *UserPortalQuery) MarkAllNotInSpace(ctx context.Context, portal networkid.PortalKey) error {
+	return upq.Exec(ctx, markAllNotInSpaceQuery, upq.BridgeID, portal.ID, portal.Receiver)
+}
+
+func (upq *UserPortalQuery) Delete(ctx context.Context, up *UserPortal) error {
+	return upq.Exec(ctx, deleteUserPortalQuery, up.BridgeID, up.UserMXID, up.LoginID, up.Portal.ID, up.Portal.Receiver)
+}
+
 func (up *UserPortal) Scan(row dbutil.Scannable) (*UserPortal, error) {
 	var lastRead sql.NullInt64
 	err := row.Scan(

bridgev2/disappear.go

@@ -8,6 +8,7 @@ package bridgev2
 
 import (
 	"context"
+	"sync/atomic"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -20,27 +21,44 @@ import (
 
 type DisappearLoop struct {
 	br        *Bridge
-	NextCheck time.Time
-	stop      context.CancelFunc
+	nextCheck atomic.Pointer[time.Time]
+	stop      atomic.Pointer[context.CancelFunc]
 }
 
 const DisappearCheckInterval = 1 * time.Hour
 
 func (dl *DisappearLoop) Start() {
 	log := dl.br.Log.With().Str("component", "disappear loop").Logger()
-	ctx := log.WithContext(context.Background())
-	ctx, dl.stop = context.WithCancel(ctx)
+	ctx, stop := context.WithCancel(log.WithContext(context.Background()))
+	if oldStop := dl.stop.Swap(&stop); oldStop != nil {
+		(*oldStop)()
+	}
 	log.Debug().Msg("Disappearing message loop starting")
 	for {
-		dl.NextCheck = time.Now().Add(DisappearCheckInterval)
-		messages, err := dl.br.DB.DisappearingMessage.GetUpcoming(ctx, DisappearCheckInterval)
+		nextCheck := time.Now().Add(DisappearCheckInterval)
+		dl.nextCheck.Store(&nextCheck)
+		const MessageLimit = 200
+		messages, err := dl.br.DB.DisappearingMessage.GetUpcoming(ctx, DisappearCheckInterval, MessageLimit)
 		if err != nil {
 			log.Err(err).Msg("Failed to get upcoming disappearing messages")
 		} else if len(messages) > 0 {
+			if len(messages) >= MessageLimit {
+				lastDisappearTime := messages[len(messages)-1].DisappearAt
+				log.Debug().
+					Int("message_count", len(messages)).
+					Time("last_due", lastDisappearTime).
+					Msg("Deleting disappearing messages synchronously and checking again immediately")
+				// Store the expected next check time to avoid Add spawning unnecessary goroutines.
+				// This can be in the past, in which case Add will put everything in the db, which is also fine.
+				dl.nextCheck.Store(&lastDisappearTime)
+				// If there are many messages, process them synchronously and then check again.
+				dl.sleepAndDisappear(ctx, messages...)
+				continue
+			}
 			go dl.sleepAndDisappear(ctx, messages...)
 		}
 		select {
-		case <-time.After(time.Until(dl.NextCheck)):
+		case <-time.After(time.Until(dl.GetNextCheck())):
 		case <-ctx.Done():
 			log.Debug().Msg("Disappearing message loop stopping")
 			return
@@ -48,20 +66,34 @@ func (dl *DisappearLoop) Start() {
 	}
 }
 
+func (dl *DisappearLoop) GetNextCheck() time.Time {
+	if dl == nil {
+		return time.Time{}
+	}
+	nextCheck := dl.nextCheck.Load()
+	if nextCheck == nil {
+		return time.Time{}
+	}
+	return *nextCheck
+}
+
 func (dl *DisappearLoop) Stop() {
-	if dl.stop != nil {
-		dl.stop()
+	if dl == nil {
+		return
+	}
+	if stop := dl.stop.Load(); stop != nil {
+		(*stop)()
 	}
 }
 
-func (dl *DisappearLoop) StartAll(ctx context.Context, roomID id.RoomID) {
-	startedMessages, err := dl.br.DB.DisappearingMessage.StartAll(ctx, roomID)
+func (dl *DisappearLoop) StartAllBefore(ctx context.Context, roomID id.RoomID, beforeTS time.Time) {
+	startedMessages, err := dl.br.DB.DisappearingMessage.StartAllBefore(ctx, roomID, beforeTS)
 	if err != nil {
 		zerolog.Ctx(ctx).Err(err).Msg("Failed to start disappearing messages")
 		return
 	}
 	startedMessages = slices.DeleteFunc(startedMessages, func(dm *database.DisappearingMessage) bool {
-		return dm.DisappearAt.After(dl.NextCheck)
+		return dm.DisappearAt.After(dl.GetNextCheck())
 	})
 	slices.SortFunc(startedMessages, func(a, b *database.DisappearingMessage) int {
 		return a.DisappearAt.Compare(b.DisappearAt)
@@ -78,20 +110,31 @@ func (dl *DisappearLoop) Add(ctx context.Context, dm *database.DisappearingMessa
 			Stringer("event_id", dm.EventID).
 			Msg("Failed to save disappearing message")
 	}
-	if !dm.DisappearAt.IsZero() && dm.DisappearAt.Before(dl.NextCheck) {
-		go dl.sleepAndDisappear(context.WithoutCancel(ctx), dm)
+	if !dm.DisappearAt.IsZero() && dm.DisappearAt.Before(dl.GetNextCheck()) {
+		go dl.sleepAndDisappear(zerolog.Ctx(ctx).WithContext(dl.br.BackgroundCtx), dm)
 	}
 }
 
 func (dl *DisappearLoop) sleepAndDisappear(ctx context.Context, dms ...*database.DisappearingMessage) {
 	for _, msg := range dms {
-		time.Sleep(time.Until(msg.DisappearAt))
+		timeUntilDisappear := time.Until(msg.DisappearAt)
+		if timeUntilDisappear <= 0 {
+			if ctx.Err() != nil {
+				return
+			}
+		} else {
+			select {
+			case <-time.After(timeUntilDisappear):
+			case <-ctx.Done():
+				return
+			}
+		}
 		resp, err := dl.br.Bot.SendMessage(ctx, msg.RoomID, event.EventRedaction, &event.Content{
 			Parsed: &event.RedactionEventContent{
 				Redacts: msg.EventID,
 				Reason:  "Message disappeared",
 			},
-		}, time.Now())
+		}, nil)
 		if err != nil {
 			zerolog.Ctx(ctx).Err(err).Stringer("target_event_id", msg.EventID).Msg("Failed to disappear message")
 		} else {

bridgev2/errors.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2024 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package bridgev2
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/event"
+)
+
+// ErrIgnoringRemoteEvent can be returned by [RemoteMessage.ConvertMessage] or [RemoteEdit.ConvertEdit]
+// to indicate that the event should be ignored after all. Handling the event will be cancelled immediately.
+var ErrIgnoringRemoteEvent = errors.New("ignoring remote event")
+
+// ErrNoStatus can be returned by [MatrixMessageResponse.HandleEcho] to indicate that the message is still in-flight
+// and a status should not be sent yet. The message will still be saved into the database.
+var ErrNoStatus = errors.New("omit message status")
+
+// ErrResolveIdentifierTryNext can be returned by ResolveIdentifier or CreateChatWithGhost to signal that
+// the identifier is valid, but can't be reached by the current login, and the caller should try the next
+// login if there are more.
+//
+// This should generally only be returned when resolving internal IDs (which happens when initiating chats via Matrix).
+// For example, Google Messages would return this when trying to resolve another login's user ID,
+// and Telegram would return this when the access hash isn't available.
+var ErrResolveIdentifierTryNext = errors.New("that identifier is not available via this login")
+
+var ErrNotLoggedIn = errors.New("not logged in")
+
+// ErrDirectMediaNotEnabled may be returned by Matrix connectors if [MatrixConnector.GenerateContentURI] is called,
+// but direct media is not enabled.
+var ErrDirectMediaNotEnabled = errors.New("direct media is not enabled")
+
+var ErrPortalIsDeleted = errors.New("portal is deleted")
+var ErrPortalNotFoundInEventHandler = errors.New("portal not found to handle remote event")
+
+// Common message status errors
+var (
+	ErrPanicInEventHandler              error = WrapErrorInStatus(errors.New("panic in event handler")).WithSendNotice(true).WithErrorAsMessage()
+	ErrNoPortal                         error = WrapErrorInStatus(errors.New("room is not a portal")).WithIsCertain(true).WithSendNotice(false)
+	ErrIgnoringReactionFromRelayedUser  error = WrapErrorInStatus(errors.New("ignoring reaction event from relayed user")).WithIsCertain(true).WithSendNotice(false)
+	ErrIgnoringPollFromRelayedUser      error = WrapErrorInStatus(errors.New("ignoring poll event from relayed user")).WithIsCertain(true).WithSendNotice(false)
+	ErrIgnoringDeleteChatRelayedUser    error = WrapErrorInStatus(errors.New("ignoring delete chat event from relayed user")).WithIsCertain(true).WithSendNotice(false)
+	ErrIgnoringAcceptRequestRelayedUser error = WrapErrorInStatus(errors.New("ignoring accept message request event from relayed user")).WithIsCertain(true).WithSendNotice(false)
+	ErrEditsNotSupported                error = WrapErrorInStatus(errors.New("this bridge does not support edits")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrEditsNotSupportedInPortal        error = WrapErrorInStatus(errors.New("edits are not allowed in this chat")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrCaptionsNotAllowed               error = WrapErrorInStatus(errors.New("captions are not supported here")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrLocationMessagesNotAllowed       error = WrapErrorInStatus(errors.New("location messages are not supported here")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrEditTargetTooOld                 error = WrapErrorInStatus(errors.New("the message is too old to be edited")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrEditTargetTooManyEdits           error = WrapErrorInStatus(errors.New("the message has been edited too many times")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrReactionsNotSupported            error = WrapErrorInStatus(errors.New("this bridge does not support reactions")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrPollsNotSupported                error = WrapErrorInStatus(errors.New("this bridge does not support polls")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrRoomMetadataNotSupported         error = WrapErrorInStatus(errors.New("this bridge does not support changing room metadata")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrRoomMetadataNotAllowed           error = WrapErrorInStatus(errors.New("changes are not allowed here")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrRedactionsNotSupported           error = WrapErrorInStatus(errors.New("this bridge does not support deleting messages")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported)
+	ErrUnexpectedParsedContentType      error = WrapErrorInStatus(errors.New("unexpected parsed content type")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(true)
+	ErrInvalidStateKey                  error = WrapErrorInStatus(errors.New("room metadata state key is unset or non-empty")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(false)
+	ErrDatabaseError                    error = WrapErrorInStatus(errors.New("database error")).WithMessage("internal database error").WithIsCertain(true).WithSendNotice(true)
+	ErrTargetMessageNotFound            error = WrapErrorInStatus(errors.New("target message not found")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(false)
+	ErrUnsupportedMessageType           error = WrapErrorInStatus(errors.New("unsupported message type")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported)
+	ErrUnsupportedMediaType             error = WrapErrorInStatus(errors.New("unsupported media type")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported)
+	ErrMediaDurationTooLong             error = WrapErrorInStatus(errors.New("media duration too long")).WithErrorAsMessage().WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported)
+	ErrVoiceMessageDurationTooLong      error = WrapErrorInStatus(errors.New("voice message too long")).WithErrorAsMessage().WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported)
+	ErrMediaTooLarge                    error = WrapErrorInStatus(errors.New("media too large")).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported)
+	ErrIgnoringMNotice                  error = WrapErrorInStatus(errors.New("ignoring m.notice message")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false)
+	ErrMediaDownloadFailed              error = WrapErrorInStatus(errors.New("failed to download media")).WithMessage("failed to download media").WithIsCertain(true).WithSendNotice(true)
+	ErrMediaReuploadFailed              error = WrapErrorInStatus(errors.New("failed to reupload media")).WithMessage("failed to reupload media").WithIsCertain(true).WithSendNotice(true)
+	ErrMediaConvertFailed               error = WrapErrorInStatus(errors.New("failed to convert media")).WithMessage("failed to convert media").WithIsCertain(true).WithSendNotice(true)
+	ErrMembershipNotSupported           error = WrapErrorInStatus(errors.New("this bridge does not support changing group membership")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrDeleteChatNotSupported           error = WrapErrorInStatus(errors.New("this bridge does not support deleting chats")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrBeeperAIStreamNotSupported       error = WrapErrorInStatus(errors.New("this bridge does not support Beeper AI stream events")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrPowerLevelsNotSupported          error = WrapErrorInStatus(errors.New("this bridge does not support changing group power levels")).WithIsCertain(true).WithErrorAsMessage().WithSendNotice(false).WithErrorReason(event.MessageStatusUnsupported)
+	ErrRemoteEchoTimeout                      = WrapErrorInStatus(errors.New("remote echo timed out")).WithIsCertain(false).WithSendNotice(true).WithErrorReason(event.MessageStatusTooOld)
+	ErrRemoteAckTimeout                       = WrapErrorInStatus(errors.New("remote ack timed out")).WithIsCertain(false).WithSendNotice(true).WithErrorReason(event.MessageStatusTooOld)
+
+	ErrPublicMediaDisabled         = WrapErrorInStatus(errors.New("public media is not enabled in the bridge config")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported).WithSendNotice(true)
+	ErrPublicMediaDatabaseDisabled = WrapErrorInStatus(errors.New("public media database storage is disabled")).WithIsCertain(true).WithErrorAsMessage().WithErrorReason(event.MessageStatusUnsupported).WithSendNotice(true)
+	ErrPublicMediaGenerateFailed   = WrapErrorInStatus(errors.New("failed to generate public media URL")).WithIsCertain(true).WithMessage("failed to generate public media URL").WithErrorReason(event.MessageStatusUnsupported).WithSendNotice(true)
+
+	ErrDisappearingTimerUnsupported error = WrapErrorInStatus(errors.New("invalid disappearing timer")).WithIsCertain(true)
+)
+
+// Common login interface errors
+var (
+	ErrInvalidLoginFlowID error = RespError(mautrix.MNotFound.WithMessage("Invalid login flow ID"))
+)
+
+// RespError is a class of error that certain network interface methods can return to ensure that the error
+// is properly translated into an HTTP error when the method is called via the provisioning API.
+//
+// However, unlike mautrix.RespError, this does not include the error code
+// in the message shown to users when used outside HTTP contexts.
+type RespError mautrix.RespError
+
+func (re RespError) Error() string {
+	return re.Err
+}
+
+func (re RespError) Is(err error) bool {
+	var e2 RespError
+	if errors.As(err, &e2) {
+		return e2.Err == re.Err
+	}
+	return errors.Is(err, mautrix.RespError(re))
+}
+
+func (re RespError) Write(w http.ResponseWriter) {
+	mautrix.RespError(re).Write(w)
+}
+
+func (re RespError) WithMessage(msg string, args ...any) RespError {
+	return RespError(mautrix.RespError(re).WithMessage(msg, args...))
+}
+
+func (re RespError) AppendMessage(append string, args ...any) RespError {
+	re.Err += fmt.Sprintf(append, args...)
+	return re
+}
+
+func WrapRespErrManual(err error, code string, status int) RespError {
+	return RespError{ErrCode: code, Err: err.Error(), StatusCode: status}
+}
+
+func WrapRespErr(err error, target mautrix.RespError) RespError {
+	return RespError{ErrCode: target.ErrCode, Err: err.Error(), StatusCode: target.StatusCode}
+}

bridgev2/ghost.go

@@ -9,12 +9,15 @@ package bridgev2
 import (
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
+	"maps"
 	"net/http"
+	"slices"
 
 	"github.com/rs/zerolog"
+	"go.mau.fi/util/exerrors"
 	"go.mau.fi/util/exmime"
-	"golang.org/x/exp/slices"
 
 	"maunium.net/go/mautrix/bridgev2/database"
 	"maunium.net/go/mautrix/bridgev2/networkid"
@@ -69,6 +72,11 @@ func (br *Bridge) unlockedGetGhostByID(ctx context.Context, id networkid.UserID,
 	return br.loadGhost(ctx, db, err, idPtr)
 }
 
+func (br *Bridge) IsGhostMXID(userID id.UserID) bool {
+	_, isGhost := br.Matrix.ParseGhostMXID(userID)
+	return isGhost
+}
+
 func (br *Bridge) GetGhostByMXID(ctx context.Context, mxid id.UserID) (*Ghost, error) {
 	ghostID, ok := br.Matrix.ParseGhostMXID(mxid)
 	if !ok {
@@ -80,7 +88,19 @@ func (br *Bridge) GetGhostByMXID(ctx context.Context, mxid id.UserID) (*Ghost, e
 func (br *Bridge) GetGhostByID(ctx context.Context, id networkid.UserID) (*Ghost, error) {
 	br.cacheLock.Lock()
 	defer br.cacheLock.Unlock()
-	return br.unlockedGetGhostByID(ctx, id, false)
+	ghost, err := br.unlockedGetGhostByID(ctx, id, false)
+	if err != nil {
+		return nil, err
+	} else if ghost == nil {
+		panic(fmt.Errorf("unlockedGetGhostByID(ctx, %q, false) returned nil", id))
+	}
+	return ghost, nil
+}
+
+func (br *Bridge) GetExistingGhostByID(ctx context.Context, id networkid.UserID) (*Ghost, error) {
+	br.cacheLock.Lock()
+	defer br.cacheLock.Unlock()
+	return br.unlockedGetGhostByID(ctx, id, true)
 }
 
 type Avatar struct {
@@ -93,17 +113,19 @@ type Avatar struct {
 	Hash [32]byte
 }
 
-func (a *Avatar) Reupload(ctx context.Context, intent MatrixAPI, currentHash [32]byte) (id.ContentURIString, [32]byte, error) {
-	if a.MXC != "" {
+func (a *Avatar) Reupload(ctx context.Context, intent MatrixAPI, currentHash [32]byte, currentMXC id.ContentURIString) (id.ContentURIString, [32]byte, error) {
+	if a.MXC != "" || a.Hash != [32]byte{} {
 		return a.MXC, a.Hash, nil
+	} else if a.Get == nil {
+		return "", [32]byte{}, fmt.Errorf("no Get function provided for avatar")
 	}
 	data, err := a.Get(ctx)
 	if err != nil {
 		return "", [32]byte{}, err
 	}
 	hash := sha256.Sum256(data)
-	if hash == currentHash {
-		return "", hash, nil
+	if hash == currentHash && currentMXC != "" {
+		return currentMXC, hash, nil
 	}
 	mime := http.DetectContentType(data)
 	fileName := "avatar" + exmime.ExtensionFromMimetype(mime)
@@ -115,12 +137,13 @@ func (a *Avatar) Reupload(ctx context.Context, intent MatrixAPI, currentHash [32
 }
 
 type UserInfo struct {
-	Identifiers []string
-	Name        *string
-	Avatar      *Avatar
-	IsBot       *bool
+	Identifiers  []string
+	Name         *string
+	Avatar       *Avatar
+	IsBot        *bool
+	ExtraProfile database.ExtraProfile
 
-	ExtraUpdates func(context.Context, *Ghost) bool
+	ExtraUpdates ExtraUpdater[*Ghost]
 }
 
 func (ghost *Ghost) UpdateName(ctx context.Context, name string) bool {
@@ -139,18 +162,20 @@ func (ghost *Ghost) UpdateName(ctx context.Context, name string) bool {
 }
 
 func (ghost *Ghost) UpdateAvatar(ctx context.Context, avatar *Avatar) bool {
-	if ghost.AvatarID == avatar.ID && ghost.AvatarSet {
+	if ghost.AvatarID == avatar.ID && (avatar.Remove || ghost.AvatarMXC != "") && ghost.AvatarSet {
 		return false
 	}
 	ghost.AvatarID = avatar.ID
 	if !avatar.Remove {
-		newMXC, newHash, err := avatar.Reupload(ctx, ghost.Intent, ghost.AvatarHash)
+		newMXC, newHash, err := avatar.Reupload(ctx, ghost.Intent, ghost.AvatarHash, ghost.AvatarMXC)
 		if err != nil {
+			ghost.AvatarSet = false
 			zerolog.Ctx(ctx).Err(err).Msg("Failed to reupload avatar")
 			return true
-		} else if newHash == ghost.AvatarHash {
+		} else if newHash == ghost.AvatarHash && ghost.AvatarMXC != "" && ghost.AvatarSet {
 			return true
 		}
+		ghost.AvatarHash = newHash
 		ghost.AvatarMXC = newMXC
 	} else {
 		ghost.AvatarMXC = ""
@@ -164,23 +189,9 @@ func (ghost *Ghost) UpdateAvatar(ctx context.Context, avatar *Avatar) bool {
 	return true
 }
 
-func (ghost *Ghost) UpdateContactInfo(ctx context.Context, identifiers []string, isBot *bool) bool {
-	if identifiers != nil {
-		slices.Sort(identifiers)
-	}
-	if ghost.ContactInfoSet &&
-		(identifiers == nil || slices.Equal(identifiers, ghost.Identifiers)) &&
-		(isBot == nil || *isBot == ghost.IsBot) {
-		return false
-	}
-	if identifiers != nil {
-		ghost.Identifiers = identifiers
-	}
-	if isBot != nil {
-		ghost.IsBot = *isBot
-	}
+func (ghost *Ghost) getExtraProfileMeta() any {
 	bridgeName := ghost.Bridge.Network.GetName()
-	meta := &event.BeeperProfileExtra{
+	baseExtra := &event.BeeperProfileExtra{
 		RemoteID:     string(ghost.ID),
 		Identifiers:  ghost.Identifiers,
 		Service:      bridgeName.BeeperBridgeType,
@@ -188,7 +199,36 @@ func (ghost *Ghost) UpdateContactInfo(ctx context.Context, identifiers []string,
 		IsBridgeBot:  false,
 		IsNetworkBot: ghost.IsBot,
 	}
-	err := ghost.Intent.SetExtraProfileMeta(ctx, meta)
+	if len(ghost.ExtraProfile) == 0 {
+		return baseExtra
+	}
+	mergedExtra := maps.Clone(ghost.ExtraProfile)
+	baseExtraMarshaled := exerrors.Must(json.Marshal(baseExtra))
+	exerrors.PanicIfNotNil(json.Unmarshal(baseExtraMarshaled, &mergedExtra))
+	return mergedExtra
+}
+
+func (ghost *Ghost) UpdateContactInfo(ctx context.Context, identifiers []string, isBot *bool, extraProfile database.ExtraProfile) bool {
+	if !ghost.Bridge.Matrix.GetCapabilities().ExtraProfileMeta {
+		ghost.ContactInfoSet = false
+		return false
+	}
+	if identifiers != nil {
+		slices.Sort(identifiers)
+	}
+	changed := extraProfile.CopyTo(&ghost.ExtraProfile)
+	if identifiers != nil {
+		changed = changed || !slices.Equal(identifiers, ghost.Identifiers)
+		ghost.Identifiers = identifiers
+	}
+	if isBot != nil {
+		changed = changed || *isBot != ghost.IsBot
+		ghost.IsBot = *isBot
+	}
+	if ghost.ContactInfoSet && !changed {
+		return false
+	}
+	err := ghost.Intent.SetExtraProfileMeta(ctx, ghost.getExtraProfileMeta())
 	if err != nil {
 		zerolog.Ctx(ctx).Err(err).Msg("Failed to set extra profile metadata")
 	} else {
@@ -210,31 +250,68 @@ func (br *Bridge) allowAggressiveUpdateForType(evtType RemoteEventType) bool {
 }
 
 func (ghost *Ghost) UpdateInfoIfNecessary(ctx context.Context, source *UserLogin, evtType RemoteEventType) {
-	if ghost.Name != "" && ghost.NameSet && !ghost.Bridge.allowAggressiveUpdateForType(evtType) {
+	if ghost.Name != "" && ghost.NameSet && ghost.AvatarSet && !ghost.Bridge.allowAggressiveUpdateForType(evtType) {
 		return
 	}
 	info, err := source.Client.GetUserInfo(ctx, ghost)
 	if err != nil {
 		zerolog.Ctx(ctx).Err(err).Str("ghost_id", string(ghost.ID)).Msg("Failed to get info to update ghost")
 	} else if info != nil {
+		zerolog.Ctx(ctx).Debug().
+			Bool("has_name", ghost.Name != "").
+			Bool("name_set", ghost.NameSet).
+			Bool("has_avatar", ghost.AvatarMXC != "").
+			Bool("avatar_set", ghost.AvatarSet).
+			Msg("Updating ghost info in IfNecessary call")
 		ghost.UpdateInfo(ctx, info)
+	} else {
+		zerolog.Ctx(ctx).Trace().
+			Bool("has_name", ghost.Name != "").
+			Bool("name_set", ghost.NameSet).
+			Bool("has_avatar", ghost.AvatarMXC != "").
+			Bool("avatar_set", ghost.AvatarSet).
+			Msg("No ghost info received in IfNecessary call")
+	}
+}
+
+func (ghost *Ghost) updateDMPortals(ctx context.Context) {
+	if !ghost.Bridge.Config.PrivateChatPortalMeta {
+		return
+	}
+	dmPortals, err := ghost.Bridge.GetDMPortalsWith(ctx, ghost.ID)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to get DM portals to update info")
+		return
+	}
+	for _, portal := range dmPortals {
+		go portal.lockedUpdateInfoFromGhost(ctx, ghost)
 	}
 }
 
 func (ghost *Ghost) UpdateInfo(ctx context.Context, info *UserInfo) {
 	update := false
+	oldName := ghost.Name
+	oldAvatar := ghost.AvatarMXC
 	if info.Name != nil {
 		update = ghost.UpdateName(ctx, *info.Name) || update
 	}
 	if info.Avatar != nil {
 		update = ghost.UpdateAvatar(ctx, info.Avatar) || update
+	} else if oldAvatar == "" && !ghost.AvatarSet {
+		// Special case: nil avatar means we're not expecting one ever, if we don't currently have
+		// one we flag it as set to avoid constantly refetching in UpdateInfoIfNecessary.
+		ghost.AvatarSet = true
+		update = true
 	}
-	if info.Identifiers != nil || info.IsBot != nil {
-		update = ghost.UpdateContactInfo(ctx, info.Identifiers, info.IsBot) || update
+	if info.Identifiers != nil || info.IsBot != nil || info.ExtraProfile != nil {
+		update = ghost.UpdateContactInfo(ctx, info.Identifiers, info.IsBot, info.ExtraProfile) || update
 	}
 	if info.ExtraUpdates != nil {
 		update = info.ExtraUpdates(ctx, ghost) || update
 	}
+	if oldName != ghost.Name || oldAvatar != ghost.AvatarMXC {
+		ghost.updateDMPortals(ctx)
+	}
 	if update {
 		err := ghost.Bridge.DB.Ghost.Update(ctx, ghost.Ghost)
 		if err != nil {

bridgev2/login.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/event"
 )
 
 // LoginProcess represents a single occurrence of a user logging into the remote network.
@@ -32,6 +33,17 @@ type LoginProcess interface {
 	Cancel()
 }
 
+type LoginProcessWithOverride interface {
+	LoginProcess
+	// StartWithOverride starts the process with the intent of re-authenticating an existing login.
+	//
+	// The call to this is mutually exclusive with the call to the default Start method.
+	//
+	// The user login being overridden will still be logged out automatically
+	// in case the complete step returns a different login.
+	StartWithOverride(ctx context.Context, override *UserLogin) (*LoginStep, error)
+}
+
 type LoginProcessDisplayAndWait interface {
 	LoginProcess
 	Wait(ctx context.Context) (*LoginStep, error)
@@ -101,15 +113,59 @@ type LoginDisplayAndWaitParams struct {
 	ImageURL string `json:"image_url,omitempty"`
 }
 
+type LoginCookieFieldSourceType string
+
+const (
+	LoginCookieTypeCookie        LoginCookieFieldSourceType = "cookie"
+	LoginCookieTypeLocalStorage  LoginCookieFieldSourceType = "local_storage"
+	LoginCookieTypeRequestHeader LoginCookieFieldSourceType = "request_header"
+	LoginCookieTypeRequestBody   LoginCookieFieldSourceType = "request_body"
+	LoginCookieTypeSpecial       LoginCookieFieldSourceType = "special"
+)
+
+type LoginCookieFieldSource struct {
+	// The type of source.
+	Type LoginCookieFieldSourceType `json:"type"`
+	// The name of the field. The exact meaning depends on the type of source.
+	// Cookie:         cookie name
+	// Local storage:  key in local storage
+	// Request header: header name
+	// Request body:   field name inside body after it's parsed (as JSON or multipart form data)
+	// Special:        a namespaced identifier that clients can implement special handling for
+	Name string `json:"name"`
+
+	// For request header & body types, a regex matching request URLs where the value can be extracted from.
+	RequestURLRegex string `json:"request_url_regex,omitempty"`
+	// For cookie types, the domain the cookie is present on.
+	CookieDomain string `json:"cookie_domain,omitempty"`
+}
+
+type LoginCookieField struct {
+	// The key in the map that is submitted to the connector.
+	ID       string `json:"id"`
+	Required bool   `json:"required"`
+	// The sources that can be used to acquire the field value. Only one of these needs to be used.
+	Sources []LoginCookieFieldSource `json:"sources"`
+	// A regex pattern that the client can use to validate value client-side.
+	Pattern string `json:"pattern,omitempty"`
+}
+
 type LoginCookiesParams struct {
 	URL       string `json:"url"`
 	UserAgent string `json:"user_agent,omitempty"`
 
-	CookieDomain     string   `json:"cookie_domain,omitempty"`
-	CookieKeys       []string `json:"cookie_keys,omitempty"`
-	LocalStorageKeys []string `json:"local_storage_keys,omitempty"`
-	SpecialKeys      []string `json:"special_keys,omitempty"`
-	SpecialExtractJS string   `json:"special_extract_js,omitempty"`
+	// The fields that are needed for this cookie login.
+	Fields []LoginCookieField `json:"fields"`
+	// A JavaScript snippet that can extract some or all of the fields.
+	// The snippet will evaluate to a promise that resolves when the relevant fields are found.
+	// Fields that are not present in the promise result must be extracted another way.
+	ExtractJS string `json:"extract_js,omitempty"`
+	// A regex pattern that the URL should match before the client closes the webview.
+	//
+	// The client may submit the login if the user closes the webview after all cookies are collected
+	// even if this URL is not reached, but it should only automatically close the webview after
+	// both cookies and the URL match.
+	WaitForURLPattern string `json:"wait_for_url_pattern,omitempty"`
 }
 
 type LoginInputFieldType string
@@ -120,6 +176,11 @@ const (
 	LoginInputFieldTypePhoneNumber LoginInputFieldType = "phone_number"
 	LoginInputFieldTypeEmail       LoginInputFieldType = "email"
 	LoginInputFieldType2FACode     LoginInputFieldType = "2fa_code"
+	LoginInputFieldTypeToken       LoginInputFieldType = "token"
+	LoginInputFieldTypeURL         LoginInputFieldType = "url"
+	LoginInputFieldTypeDomain      LoginInputFieldType = "domain"
+	LoginInputFieldTypeSelect      LoginInputFieldType = "select"
+	LoginInputFieldTypeCaptchaCode LoginInputFieldType = "captcha_code"
 )
 
 type LoginInputDataField struct {
@@ -131,8 +192,13 @@ type LoginInputDataField struct {
 	Name string `json:"name"`
 	// The description of the field shown to the user.
 	Description string `json:"description"`
+	// A default value that the client can pre-fill the field with.
+	DefaultValue string `json:"default_value,omitempty"`
 	// A regex pattern that the client can use to validate input client-side.
 	Pattern string `json:"pattern,omitempty"`
+	// For fields of type select, the valid options.
+	// Pattern may also be filled with a regex that matches the same options.
+	Options []string `json:"options,omitempty"`
 	// A function that validates the input and optionally cleans it up before it's submitted to the connector.
 	Validate func(string) (string, error) `json:"-"`
 }
@@ -148,6 +214,14 @@ func isOnlyNumbers(input string) bool {
 	return true
 }
 
+func CleanNonInternationalPhoneNumber(phone string) (string, error) {
+	phone = numberCleaner.Replace(phone)
+	if !isOnlyNumbers(strings.TrimPrefix(phone, "+")) {
+		return "", fmt.Errorf("phone number must only contain numbers")
+	}
+	return phone, nil
+}
+
 func CleanPhoneNumber(phone string) (string, error) {
 	phone = numberCleaner.Replace(phone)
 	if len(phone) < 2 {
@@ -199,6 +273,23 @@ func (f *LoginInputDataField) FillDefaultValidate() {
 type LoginUserInputParams struct {
 	// The fields that the user needs to fill in.
 	Fields []LoginInputDataField `json:"fields"`
+
+	// Attachments to display alongside the input fields.
+	Attachments []*LoginUserInputAttachment `json:"attachments"`
+}
+
+type LoginUserInputAttachment struct {
+	Type     event.MessageType            `json:"type,omitempty"`
+	FileName string                       `json:"filename,omitempty"`
+	Content  []byte                       `json:"content,omitempty"`
+	Info     LoginUserInputAttachmentInfo `json:"info,omitempty"`
+}
+
+type LoginUserInputAttachmentInfo struct {
+	MimeType string `json:"mimetype,omitempty"`
+	Width    int    `json:"w,omitempty"`
+	Height   int    `json:"h,omitempty"`
+	Size     int    `json:"size,omitempty"`
 }
 
 type LoginCompleteParams struct {

bridgev2/matrix/analytics.go

@@ -0,0 +1,62 @@
+package matrix
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"maunium.net/go/mautrix/id"
+)
+
+func (br *Connector) trackSync(userID id.UserID, event string, properties map[string]any) error {
+	var buf bytes.Buffer
+	var analyticsUserID string
+	if br.Config.Analytics.UserID != "" {
+		analyticsUserID = br.Config.Analytics.UserID
+	} else {
+		analyticsUserID = userID.String()
+	}
+	err := json.NewEncoder(&buf).Encode(map[string]any{
+		"userId":     analyticsUserID,
+		"event":      event,
+		"properties": properties,
+	})
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, br.Config.Analytics.URL, &buf)
+	if err != nil {
+		return err
+	}
+	req.SetBasicAuth(br.Config.Analytics.Token, "")
+	resp, err := br.AS.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+	return nil
+}
+
+func (br *Connector) TrackAnalytics(userID id.UserID, event string, props map[string]any) {
+	if br.Config.Analytics.Token == "" || br.Config.Analytics.URL == "" {
+		return
+	}
+
+	if props == nil {
+		props = map[string]any{}
+	}
+	props["bridge"] = br.Bridge.Network.GetName().BeeperBridgeType
+	go func() {
+		err := br.trackSync(userID, event, props)
+		if err != nil {
+			br.Log.Err(err).Str("component", "analytics").Str("event", event).Msg("Error tracking event")
+		} else {
+			br.Log.Debug().Str("component", "analytics").Str("event", event).Msg("Tracked event")
+		}
+	}()
+}

bridgev2/matrix/connector.go

@@ -10,31 +10,34 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"regexp"
 	"strings"
 	"sync"
 	"time"
-	"unsafe"
 
-	"github.com/gorilla/mux"
 	_ "github.com/lib/pq"
 	"github.com/rs/zerolog"
 	"go.mau.fi/util/dbutil"
 	_ "go.mau.fi/util/dbutil/litestream"
+	"go.mau.fi/util/exbytes"
 	"go.mau.fi/util/exsync"
+	"go.mau.fi/util/ptr"
+	"go.mau.fi/util/random"
+	"golang.org/x/sync/semaphore"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/status"
 	"maunium.net/go/mautrix/bridgev2"
 	"maunium.net/go/mautrix/bridgev2/bridgeconfig"
 	"maunium.net/go/mautrix/bridgev2/commands"
+	"maunium.net/go/mautrix/bridgev2/database"
 	"maunium.net/go/mautrix/bridgev2/networkid"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 	"maunium.net/go/mautrix/mediaproxy"
@@ -67,7 +70,10 @@ type Connector struct {
 	Provisioning *ProvisioningAPI
 	DoublePuppet *doublePuppetUtil
 	MediaProxy   *mediaproxy.MediaProxy
-	dmaSigKey    [32]byte
+
+	uploadSema     *semaphore.Weighted
+	dmaSigKey      [32]byte
+	pubMediaSigKey []byte
 
 	doublePuppetIntents *exsync.Map[id.UserID, *appservice.IntentAPI]
 
@@ -75,6 +81,8 @@ type Connector struct {
 
 	MediaConfig             mautrix.RespMediaConfig
 	SpecVersions            *mautrix.RespVersions
+	SpecCaps                *mautrix.RespCapabilities
+	specCapsLock            sync.Mutex
 	Capabilities            *bridgev2.MatrixCapabilities
 	IgnoreUnsupportedServer bool
 
@@ -94,8 +102,14 @@ type Connector struct {
 }
 
 var (
-	_ bridgev2.MatrixConnector           = (*Connector)(nil)
-	_ bridgev2.MatrixConnectorWithServer = (*Connector)(nil)
+	_ bridgev2.MatrixConnector                           = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithServer                 = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithArbitraryRoomState     = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithPostRoomBridgeHandling = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithPublicMedia            = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithNameDisambiguation     = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithURLPreviews            = (*Connector)(nil)
+	_ bridgev2.MatrixConnectorWithAnalytics              = (*Connector)(nil)
 )
 
 func NewConnector(cfg *bridgeconfig.Config) *Connector {
@@ -103,6 +117,7 @@ func NewConnector(cfg *bridgeconfig.Config) *Connector {
 	c.Config = cfg
 	c.userIDRegex = cfg.MakeUserIDRegex("(.+)")
 	c.MediaConfig.UploadSize = 50 * 1024 * 1024
+	c.uploadSema = semaphore.NewWeighted(c.MediaConfig.UploadSize + 1)
 	c.Capabilities = &bridgev2.MatrixCapabilities{}
 	c.doublePuppetIntents = exsync.NewMap[id.UserID, *appservice.IntentAPI]()
 	return c
@@ -124,15 +139,25 @@ func (br *Connector) Init(bridge *bridgev2.Bridge) {
 	}
 	br.EventProcessor.On(event.EventMessage, br.handleRoomEvent)
 	br.EventProcessor.On(event.EventSticker, br.handleRoomEvent)
+	br.EventProcessor.On(event.EventUnstablePollStart, br.handleRoomEvent)
+	br.EventProcessor.On(event.EventUnstablePollResponse, br.handleRoomEvent)
 	br.EventProcessor.On(event.EventReaction, br.handleRoomEvent)
 	br.EventProcessor.On(event.EventRedaction, br.handleRoomEvent)
 	br.EventProcessor.On(event.EventEncrypted, br.handleEncryptedEvent)
+	br.EventProcessor.On(event.EphemeralEventEncrypted, br.handleEncryptedEvent)
 	br.EventProcessor.On(event.StateMember, br.handleRoomEvent)
+	br.EventProcessor.On(event.StatePowerLevels, br.handleRoomEvent)
 	br.EventProcessor.On(event.StateRoomName, br.handleRoomEvent)
+	br.EventProcessor.On(event.BeeperSendState, br.handleRoomEvent)
 	br.EventProcessor.On(event.StateRoomAvatar, br.handleRoomEvent)
 	br.EventProcessor.On(event.StateTopic, br.handleRoomEvent)
+	br.EventProcessor.On(event.StateTombstone, br.handleRoomEvent)
+	br.EventProcessor.On(event.StateBeeperDisappearingTimer, br.handleRoomEvent)
+	br.EventProcessor.On(event.BeeperDeleteChat, br.handleRoomEvent)
+	br.EventProcessor.On(event.BeeperAcceptMessageRequest, br.handleRoomEvent)
 	br.EventProcessor.On(event.EphemeralEventReceipt, br.handleEphemeralEvent)
 	br.EventProcessor.On(event.EphemeralEventTyping, br.handleEphemeralEvent)
+	br.EventProcessor.On(event.BeeperEphemeralEventAIStream, br.handleEphemeralEvent)
 	br.Bot = br.AS.BotIntent()
 	br.Crypto = NewCryptoHelper(br)
 	br.Bridge.Commands.(*commands.Processor).AddHandlers(
@@ -150,6 +175,21 @@ func (br *Connector) Start(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	err = br.initPublicMedia()
+	if err != nil {
+		return err
+	}
+	needsStateResync := br.Config.Encryption.Default &&
+		br.Bridge.DB.KV.Get(ctx, database.KeyEncryptionStateResynced) != "true"
+	if needsStateResync {
+		dbExists, err := br.StateStore.TableExists(ctx, "mx_version")
+		if err != nil {
+			return fmt.Errorf("failed to check if mx_version table exists: %w", err)
+		} else if !dbExists {
+			needsStateResync = false
+			br.Bridge.DB.KV.Set(ctx, database.KeyEncryptionStateResynced, "true")
+		}
+	}
 	err = br.StateStore.Upgrade(ctx)
 	if err != nil {
 		return bridgev2.DBUpgradeError{Section: "matrix_state", Err: err}
@@ -186,24 +226,66 @@ func (br *Connector) Start(ctx context.Context) error {
 	}
 	parsed, _ := url.Parse(br.Bridge.Network.GetName().NetworkURL)
 	if parsed != nil {
-		br.deterministicEventIDServer = parsed.Hostname()
+		br.deterministicEventIDServer = strings.TrimPrefix(parsed.Hostname(), "www.")
 	}
 	br.AS.Ready = true
 	if br.Websocket && br.Config.Homeserver.WSPingInterval > 0 {
 		br.wsStopPinger = make(chan struct{}, 1)
 		go br.websocketServerPinger()
 	}
+	if needsStateResync {
+		br.ResyncEncryptionState(ctx)
+	}
 	return nil
 }
 
+func (br *Connector) ResyncEncryptionState(ctx context.Context) {
+	log := zerolog.Ctx(ctx)
+	roomIDScanner := dbutil.ConvertRowFn[id.RoomID](dbutil.ScanSingleColumn[id.RoomID])
+	rooms, err := roomIDScanner.NewRowIter(br.Bridge.DB.Query(ctx, `
+		SELECT rooms.room_id
+		FROM (SELECT DISTINCT(room_id) FROM mx_user_profile WHERE room_id<>'') rooms
+		LEFT JOIN mx_room_state ON rooms.room_id = mx_room_state.room_id
+		WHERE mx_room_state.encryption IS NULL
+	`)).AsList()
+	if err != nil {
+		log.Err(err).Msg("Failed to get room list to resync state")
+		return
+	}
+	var failedCount, successCount, forbiddenCount int
+	for _, roomID := range rooms {
+		if roomID == "" {
+			continue
+		}
+		var outContent *event.EncryptionEventContent
+		err = br.Bot.Client.StateEvent(ctx, roomID, event.StateEncryption, "", &outContent)
+		if errors.Is(err, mautrix.MForbidden) {
+			// Most likely non-existent room
+			log.Debug().Err(err).Stringer("room_id", roomID).Msg("Failed to get state for room")
+			forbiddenCount++
+		} else if err != nil {
+			log.Err(err).Stringer("room_id", roomID).Msg("Failed to get state for room")
+			failedCount++
+		} else {
+			successCount++
+		}
+	}
+	br.Bridge.DB.KV.Set(ctx, database.KeyEncryptionStateResynced, "true")
+	log.Info().
+		Int("success_count", successCount).
+		Int("forbidden_count", forbiddenCount).
+		Int("failed_count", failedCount).
+		Msg("Resynced rooms")
+}
+
 func (br *Connector) GetPublicAddress() string {
 	if br.Config.AppService.PublicAddress == "https://bridge.example.com" {
 		return ""
 	}
-	return br.Config.AppService.PublicAddress
+	return strings.TrimRight(br.Config.AppService.PublicAddress, "/")
 }
 
-func (br *Connector) GetRouter() *mux.Router {
+func (br *Connector) GetRouter() *http.ServeMux {
 	if br.GetPublicAddress() != "" {
 		return br.AS.Router
 	}
@@ -214,27 +296,69 @@ func (br *Connector) GetCapabilities() *bridgev2.MatrixCapabilities {
 	return br.Capabilities
 }
 
-func (br *Connector) Stop() {
+func sendStopSignal(ch chan struct{}) {
+	if ch != nil {
+		select {
+		case ch <- struct{}{}:
+		default:
+		}
+	}
+}
+
+func (br *Connector) PreStop() {
 	br.stopping = true
 	br.AS.Stop()
+	if stopWebsocket := br.AS.StopWebsocket; stopWebsocket != nil {
+		stopWebsocket(appservice.ErrWebsocketManualStop)
+	}
+	sendStopSignal(br.wsStopPinger)
+	sendStopSignal(br.wsShortCircuitReconnectBackoff)
+}
+
+func (br *Connector) Stop() {
 	br.EventProcessor.Stop()
 	if br.Crypto != nil {
 		br.Crypto.Stop()
 	}
+	if wsStopChan := br.wsStopped; wsStopChan != nil {
+		select {
+		case <-wsStopChan:
+		case <-time.After(4 * time.Second):
+			br.Log.Warn().Msg("Timed out waiting for websocket to close")
+		}
+	}
 }
 
 var MinSpecVersion = mautrix.SpecV14
 
+func (br *Connector) logInitialRequestError(err error, defaultMessage string) {
+	if errors.Is(err, mautrix.MUnknownToken) {
+		br.Log.WithLevel(zerolog.FatalLevel).Msg("The as_token was not accepted. Is the registration file installed in your homeserver correctly?")
+		br.Log.Info().Msg("See https://docs.mau.fi/faq/as-token for more info")
+	} else if errors.Is(err, mautrix.MExclusive) {
+		br.Log.WithLevel(zerolog.FatalLevel).Msg("The as_token was accepted, but the /register request was not. Are the homeserver domain, bot username and username template in the config correct, and do they match the values in the registration?")
+		br.Log.Info().Msg("See https://docs.mau.fi/faq/as-register for more info")
+	} else {
+		br.Log.WithLevel(zerolog.FatalLevel).Err(err).Msg(defaultMessage)
+	}
+}
+
 func (br *Connector) ensureConnection(ctx context.Context) {
+	triedToRegister := false
 	for {
 		versions, err := br.Bot.Versions(ctx)
 		if err != nil {
-			if errors.Is(err, mautrix.MForbidden) {
+			if errors.Is(err, mautrix.MForbidden) && !triedToRegister {
 				br.Log.Debug().Msg("M_FORBIDDEN in /versions, trying to register before retrying")
 				err = br.Bot.EnsureRegistered(ctx)
 				if err != nil {
-					br.Log.Err(err).Msg("Failed to register after /versions failed")
+					br.logInitialRequestError(err, "Failed to register after /versions failed with M_FORBIDDEN")
+					os.Exit(16)
 				}
+				triedToRegister = true
+			} else if errors.Is(err, mautrix.MUnknownToken) || errors.Is(err, mautrix.MExclusive) {
+				br.logInitialRequestError(err, "/versions request failed with auth error")
+				os.Exit(16)
 			} else {
 				br.Log.Err(err).Msg("Failed to connect to homeserver, retrying in 10 seconds...")
 				time.Sleep(10 * time.Second)
@@ -244,6 +368,9 @@ func (br *Connector) ensureConnection(ctx context.Context) {
 			*br.AS.SpecVersions = *versions
 			br.Capabilities.AutoJoinInvites = br.SpecVersions.Supports(mautrix.BeeperFeatureAutojoinInvites)
 			br.Capabilities.BatchSending = br.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending)
+			br.Capabilities.ArbitraryMemberChange = br.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryMemberChange)
+			br.Capabilities.ExtraProfileMeta = br.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) ||
+				(br.SpecVersions.Supports(mautrix.FeatureArbitraryProfileFields) && br.Config.Matrix.GhostExtraProfileInfo)
 			break
 		}
 	}
@@ -267,15 +394,7 @@ func (br *Connector) ensureConnection(ctx context.Context) {
 
 	resp, err := br.Bot.Whoami(ctx)
 	if err != nil {
-		if errors.Is(err, mautrix.MUnknownToken) {
-			br.Log.WithLevel(zerolog.FatalLevel).Msg("The as_token was not accepted. Is the registration file installed in your homeserver correctly?")
-			br.Log.Info().Msg("See https://docs.mau.fi/faq/as-token for more info")
-		} else if errors.Is(err, mautrix.MExclusive) {
-			br.Log.WithLevel(zerolog.FatalLevel).Msg("The as_token was accepted, but the /register request was not. Are the homeserver domain, bot username and username template in the config correct, and do they match the values in the registration?")
-			br.Log.Info().Msg("See https://docs.mau.fi/faq/as-register for more info")
-		} else {
-			br.Log.WithLevel(zerolog.FatalLevel).Err(err).Msg("/whoami request failed with unknown error")
-		}
+		br.logInitialRequestError(err, "/whoami request failed with unknown error")
 		os.Exit(16)
 	} else if resp.UserID != br.Bot.UserID {
 		br.Log.WithLevel(zerolog.FatalLevel).
@@ -292,50 +411,23 @@ func (br *Connector) ensureConnection(ctx context.Context) {
 		br.Log.Debug().Msg("Homeserver does not support checking status of homeserver -> bridge connection")
 		return
 	}
-	var pingResp *mautrix.RespAppservicePing
-	var txnID string
-	var retryCount int
-	const maxRetries = 6
-	for {
-		txnID = br.Bot.TxnID()
-		pingResp, err = br.Bot.AppservicePing(ctx, br.Config.AppService.ID, txnID)
-		if err == nil {
-			break
-		}
-		var httpErr mautrix.HTTPError
-		var pingErrBody string
-		if errors.As(err, &httpErr) && httpErr.RespError != nil {
-			if val, ok := httpErr.RespError.ExtraData["body"].(string); ok {
-				pingErrBody = strings.TrimSpace(val)
-			}
-		}
-		outOfRetries := retryCount >= maxRetries
-		level := zerolog.ErrorLevel
-		if outOfRetries {
-			level = zerolog.FatalLevel
-		}
-		evt := br.Log.WithLevel(level).Err(err).Str("txn_id", txnID)
-		if pingErrBody != "" {
-			bodyBytes := []byte(pingErrBody)
-			if json.Valid(bodyBytes) {
-				evt.RawJSON("body", bodyBytes)
-			} else {
-				evt.Str("body", pingErrBody)
-			}
-		}
-		if outOfRetries {
-			evt.Msg("Homeserver -> bridge connection is not working")
-			br.Log.Info().Msg("See https://docs.mau.fi/faq/as-ping for more info")
-			os.Exit(13)
-		}
-		evt.Msg("Homeserver -> bridge connection is not working, retrying in 5 seconds...")
-		time.Sleep(5 * time.Second)
-		retryCount++
+
+	br.Bot.EnsureAppserviceConnection(ctx)
+}
+
+func (br *Connector) fetchCapabilities(ctx context.Context) *mautrix.RespCapabilities {
+	br.specCapsLock.Lock()
+	defer br.specCapsLock.Unlock()
+	if br.SpecCaps != nil {
+		return br.SpecCaps
 	}
-	br.Log.Debug().
-		Str("txn_id", txnID).
-		Int64("duration_ms", pingResp.DurationMS).
-		Msg("Homeserver -> bridge connection works")
+	caps, err := br.Bot.Capabilities(ctx)
+	if err != nil {
+		br.Log.Err(err).Msg("Failed to fetch capabilities from homeserver")
+		return nil
+	}
+	br.SpecCaps = caps
+	return caps
 }
 
 func (br *Connector) fetchMediaConfig(ctx context.Context) {
@@ -351,6 +443,7 @@ func (br *Connector) fetchMediaConfig(ctx context.Context) {
 		if ok {
 			mfsn.SetMaxFileSize(br.MediaConfig.UploadSize)
 		}
+		br.uploadSema = semaphore.NewWeighted(br.MediaConfig.UploadSize + 1)
 	}
 }
 
@@ -402,11 +495,15 @@ func (br *Connector) GhostIntent(userID networkid.UserID) bridgev2.MatrixAPI {
 func (br *Connector) SendBridgeStatus(ctx context.Context, state *status.BridgeState) error {
 	if br.Websocket {
 		br.hasSentAnyStates = true
-		return br.AS.SendWebsocket(&appservice.WebsocketRequest{
+		return br.AS.SendWebsocket(ctx, &appservice.WebsocketRequest{
 			Command: "bridge_status",
 			Data:    state,
 		})
 	} else if br.Config.Homeserver.StatusEndpoint != "" {
+		// Connecting states aren't really relevant unless the bridge runs somewhere with an unreliable network
+		if state.StateEvent == status.StateConnecting {
+			return nil
+		}
 		return state.SendHTTP(ctx, br.Config.Homeserver.StatusEndpoint, br.Config.AppService.ASToken)
 	} else {
 		return nil
@@ -414,50 +511,67 @@ func (br *Connector) SendBridgeStatus(ctx context.Context, state *status.BridgeS
 }
 
 func (br *Connector) SendMessageStatus(ctx context.Context, ms *bridgev2.MessageStatus, evt *bridgev2.MessageStatusEventInfo) {
-	br.internalSendMessageStatus(ctx, ms, evt, "")
+	go br.internalSendMessageStatus(ctx, ms, evt, "")
 }
 
 func (br *Connector) internalSendMessageStatus(ctx context.Context, ms *bridgev2.MessageStatus, evt *bridgev2.MessageStatusEventInfo, editEvent id.EventID) id.EventID {
-	if evt.EventType.IsEphemeral() {
+	if evt.EventType.IsEphemeral() || evt.SourceEventID == "" {
 		return ""
 	}
 	log := zerolog.Ctx(ctx)
-	err := br.SendMessageCheckpoints([]*status.MessageCheckpoint{ms.ToCheckpoint(evt)})
-	if err != nil {
-		log.Err(err).Msg("Failed to send message checkpoint")
-	}
-	if !ms.DisableMSS && br.Config.Matrix.MessageStatusEvents {
-		_, err = br.Bot.SendMessageEvent(ctx, evt.RoomID, event.BeeperMessageStatus, ms.ToMSSEvent(evt))
+
+	if !evt.IsSourceEventDoublePuppeted {
+		err := br.SendMessageCheckpoints(ctx, []*status.MessageCheckpoint{ms.ToCheckpoint(evt)})
 		if err != nil {
-			log.Err(err).Msg("Failed to send MSS event")
+			log.Err(err).Msg("Failed to send message checkpoint")
 		}
 	}
-	if ms.SendNotice && br.Config.Matrix.MessageErrorNotices && (ms.Status == event.MessageStatusFail || ms.Status == event.MessageStatusRetriable || ms.Step == status.MsgStepDecrypted) {
+
+	if !ms.DisableMSS && br.Config.Matrix.MessageStatusEvents {
+		mssEvt := ms.ToMSSEvent(evt)
+		_, err := br.Bot.SendMessageEvent(ctx, evt.RoomID, event.BeeperMessageStatus, mssEvt)
+		if err != nil {
+			log.Err(err).
+				Stringer("room_id", evt.RoomID).
+				Stringer("event_id", evt.SourceEventID).
+				Any("mss_content", mssEvt).
+				Msg("Failed to send MSS event")
+		}
+	}
+	if ms.SendNotice && br.Config.Matrix.MessageErrorNotices && evt.MessageType != event.MsgNotice &&
+		(ms.Status == event.MessageStatusFail || ms.Status == event.MessageStatusRetriable || ms.Step == status.MsgStepDecrypted) {
 		content := ms.ToNoticeEvent(evt)
 		if editEvent != "" {
 			content.SetEdit(editEvent)
 		}
 		resp, err := br.Bot.SendMessageEvent(ctx, evt.RoomID, event.EventMessage, content)
 		if err != nil {
-			log.Err(err).Msg("Failed to send notice event")
+			log.Err(err).
+				Stringer("room_id", evt.RoomID).
+				Stringer("event_id", evt.SourceEventID).
+				Str("notice_message", content.Body).
+				Msg("Failed to send notice event")
 		} else {
 			return resp.EventID
 		}
 	}
 	if ms.Status == event.MessageStatusSuccess && br.Config.Matrix.DeliveryReceipts {
-		err = br.Bot.SendReceipt(ctx, evt.RoomID, evt.EventID, event.ReceiptTypeRead, nil)
+		err := br.Bot.SendReceipt(ctx, evt.RoomID, evt.SourceEventID, event.ReceiptTypeRead, nil)
 		if err != nil {
-			log.Err(err).Msg("Failed to send Matrix delivery receipt")
+			log.Err(err).
+				Stringer("room_id", evt.RoomID).
+				Stringer("event_id", evt.SourceEventID).
+				Msg("Failed to send Matrix delivery receipt")
 		}
 	}
 	return ""
 }
 
-func (br *Connector) SendMessageCheckpoints(checkpoints []*status.MessageCheckpoint) error {
+func (br *Connector) SendMessageCheckpoints(ctx context.Context, checkpoints []*status.MessageCheckpoint) error {
 	checkpointsJSON := status.CheckpointsJSON{Checkpoints: checkpoints}
 
 	if br.Websocket {
-		return br.AS.SendWebsocket(&appservice.WebsocketRequest{
+		return br.AS.SendWebsocket(ctx, &appservice.WebsocketRequest{
 			Command: "message_checkpoint",
 			Data:    checkpointsJSON,
 		})
@@ -468,7 +582,7 @@ func (br *Connector) SendMessageCheckpoints(checkpoints []*status.MessageCheckpo
 		return nil
 	}
 
-	return checkpointsJSON.SendHTTP(endpoint, br.AS.Registration.AppToken)
+	return checkpointsJSON.SendHTTP(ctx, br.AS.HTTPClient, endpoint, br.AS.Registration.AppToken)
 }
 
 func (br *Connector) ParseGhostMXID(userID id.UserID) (networkid.UserID, bool) {
@@ -508,8 +622,38 @@ func (br *Connector) GetPowerLevels(ctx context.Context, roomID id.RoomID) (*eve
 	return br.Bot.PowerLevels(ctx, roomID)
 }
 
+func (br *Connector) GetStateEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string) (*event.Event, error) {
+	if stateKey == "" {
+		switch eventType {
+		case event.StateCreate:
+			createEvt, err := br.Bot.StateStore.GetCreate(ctx, roomID)
+			if err != nil || createEvt != nil {
+				return createEvt, err
+			}
+		case event.StateJoinRules:
+			joinRulesContent, err := br.Bot.StateStore.GetJoinRules(ctx, roomID)
+			if err != nil {
+				return nil, err
+			} else if joinRulesContent != nil {
+				return &event.Event{
+					Type:     event.StateJoinRules,
+					RoomID:   roomID,
+					StateKey: ptr.Ptr(""),
+					Content:  event.Content{Parsed: joinRulesContent},
+				}, nil
+			}
+		}
+	}
+	return br.Bot.FullStateEvent(ctx, roomID, eventType, stateKey)
+}
+
 func (br *Connector) GetMembers(ctx context.Context, roomID id.RoomID) (map[id.UserID]*event.MemberEventContent, error) {
-	// TODO use cache?
+	fetched, err := br.Bot.StateStore.HasFetchedMembers(ctx, roomID)
+	if err != nil {
+		return nil, err
+	} else if fetched {
+		return br.Bot.StateStore.GetAllMembers(ctx, roomID)
+	}
 	members, err := br.Bot.Members(ctx, roomID)
 	if err != nil {
 		return nil, err
@@ -526,29 +670,44 @@ func (br *Connector) GetMemberInfo(ctx context.Context, roomID id.RoomID, userID
 	return br.AS.StateStore.GetMember(ctx, roomID, userID)
 }
 
-func (br *Connector) BatchSend(ctx context.Context, roomID id.RoomID, req *mautrix.ReqBeeperBatchSend) (*mautrix.RespBeeperBatchSend, error) {
+func (br *Connector) IsConfusableName(ctx context.Context, roomID id.RoomID, userID id.UserID, name string) ([]id.UserID, error) {
+	return br.AS.StateStore.IsConfusableName(ctx, roomID, userID, name)
+}
+
+func (br *Connector) GetUniqueBridgeID() string {
+	return fmt.Sprintf("%s/%s", br.Config.Homeserver.Domain, br.Config.AppService.ID)
+}
+
+func (br *Connector) BatchSend(ctx context.Context, roomID id.RoomID, req *mautrix.ReqBeeperBatchSend, extras []*bridgev2.MatrixSendExtra) (*mautrix.RespBeeperBatchSend, error) {
 	if encrypted, err := br.StateStore.IsEncrypted(ctx, roomID); err != nil {
 		return nil, fmt.Errorf("failed to check if room is encrypted: %w", err)
 	} else if encrypted {
 		for _, evt := range req.Events {
 			intent, _ := br.doublePuppetIntents.Get(evt.Sender)
 			if intent != nil {
-				intent.AddDoublePuppetValueWithTS(evt.ID, evt.Timestamp)
+				intent.AddDoublePuppetValueWithTS(&evt.Content, evt.Timestamp)
 			}
-			err = br.Crypto.Encrypt(ctx, roomID, evt.Type, &evt.Content)
-			if err != nil {
-				return nil, err
+			if evt.Type != event.EventEncrypted && evt.Type != event.EventReaction {
+				err = br.Crypto.Encrypt(ctx, roomID, evt.Type, &evt.Content)
+				if err != nil {
+					return nil, err
+				}
+				evt.Type = event.EventEncrypted
+				if intent != nil {
+					intent.AddDoublePuppetValueWithTS(&evt.Content, evt.Timestamp)
+				}
 			}
-			evt.Type = event.EventEncrypted
 		}
 	}
 	return br.Bot.BeeperBatchSend(ctx, roomID, req)
 }
 
 func (br *Connector) GenerateDeterministicEventID(roomID id.RoomID, _ networkid.PortalKey, messageID networkid.MessageID, partID networkid.PartID) id.EventID {
-	data := make([]byte, 0, len(roomID)+len(messageID)+len(partID))
+	data := make([]byte, 0, len(roomID)+1+len(messageID)+1+len(partID))
 	data = append(data, roomID...)
+	data = append(data, 0)
 	data = append(data, messageID...)
+	data = append(data, 0)
 	data = append(data, partID...)
 
 	hash := sha256.Sum256(data)
@@ -560,9 +719,40 @@ func (br *Connector) GenerateDeterministicEventID(roomID id.RoomID, _ networkid.
 	eventID[1+hashB64Len] = ':'
 	copy(eventID[1+hashB64Len+1:], br.deterministicEventIDServer)
 
-	return id.EventID(unsafe.String(unsafe.SliceData(eventID), len(eventID)))
+	return id.EventID(exbytes.UnsafeString(eventID))
+}
+
+func (br *Connector) GenerateDeterministicRoomID(key networkid.PortalKey) id.RoomID {
+	return id.RoomID(fmt.Sprintf("!%s.%s:%s", key.ID, key.Receiver, br.ServerName()))
+}
+
+func (br *Connector) GenerateReactionEventID(roomID id.RoomID, targetMessage *database.Message, sender networkid.UserID, emojiID networkid.EmojiID) id.EventID {
+	// We don't care about determinism for reactions
+	return id.EventID(fmt.Sprintf("$%s:%s", base64.RawURLEncoding.EncodeToString(random.Bytes(32)), br.deterministicEventIDServer))
 }
 
 func (br *Connector) ServerName() string {
 	return br.Config.Homeserver.Domain
 }
+
+func (br *Connector) HandleNewlyBridgedRoom(ctx context.Context, roomID id.RoomID) error {
+	_, err := br.Bot.Members(ctx, roomID)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to fetch members in newly bridged room")
+	}
+	if !br.Config.Encryption.Default {
+		return nil
+	}
+	_, err = br.Bot.SendStateEvent(ctx, roomID, event.StateEncryption, "", &event.Content{
+		Parsed: br.getDefaultEncryptionEvent(),
+	})
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to enable encryption in newly bridged room")
+		return fmt.Errorf("failed to enable encryption")
+	}
+	return nil
+}
+
+func (br *Connector) GetURLPreview(ctx context.Context, url string) (*event.LinkPreview, error) {
+	return br.Bot.GetURLPreview(ctx, url)
+}

bridgev2/matrix/crypto.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"os"
 	"runtime/debug"
+	"strings"
 	"sync"
 	"time"
 
@@ -23,6 +24,7 @@ import (
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/database"
 	"maunium.net/go/mautrix/crypto"
 	"maunium.net/go/mautrix/crypto/olm"
 	"maunium.net/go/mautrix/event"
@@ -36,9 +38,9 @@ func init() {
 
 var _ crypto.StateStore = (*sqlstatestore.SQLStateStore)(nil)
 
-var NoSessionFound = crypto.NoSessionFound
-var DuplicateMessageIndex = crypto.DuplicateMessageIndex
-var UnknownMessageIndex = olm.UnknownMessageIndex
+var NoSessionFound = crypto.ErrNoSessionFound
+var DuplicateMessageIndex = crypto.ErrDuplicateMessageIndex
+var UnknownMessageIndex = olm.ErrUnknownMessageIndex
 
 type CryptoHelper struct {
 	bridge *Connector
@@ -77,7 +79,7 @@ func (helper *CryptoHelper) Init(ctx context.Context) error {
 		dbutil.ZeroLogger(helper.bridge.Log.With().Str("db_section", "crypto").Logger()),
 		string(helper.bridge.Bridge.ID),
 		helper.bridge.AS.BotMXID(),
-		fmt.Sprintf("@%s:%s", helper.bridge.Config.AppService.FormatUsername("%"), helper.bridge.AS.HomeserverDomain),
+		fmt.Sprintf("@%s:%s", strings.ReplaceAll(helper.bridge.Config.AppService.FormatUsername("%"), "_", `\_`), helper.bridge.AS.HomeserverDomain),
 		helper.bridge.Config.Encryption.PickleKey,
 	)
 
@@ -96,6 +98,7 @@ func (helper *CryptoHelper) Init(ctx context.Context) error {
 		Str("device_id", helper.client.DeviceID.String()).
 		Msg("Logged in as bridge bot")
 	helper.mach = crypto.NewOlmMachine(helper.client, helper.log, helper.store, helper.bridge.StateStore)
+	helper.mach.DisableSharedGroupSessionTracking = true
 	helper.mach.AllowKeyShare = helper.allowKeyShare
 
 	encryptionConfig := helper.bridge.Config.Encryption
@@ -133,7 +136,19 @@ func (helper *CryptoHelper) Init(ctx context.Context) error {
 		return err
 	}
 	if isExistingDevice {
-		helper.verifyKeysAreOnServer(ctx)
+		if !helper.verifyKeysAreOnServer(ctx) {
+			return nil
+		}
+	} else {
+		err = helper.ShareKeys(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to share device keys: %w", err)
+		}
+	}
+	if helper.bridge.Config.Encryption.SelfSign {
+		if !helper.doSelfSign(ctx) {
+			os.Exit(34)
+		}
 	}
 
 	go helper.resyncEncryptionInfo(context.TODO())
@@ -141,30 +156,66 @@ func (helper *CryptoHelper) Init(ctx context.Context) error {
 	return nil
 }
 
+func (helper *CryptoHelper) doSelfSign(ctx context.Context) bool {
+	log := zerolog.Ctx(ctx)
+	hasKeys, isVerified, err := helper.mach.GetOwnVerificationStatus(ctx)
+	if err != nil {
+		log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to check verification status")
+		return false
+	}
+	log.Debug().Bool("has_keys", hasKeys).Bool("is_verified", isVerified).Msg("Checked verification status")
+	keyInDB := helper.bridge.Bridge.DB.KV.Get(ctx, database.KeyRecoveryKey)
+	if !hasKeys || keyInDB == "overwrite" {
+		if keyInDB != "" && keyInDB != "overwrite" {
+			log.WithLevel(zerolog.FatalLevel).
+				Msg("No keys on server, but database already has recovery key. Delete `recovery_key` from `kv_store` manually to continue.")
+			return false
+		}
+		recoveryKey, err := helper.mach.GenerateAndVerifyWithRecoveryKey(ctx)
+		if recoveryKey != "" {
+			helper.bridge.Bridge.DB.KV.Set(ctx, database.KeyRecoveryKey, recoveryKey)
+		}
+		if err != nil {
+			log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to generate recovery key and self-sign")
+			return false
+		}
+		log.Info().Msg("Generated new recovery key and self-signed bot device")
+	} else if !isVerified {
+		if keyInDB == "" {
+			log.WithLevel(zerolog.FatalLevel).
+				Msg("Server already has cross-signing keys, but no key in database. Add `recovery_key` to `kv_store`, or set it to `overwrite` to generate new keys.")
+			return false
+		}
+		err = helper.mach.VerifyWithRecoveryKey(ctx, keyInDB)
+		if err != nil {
+			log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to verify with recovery key")
+			return false
+		}
+		log.Info().Msg("Verified bot device with existing recovery key")
+	}
+	return true
+}
+
 func (helper *CryptoHelper) resyncEncryptionInfo(ctx context.Context) {
 	log := helper.log.With().Str("action", "resync encryption event").Logger()
 	rows, err := helper.store.DB.Query(ctx, `SELECT room_id FROM mx_room_state WHERE encryption='{"resync":true}'`)
+	roomIDs, err := dbutil.NewRowIterWithError(rows, dbutil.ScanSingleColumn[id.RoomID], err).AsList()
 	if err != nil {
 		log.Err(err).Msg("Failed to query rooms for resync")
 		return
 	}
-	roomIDs, err := dbutil.NewRowIter(rows, dbutil.ScanSingleColumn[id.RoomID]).AsList()
-	if err != nil {
-		log.Err(err).Msg("Failed to scan rooms for resync")
-		return
-	}
 	if len(roomIDs) > 0 {
 		log.Debug().Interface("room_ids", roomIDs).Msg("Resyncing rooms")
 		for _, roomID := range roomIDs {
 			var evt event.EncryptionEventContent
 			err = helper.client.StateEvent(ctx, roomID, event.StateEncryption, "", &evt)
 			if err != nil {
-				log.Err(err).Str("room_id", roomID.String()).Msg("Failed to get encryption event")
+				log.Err(err).Stringer("room_id", roomID).Msg("Failed to get encryption event")
 				_, err = helper.store.DB.Exec(ctx, `
 					UPDATE mx_room_state SET encryption=NULL WHERE room_id=$1 AND encryption='{"resync":true}'
 				`, roomID)
 				if err != nil {
-					log.Err(err).Str("room_id", roomID.String()).Msg("Failed to unmark room for resync after failed sync")
+					log.Err(err).Stringer("room_id", roomID).Msg("Failed to unmark room for resync after failed sync")
 				}
 			} else {
 				maxAge := evt.RotationPeriodMillis
@@ -187,9 +238,9 @@ func (helper *CryptoHelper) resyncEncryptionInfo(ctx context.Context) {
 					WHERE room_id=$3 AND max_age IS NULL AND max_messages IS NULL
 				`, maxAge, maxMessages, roomID)
 				if err != nil {
-					log.Err(err).Str("room_id", roomID.String()).Msg("Failed to update megolm session table")
+					log.Err(err).Stringer("room_id", roomID).Msg("Failed to update megolm session table")
 				} else {
-					log.Debug().Str("room_id", roomID.String()).Msg("Updated megolm session table")
+					log.Debug().Stringer("room_id", roomID).Msg("Updated megolm session table")
 				}
 			}
 		}
@@ -202,7 +253,7 @@ func (helper *CryptoHelper) allowKeyShare(ctx context.Context, device *id.Device
 		return &crypto.KeyShareRejectNoResponse
 	} else if device.Trust == id.TrustStateBlacklisted {
 		return &crypto.KeyShareRejectBlacklisted
-	} else if trustState := helper.mach.ResolveTrust(device); trustState >= cfg.VerificationLevels.Share {
+	} else if trustState, _ := helper.mach.ResolveTrustContext(ctx, device); trustState >= cfg.VerificationLevels.Share {
 		portal, err := helper.bridge.Bridge.GetPortalByMXID(ctx, info.RoomID)
 		if err != nil {
 			zerolog.Ctx(ctx).Err(err).Msg("Failed to get portal to handle key request")
@@ -216,11 +267,12 @@ func (helper *CryptoHelper) allowKeyShare(ctx context.Context, device *id.Device
 			zerolog.Ctx(ctx).Err(err).Msg("Failed to get user to handle key request")
 			return &crypto.KeyShareRejectNoResponse
 		} else if user == nil {
-			// TODO
+			zerolog.Ctx(ctx).Debug().Msg("Couldn't find user to handle key request")
 			return &crypto.KeyShareRejectNoResponse
-		} else if true {
-			// TODO admin check and is in room check
-			return &crypto.KeyShareRejection{Code: event.RoomKeyWithheldUnauthorized, Reason: "Key sharing is not yet implemented in bridgev2"}
+		} else if !user.Permissions.Admin {
+			zerolog.Ctx(ctx).Debug().Msg("Rejecting key request: user is not admin")
+			// TODO is in room check?
+			return &crypto.KeyShareRejection{Code: event.RoomKeyWithheldUnauthorized, Reason: "Key sharing for non-admins is not yet implemented"}
 		}
 		zerolog.Ctx(ctx).Debug().Msg("Accepting key request")
 		return nil
@@ -234,28 +286,39 @@ func (helper *CryptoHelper) loginBot(ctx context.Context) (*mautrix.Client, bool
 	if err != nil {
 		return nil, false, fmt.Errorf("failed to find existing device ID: %w", err)
 	} else if len(deviceID) > 0 {
-		helper.log.Debug().Str("device_id", deviceID.String()).Msg("Found existing device ID for bot in database")
+		helper.log.Debug().Stringer("device_id", deviceID).Msg("Found existing device ID for bot in database")
 	}
 	// Create a new client instance with the default AS settings (including as_token),
 	// the Login call will then override the access token in the client.
 	client := helper.bridge.AS.NewMautrixClient(helper.bridge.AS.BotMXID())
+
+	initialDeviceDisplayName := fmt.Sprintf("%s bridge", helper.bridge.Bridge.Network.GetName().DisplayName)
+	if helper.bridge.Config.Encryption.MSC4190 {
+		helper.log.Debug().Msg("Creating bot device with MSC4190")
+		err = client.CreateDeviceMSC4190(ctx, deviceID, initialDeviceDisplayName)
+		if err != nil {
+			return nil, deviceID != "", fmt.Errorf("failed to create device for bridge bot: %w", err)
+		}
+		helper.store.DeviceID = client.DeviceID
+		return client, deviceID != "", nil
+	}
+
 	flows, err := client.GetLoginFlows(ctx)
 	if err != nil {
 		return nil, deviceID != "", fmt.Errorf("failed to get supported login flows: %w", err)
 	} else if !flows.HasFlow(mautrix.AuthTypeAppservice) {
 		return nil, deviceID != "", fmt.Errorf("homeserver does not support appservice login")
 	}
+
 	resp, err := client.Login(ctx, &mautrix.ReqLogin{
 		Type: mautrix.AuthTypeAppservice,
 		Identifier: mautrix.UserIdentifier{
 			Type: mautrix.IdentifierTypeUser,
 			User: string(helper.bridge.AS.BotMXID()),
 		},
-		DeviceID:         deviceID,
-		StoreCredentials: true,
-
-		// TODO find proper bridge name
-		InitialDeviceDisplayName: "Megabridge", // fmt.Sprintf("%s bridge", helper.bridge.ProtocolName),
+		DeviceID:                 deviceID,
+		StoreCredentials:         true,
+		InitialDeviceDisplayName: initialDeviceDisplayName,
 	})
 	if err != nil {
 		return nil, deviceID != "", fmt.Errorf("failed to log in as bridge bot: %w", err)
@@ -264,7 +327,7 @@ func (helper *CryptoHelper) loginBot(ctx context.Context) (*mautrix.Client, bool
 	return client, deviceID != "", nil
 }
 
-func (helper *CryptoHelper) verifyKeysAreOnServer(ctx context.Context) {
+func (helper *CryptoHelper) verifyKeysAreOnServer(ctx context.Context) bool {
 	helper.log.Debug().Msg("Making sure keys are still on server")
 	resp, err := helper.client.QueryKeys(ctx, &mautrix.ReqQueryKeys{
 		DeviceKeys: map[id.UserID]mautrix.DeviceIDList{
@@ -277,10 +340,11 @@ func (helper *CryptoHelper) verifyKeysAreOnServer(ctx context.Context) {
 	}
 	device, ok := resp.DeviceKeys[helper.client.UserID][helper.client.DeviceID]
 	if ok && len(device.Keys) > 0 {
-		return
+		return true
 	}
 	helper.log.Warn().Msg("Existing device doesn't have keys on server, resetting crypto")
 	helper.Reset(ctx, false)
+	return false
 }
 
 func (helper *CryptoHelper) Start() {
@@ -375,7 +439,7 @@ func (helper *CryptoHelper) Encrypt(ctx context.Context, roomID id.RoomID, evtTy
 	var encrypted *event.EncryptedEventContent
 	encrypted, err = helper.mach.EncryptMegolmEvent(ctx, roomID, evtType, content)
 	if err != nil {
-		if !errors.Is(err, crypto.SessionExpired) && !errors.Is(err, crypto.SessionNotShared) && !errors.Is(err, crypto.NoGroupSession) {
+		if !errors.Is(err, crypto.ErrSessionExpired) && !errors.Is(err, crypto.ErrSessionNotShared) && !errors.Is(err, crypto.ErrNoGroupSession) {
 			return
 		}
 		helper.log.Debug().Err(err).
@@ -490,14 +554,14 @@ func (syncer *cryptoSyncer) OnFailedSync(_ *mautrix.RespSync, err error) (time.D
 func (syncer *cryptoSyncer) GetFilterJSON(_ id.UserID) *mautrix.Filter {
 	everything := []event.Type{{Type: "*"}}
 	return &mautrix.Filter{
-		Presence:    mautrix.FilterPart{NotTypes: everything},
-		AccountData: mautrix.FilterPart{NotTypes: everything},
-		Room: mautrix.RoomFilter{
+		Presence:    &mautrix.FilterPart{NotTypes: everything},
+		AccountData: &mautrix.FilterPart{NotTypes: everything},
+		Room: &mautrix.RoomFilter{
 			IncludeLeave: false,
-			Ephemeral:    mautrix.FilterPart{NotTypes: everything},
-			AccountData:  mautrix.FilterPart{NotTypes: everything},
-			State:        mautrix.FilterPart{NotTypes: everything},
-			Timeline:     mautrix.FilterPart{NotTypes: everything},
+			Ephemeral:    &mautrix.FilterPart{NotTypes: everything},
+			AccountData:  &mautrix.FilterPart{NotTypes: everything},
+			State:        &mautrix.FilterPart{NotTypes: everything},
+			Timeline:     &mautrix.FilterPart{NotTypes: everything},
 		},
 	}
 }

bridgev2/matrix/cryptoerror.go

@@ -11,8 +11,8 @@ import (
 	"errors"
 	"fmt"
 
-	"maunium.net/go/mautrix/bridge/status"
 	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )

bridgev2/matrix/cryptostore.go

@@ -45,7 +45,7 @@ func (store *SQLCryptoStore) GetRoomJoinedOrInvitedMembers(ctx context.Context,
 		WHERE room_id=$1
 			AND (membership='join' OR membership='invite')
 			AND user_id<>$2
-			AND user_id NOT LIKE $3
+			AND user_id NOT LIKE $3 ESCAPE '\'
 	`, roomID, store.UserID, store.GhostIDFormat)
 	if err != nil {
 		return

bridgev2/matrix/directmedia.go

@@ -13,7 +13,6 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
-	"net/http"
 	"strings"
 
 	"maunium.net/go/mautrix"
@@ -40,7 +39,7 @@ func (br *Connector) initDirectMedia() error {
 	if err != nil {
 		return fmt.Errorf("failed to initialize media proxy: %w", err)
 	}
-	br.MediaProxy.RegisterRoutes(br.AS.Router)
+	br.MediaProxy.RegisterRoutes(br.AS.Router, br.Log.With().Str("component", "media proxy").Logger())
 	br.dmaSigKey = sha256.Sum256(br.MediaProxy.GetServerKey().Priv.Seed())
 	dmn.SetUseDirectMedia()
 	br.Log.Debug().Str("server_name", br.MediaProxy.GetServerName()).Msg("Enabled direct media access")
@@ -72,7 +71,7 @@ func (br *Connector) GenerateContentURI(ctx context.Context, mediaID networkid.M
 	return mxc, nil
 }
 
-func (br *Connector) getDirectMedia(ctx context.Context, mediaIDStr string) (response mediaproxy.GetMediaResponse, err error) {
+func (br *Connector) getDirectMedia(ctx context.Context, mediaIDStr string, params map[string]string) (response mediaproxy.GetMediaResponse, err error) {
 	mediaID, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(mediaIDStr, br.Config.DirectMedia.MediaIDPrefix))
 	if err != nil || !bytes.HasPrefix(mediaID, []byte(MediaIDPrefix)) || len(mediaID) < len(MediaIDPrefix)+MediaIDTruncatedHashLength+1 {
 		return nil, mediaproxy.ErrInvalidMediaIDSyntax
@@ -80,14 +79,8 @@ func (br *Connector) getDirectMedia(ctx context.Context, mediaIDStr string) (res
 	receivedHash := mediaID[len(mediaID)-MediaIDTruncatedHashLength:]
 	expectedHash := br.hashMediaID(mediaID[:len(mediaID)-MediaIDTruncatedHashLength])
 	if !hmac.Equal(receivedHash, expectedHash) {
-		return nil, &mediaproxy.ResponseError{
-			Status: http.StatusNotFound,
-			Data: &mautrix.RespError{
-				ErrCode: mautrix.MNotFound.ErrCode,
-				Err:     "Invalid checksum in media ID part",
-			},
-		}
+		return nil, mautrix.MNotFound.WithMessage("Invalid checksum in media ID part")
 	}
 	remoteMediaID := networkid.MediaID(mediaID[len(MediaIDPrefix) : len(mediaID)-MediaIDTruncatedHashLength])
-	return br.Bridge.Network.(bridgev2.DirectMediableNetwork).Download(ctx, remoteMediaID)
+	return br.Bridge.Network.(bridgev2.DirectMediableNetwork).Download(ctx, remoteMediaID, params)
 }

bridgev2/matrix/intent.go

@@ -7,17 +7,28 @@
 package matrix
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.mau.fi/util/fallocate"
+	"go.mau.fi/util/ptr"
+	"golang.org/x/exp/slices"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/bridgeconfig"
 	"maunium.net/go/mautrix/crypto/attachment"
+	"maunium.net/go/mautrix/crypto/canonicaljson"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 	"maunium.net/go/mautrix/pushrules"
@@ -27,25 +38,44 @@ import (
 type ASIntent struct {
 	Matrix    *appservice.IntentAPI
 	Connector *Connector
+
+	dmUpdateLock     sync.Mutex
+	directChatsCache event.DirectChatsEventContent
 }
 
 var _ bridgev2.MatrixAPI = (*ASIntent)(nil)
+var _ bridgev2.MarkAsDMMatrixAPI = (*ASIntent)(nil)
+var _ bridgev2.EphemeralSendingMatrixAPI = (*ASIntent)(nil)
 
-func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType event.Type, content *event.Content, ts time.Time) (*mautrix.RespSendEvent, error) {
-	// TODO remove this once hungryserv and synapse support sending m.room.redactions directly in all room versions
-	if eventType == event.EventRedaction {
+func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType event.Type, content *event.Content, extra *bridgev2.MatrixSendExtra) (*mautrix.RespSendEvent, error) {
+	if extra == nil {
+		extra = &bridgev2.MatrixSendExtra{}
+	}
+	if eventType == event.EventRedaction && !as.Connector.SpecVersions.Supports(mautrix.FeatureRedactSendAsEvent) {
 		parsedContent := content.Parsed.(*event.RedactionEventContent)
+		as.Matrix.AddDoublePuppetValue(content)
 		return as.Matrix.RedactEvent(ctx, roomID, parsedContent.Redacts, mautrix.ReqRedact{
 			Reason: parsedContent.Reason,
 			Extra:  content.Raw,
 		})
 	}
-	if eventType != event.EventReaction && eventType != event.EventRedaction {
+	if (eventType != event.EventReaction || as.Connector.Config.Encryption.MSC4392) && eventType != event.EventRedaction {
+		msgContent, ok := content.Parsed.(*event.MessageEventContent)
+		if ok {
+			msgContent.AddPerMessageProfileFallback()
+		}
 		if encrypted, err := as.Matrix.StateStore.IsEncrypted(ctx, roomID); err != nil {
 			return nil, fmt.Errorf("failed to check if room is encrypted: %w", err)
 		} else if encrypted {
+			if as.Connector.Crypto == nil {
+				return nil, fmt.Errorf("room is encrypted, but bridge isn't configured to support encryption")
+			}
 			if as.Matrix.IsCustomPuppet {
-				as.Matrix.AddDoublePuppetValueWithTS(content, ts.UnixMilli())
+				if extra.Timestamp.IsZero() {
+					as.Matrix.AddDoublePuppetValue(content)
+				} else {
+					as.Matrix.AddDoublePuppetValueWithTS(content, extra.Timestamp.UnixMilli())
+				}
 			}
 			err = as.Connector.Crypto.Encrypt(ctx, roomID, eventType, content)
 			if err != nil {
@@ -54,16 +84,27 @@ func (as *ASIntent) SendMessage(ctx context.Context, roomID id.RoomID, eventType
 			eventType = event.EventEncrypted
 		}
 	}
-	if ts.IsZero() {
-		return as.Matrix.SendMessageEvent(ctx, roomID, eventType, content)
-	} else {
-		return as.Matrix.SendMassagedMessageEvent(ctx, roomID, eventType, content, ts.UnixMilli())
+	return as.Matrix.SendMessageEvent(ctx, roomID, eventType, content, mautrix.ReqSendEvent{Timestamp: extra.Timestamp.UnixMilli()})
+}
+
+func (as *ASIntent) BeeperSendEphemeralEvent(ctx context.Context, roomID id.RoomID, eventType event.Type, content *event.Content, txnID string) (*mautrix.RespSendEvent, error) {
+	if !as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureEphemeralEvents) {
+		return nil, mautrix.MUnrecognized.WithMessage("Homeserver does not advertise com.beeper.ephemeral support")
 	}
+	if encrypted, err := as.Matrix.StateStore.IsEncrypted(ctx, roomID); err != nil {
+		return nil, fmt.Errorf("failed to check if room is encrypted: %w", err)
+	} else if encrypted && as.Connector.Crypto != nil {
+		if err = as.Connector.Crypto.Encrypt(ctx, roomID, eventType, content); err != nil {
+			return nil, err
+		}
+		eventType = event.EventEncrypted
+	}
+	return as.Matrix.BeeperSendEphemeralEvent(ctx, roomID, eventType, content, mautrix.ReqSendEvent{TransactionID: txnID})
 }
 
 func (as *ASIntent) fillMemberEvent(ctx context.Context, roomID id.RoomID, userID id.UserID, content *event.Content) {
-	targetContent := content.Parsed.(*event.MemberEventContent)
-	if targetContent.Displayname != "" || targetContent.AvatarURL != "" {
+	targetContent, ok := content.Parsed.(*event.MemberEventContent)
+	if !ok || targetContent.Displayname != "" || targetContent.AvatarURL != "" {
 		return
 	}
 	memberContent, err := as.Matrix.StateStore.TryGetMember(ctx, roomID, userID)
@@ -94,18 +135,22 @@ func (as *ASIntent) fillMemberEvent(ctx context.Context, roomID id.RoomID, userI
 	}
 }
 
-func (as *ASIntent) SendState(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, content *event.Content, ts time.Time) (*mautrix.RespSendEvent, error) {
+func (as *ASIntent) SendState(ctx context.Context, roomID id.RoomID, eventType event.Type, stateKey string, content *event.Content, ts time.Time) (resp *mautrix.RespSendEvent, err error) {
 	if eventType == event.StateMember {
 		as.fillMemberEvent(ctx, roomID, id.UserID(stateKey), content)
 	}
-	if ts.IsZero() {
-		return as.Matrix.SendStateEvent(ctx, roomID, eventType, stateKey, content)
-	} else {
-		return as.Matrix.SendMassagedStateEvent(ctx, roomID, eventType, stateKey, content, ts.UnixMilli())
+	resp, err = as.Matrix.SendStateEvent(ctx, roomID, eventType, stateKey, content, mautrix.ReqSendEvent{Timestamp: ts.UnixMilli()})
+	if err != nil && eventType == event.StateMember {
+		var httpErr mautrix.HTTPError
+		if errors.As(err, &httpErr) && httpErr.RespError != nil &&
+			(strings.Contains(httpErr.RespError.Err, "is already in the room") || strings.Contains(httpErr.RespError.Err, "is already joined to room")) {
+			err = as.Matrix.StateStore.SetMembership(ctx, roomID, id.UserID(stateKey), event.MembershipJoin)
+		}
 	}
+	return resp, err
 }
 
-func (as *ASIntent) MarkRead(ctx context.Context, roomID id.RoomID, eventID id.EventID, ts time.Time) error {
+func (as *ASIntent) MarkRead(ctx context.Context, roomID id.RoomID, eventID id.EventID, ts time.Time) (err error) {
 	extraData := map[string]any{}
 	if !ts.IsZero() {
 		extraData["ts"] = ts.UnixMilli()
@@ -119,25 +164,35 @@ func (as *ASIntent) MarkRead(ctx context.Context, roomID id.RoomID, eventID id.E
 		req.FullyRead = eventID
 		req.BeeperFullyReadExtra = extraData
 	}
-	err := as.Matrix.SetReadMarkers(ctx, roomID, &req)
-	if err != nil {
-		return err
-	}
-	if as.Matrix.IsCustomPuppet {
-		err = as.Matrix.SetRoomAccountData(ctx, roomID, event.AccountDataMarkedUnread.Type, &event.MarkedUnreadEventContent{
-			Unread: false,
+	if as.Matrix.IsCustomPuppet && as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureInboxState) && as.Connector.Config.Homeserver.Software != bridgeconfig.SoftwareHungry {
+		err = as.Matrix.SetBeeperInboxState(ctx, roomID, &mautrix.ReqSetBeeperInboxState{
+			//MarkedUnread: ptr.Ptr(false),
+			ReadMarkers: &req,
 		})
-		if err != nil {
-			return err
+	} else {
+		err = as.Matrix.SetReadMarkers(ctx, roomID, &req)
+		if err == nil && as.Matrix.IsCustomPuppet && as.Connector.Config.Homeserver.Software != bridgeconfig.SoftwareHungry {
+			err = as.Matrix.SetRoomAccountData(ctx, roomID, event.AccountDataMarkedUnread.Type, &event.MarkedUnreadEventContent{
+				Unread: false,
+			})
 		}
 	}
-	return nil
+	return
 }
 
 func (as *ASIntent) MarkUnread(ctx context.Context, roomID id.RoomID, unread bool) error {
-	return as.Matrix.SetRoomAccountData(ctx, roomID, event.AccountDataMarkedUnread.Type, &event.MarkedUnreadEventContent{
-		Unread: unread,
-	})
+	if as.Connector.Config.Homeserver.Software == bridgeconfig.SoftwareHungry {
+		return nil
+	}
+	if as.Matrix.IsCustomPuppet && as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureInboxState) {
+		return as.Matrix.SetBeeperInboxState(ctx, roomID, &mautrix.ReqSetBeeperInboxState{
+			MarkedUnread: ptr.Ptr(unread),
+		})
+	} else {
+		return as.Matrix.SetRoomAccountData(ctx, roomID, event.AccountDataMarkedUnread.Type, &event.MarkedUnreadEventContent{
+			Unread: unread,
+		})
+	}
 }
 
 func (as *ASIntent) MarkTyping(ctx context.Context, roomID id.RoomID, typingType bridgev2.TypingType, timeout time.Duration) error {
@@ -172,7 +227,64 @@ func (as *ASIntent) DownloadMedia(ctx context.Context, uri id.ContentURIString,
 	return data, nil
 }
 
+func (as *ASIntent) DownloadMediaToFile(ctx context.Context, uri id.ContentURIString, file *event.EncryptedFileInfo, writable bool, callback func(*os.File) error) error {
+	if file != nil {
+		uri = file.URL
+		err := file.PrepareForDecryption()
+		if err != nil {
+			return err
+		}
+	}
+	parsedURI, err := uri.Parse()
+	if err != nil {
+		return err
+	}
+	tempFile, err := os.CreateTemp("", "mautrix-download-*")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
+	resp, err := as.Matrix.Download(ctx, parsedURI)
+	if err != nil {
+		return fmt.Errorf("failed to send download request: %w", err)
+	}
+	defer resp.Body.Close()
+	reader := resp.Body
+	if file != nil {
+		reader = file.DecryptStream(reader)
+	}
+	if resp.ContentLength > 0 {
+		err = fallocate.Fallocate(tempFile, int(resp.ContentLength))
+		if err != nil {
+			return fmt.Errorf("failed to preallocate file: %w", err)
+		}
+	}
+	_, err = io.Copy(tempFile, reader)
+	if err != nil {
+		return fmt.Errorf("failed to read response: %w", err)
+	}
+	err = reader.Close()
+	if err != nil {
+		return fmt.Errorf("failed to close response body: %w", err)
+	}
+	_, err = tempFile.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("failed to seek to start of temp file: %w", err)
+	}
+	err = callback(tempFile)
+	if err != nil {
+		return bridgev2.CallbackError{Type: "read", Wrapped: err}
+	}
+	return nil
+}
+
 func (as *ASIntent) UploadMedia(ctx context.Context, roomID id.RoomID, data []byte, fileName, mimeType string) (url id.ContentURIString, file *event.EncryptedFileInfo, err error) {
+	if int64(len(data)) > as.Connector.MediaConfig.UploadSize {
+		return "", nil, fmt.Errorf("file too large (%.2f MB > %.2f MB)", float64(len(data))/1000/1000, float64(as.Connector.MediaConfig.UploadSize)/1000/1000)
+	}
 	if roomID != "" {
 		var encrypted bool
 		if encrypted, err = as.Matrix.StateStore.IsEncrypted(ctx, roomID); err != nil {
@@ -187,12 +299,162 @@ func (as *ASIntent) UploadMedia(ctx context.Context, roomID id.RoomID, data []by
 			fileName = ""
 		}
 	}
-	req := mautrix.ReqUploadMedia{
+	url, err = as.doUploadReq(ctx, file, mautrix.ReqUploadMedia{
 		ContentBytes: data,
 		ContentType:  mimeType,
 		FileName:     fileName,
+	})
+	return
+}
+
+func (as *ASIntent) UploadMediaStream(
+	ctx context.Context,
+	roomID id.RoomID,
+	size int64,
+	requireFile bool,
+	cb bridgev2.FileStreamCallback,
+) (url id.ContentURIString, file *event.EncryptedFileInfo, err error) {
+	if size > as.Connector.MediaConfig.UploadSize {
+		return "", nil, fmt.Errorf("file too large (%.2f MB > %.2f MB)", float64(size)/1000/1000, float64(as.Connector.MediaConfig.UploadSize)/1000/1000)
+	}
+	if !requireFile && 0 < size && size < as.Connector.Config.Matrix.UploadFileThreshold {
+		var buf bytes.Buffer
+		res, err := cb(&buf)
+		if err != nil {
+			return "", nil, err
+		} else if res.ReplacementFile != "" {
+			panic(fmt.Errorf("logic error: replacement path must only be returned if requireFile is true"))
+		}
+		return as.UploadMedia(ctx, roomID, buf.Bytes(), res.FileName, res.MimeType)
+	}
+	var tempFile *os.File
+	tempFile, err = os.CreateTemp("", "mautrix-upload-*")
+	if err != nil {
+		err = fmt.Errorf("failed to create temp file: %w", err)
+		return
+	}
+	removeAndClose := func(f *os.File) {
+		_ = f.Close()
+		_ = os.Remove(f.Name())
+	}
+	startedAsyncUpload := false
+	defer func() {
+		if !startedAsyncUpload {
+			removeAndClose(tempFile)
+		}
+	}()
+	if size > 0 {
+		err = fallocate.Fallocate(tempFile, int(size))
+		if err != nil {
+			err = fmt.Errorf("failed to preallocate file: %w", err)
+			return
+		}
+	}
+	if roomID != "" {
+		var encrypted bool
+		if encrypted, err = as.Matrix.StateStore.IsEncrypted(ctx, roomID); err != nil {
+			err = fmt.Errorf("failed to check if room is encrypted: %w", err)
+			return
+		} else if encrypted {
+			file = &event.EncryptedFileInfo{
+				EncryptedFile: *attachment.NewEncryptedFile(),
+			}
+		}
+	}
+	var res *bridgev2.FileStreamResult
+	res, err = cb(tempFile)
+	if err != nil {
+		err = bridgev2.CallbackError{Type: "write", Wrapped: err}
+		return
+	}
+	var replFile *os.File
+	if res.ReplacementFile != "" {
+		replFile, err = os.OpenFile(res.ReplacementFile, os.O_RDWR, 0)
+		if err != nil {
+			err = fmt.Errorf("failed to open replacement file: %w", err)
+			return
+		}
+		defer func() {
+			if !startedAsyncUpload {
+				removeAndClose(replFile)
+			}
+		}()
+	} else {
+		replFile = tempFile
+		_, err = replFile.Seek(0, io.SeekStart)
+		if err != nil {
+			err = fmt.Errorf("failed to seek to start of temp file: %w", err)
+			return
+		}
+	}
+	if file != nil {
+		res.FileName = ""
+		res.MimeType = "application/octet-stream"
+		err = file.EncryptFile(replFile)
+		if err != nil {
+			err = fmt.Errorf("failed to encrypt file: %w", err)
+			return
+		}
+		_, err = replFile.Seek(0, io.SeekStart)
+		if err != nil {
+			err = fmt.Errorf("failed to seek to start of temp file after encrypting: %w", err)
+			return
+		}
+	}
+	info, err := replFile.Stat()
+	if err != nil {
+		err = fmt.Errorf("failed to get temp file info: %w", err)
+		return
+	}
+	size = info.Size()
+	if size > as.Connector.MediaConfig.UploadSize {
+		return "", nil, fmt.Errorf("file too large (%.2f MB > %.2f MB)", float64(size)/1000/1000, float64(as.Connector.MediaConfig.UploadSize)/1000/1000)
+	}
+	req := mautrix.ReqUploadMedia{
+		Content:       replFile,
+		ContentLength: size,
+		ContentType:   res.MimeType,
+		FileName:      res.FileName,
 	}
 	if as.Connector.Config.Homeserver.AsyncMedia {
+		req.DoneCallback = func() {
+			removeAndClose(replFile)
+			removeAndClose(tempFile)
+		}
+		req.AsyncContext = zerolog.Ctx(ctx).WithContext(as.Connector.Bridge.BackgroundCtx)
+		startedAsyncUpload = true
+		var resp *mautrix.RespCreateMXC
+		resp, err = as.Matrix.UploadAsync(ctx, req)
+		if resp != nil {
+			url = resp.ContentURI.CUString()
+		}
+	} else {
+		var resp *mautrix.RespMediaUpload
+		resp, err = as.Matrix.UploadMedia(ctx, req)
+		if resp != nil {
+			url = resp.ContentURI.CUString()
+		}
+	}
+	if file != nil {
+		file.URL = url
+		url = ""
+	}
+	return
+}
+
+func (as *ASIntent) doUploadReq(ctx context.Context, file *event.EncryptedFileInfo, req mautrix.ReqUploadMedia) (url id.ContentURIString, err error) {
+	if as.Connector.Config.Homeserver.AsyncMedia {
+		if req.ContentBytes != nil {
+			// Prevent too many background uploads at once
+			err = as.Connector.uploadSema.Acquire(ctx, int64(len(req.ContentBytes)))
+			if err != nil {
+				return
+			}
+			req.DoneCallback = func() {
+				as.Connector.uploadSema.Release(int64(len(req.ContentBytes)))
+			}
+		}
+		req.AsyncContext = zerolog.Ctx(ctx).WithContext(as.Connector.Bridge.BackgroundCtx)
 		var resp *mautrix.RespCreateMXC
 		resp, err = as.Matrix.UploadAsync(ctx, req)
 		if resp != nil {
@@ -224,27 +486,78 @@ func (as *ASIntent) SetAvatarURL(ctx context.Context, avatarURL id.ContentURIStr
 	return as.Matrix.SetAvatarURL(ctx, parsedAvatarURL)
 }
 
-func (as *ASIntent) SetExtraProfileMeta(ctx context.Context, data any) error {
-	if !as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) {
-		return nil
+func dataToFields(data any) (map[string]json.RawMessage, error) {
+	fields, ok := data.(map[string]json.RawMessage)
+	if ok {
+		return fields, nil
 	}
-	return as.Matrix.BeeperUpdateProfile(ctx, data)
+	d, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	d = canonicaljson.CanonicalJSONAssumeValid(d)
+	err = json.Unmarshal(d, &fields)
+	return fields, err
+}
+
+func marshalField(val any) json.RawMessage {
+	data, _ := json.Marshal(val)
+	if len(data) > 0 && (data[0] == '{' || data[0] == '[') {
+		return canonicaljson.CanonicalJSONAssumeValid(data)
+	}
+	return data
+}
+
+var nullJSON = json.RawMessage("null")
+
+func (as *ASIntent) SetExtraProfileMeta(ctx context.Context, data any) error {
+	if as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) {
+		return as.Matrix.BeeperUpdateProfile(ctx, data)
+	} else if as.Connector.SpecVersions.Supports(mautrix.FeatureArbitraryProfileFields) && as.Connector.Config.Matrix.GhostExtraProfileInfo {
+		fields, err := dataToFields(data)
+		if err != nil {
+			return fmt.Errorf("failed to marshal fields: %w", err)
+		}
+		currentProfile, err := as.Matrix.GetProfile(ctx, as.Matrix.UserID)
+		if err != nil {
+			return fmt.Errorf("failed to get current profile: %w", err)
+		}
+		for key, val := range fields {
+			existing, ok := currentProfile.Extra[key]
+			if !ok {
+				if bytes.Equal(val, nullJSON) {
+					continue
+				}
+				err = as.Matrix.SetProfileField(ctx, key, val)
+			} else if !bytes.Equal(marshalField(existing), val) {
+				if bytes.Equal(val, nullJSON) {
+					err = as.Matrix.DeleteProfileField(ctx, key)
+				} else {
+					err = as.Matrix.SetProfileField(ctx, key, val)
+				}
+			}
+			if err != nil {
+				return fmt.Errorf("failed to set profile field %q: %w", key, err)
+			}
+		}
+	}
+	return nil
 }
 
 func (as *ASIntent) GetMXID() id.UserID {
 	return as.Matrix.UserID
 }
 
-func (as *ASIntent) InviteUser(ctx context.Context, roomID id.RoomID, userID id.UserID) error {
-	_, err := as.Matrix.InviteUser(ctx, roomID, &mautrix.ReqInviteUser{
-		Reason: "",
-		UserID: userID,
-	})
-	return err
+func (as *ASIntent) IsDoublePuppet() bool {
+	return as.Matrix.IsDoublePuppet()
 }
 
-func (as *ASIntent) EnsureJoined(ctx context.Context, roomID id.RoomID) error {
-	err := as.Matrix.EnsureJoined(ctx, roomID)
+func (as *ASIntent) EnsureJoined(ctx context.Context, roomID id.RoomID, extra ...bridgev2.EnsureJoinedParams) error {
+	var params bridgev2.EnsureJoinedParams
+	if len(extra) > 0 {
+		params = extra[0]
+	}
+	err := as.Matrix.EnsureJoined(ctx, roomID, appservice.EnsureJoinedParams{Via: params.Via})
 	if err != nil {
 		return err
 	}
@@ -261,17 +574,54 @@ func (as *ASIntent) EnsureInvited(ctx context.Context, roomID id.RoomID, userID
 	return as.Matrix.EnsureInvited(ctx, roomID, userID)
 }
 
+func (br *Connector) getDefaultEncryptionEvent() *event.EncryptionEventContent {
+	content := &event.EncryptionEventContent{Algorithm: id.AlgorithmMegolmV1}
+	if rot := br.Config.Encryption.Rotation; rot.EnableCustom {
+		content.RotationPeriodMillis = rot.Milliseconds
+		content.RotationPeriodMessages = rot.Messages
+	}
+	return content
+}
+
+func (as *ASIntent) filterCreateRequestForV12(ctx context.Context, req *mautrix.ReqCreateRoom) {
+	if as.Connector.Config.Homeserver.Software == bridgeconfig.SoftwareHungry {
+		// Hungryserv doesn't override the capabilities endpoint nor do room versions
+		return
+	}
+	caps := as.Connector.fetchCapabilities(ctx)
+	roomVer := req.RoomVersion
+	if roomVer == "" && caps != nil && caps.RoomVersions != nil {
+		roomVer = id.RoomVersion(caps.RoomVersions.Default)
+	}
+	if roomVer != "" && !roomVer.PrivilegedRoomCreators() {
+		return
+	}
+	creators, _ := req.CreationContent["additional_creators"].([]id.UserID)
+	creators = append(slices.Clone(creators), as.GetMXID())
+	if req.PowerLevelOverride != nil {
+		for _, creator := range creators {
+			delete(req.PowerLevelOverride.Users, creator)
+		}
+	}
+	for _, evt := range req.InitialState {
+		if evt.Type != event.StatePowerLevels {
+			continue
+		}
+		content, ok := evt.Content.Parsed.(*event.PowerLevelsEventContent)
+		if ok {
+			for _, creator := range creators {
+				delete(content.Users, creator)
+			}
+		}
+	}
+}
+
 func (as *ASIntent) CreateRoom(ctx context.Context, req *mautrix.ReqCreateRoom) (id.RoomID, error) {
 	if as.Connector.Config.Encryption.Default {
-		content := &event.EncryptionEventContent{Algorithm: id.AlgorithmMegolmV1}
-		if rot := as.Connector.Config.Encryption.Rotation; rot.EnableCustom {
-			content.RotationPeriodMillis = rot.Milliseconds
-			content.RotationPeriodMessages = rot.Messages
-		}
 		req.InitialState = append(req.InitialState, &event.Event{
 			Type: event.StateEncryption,
 			Content: event.Content{
-				Parsed: content,
+				Parsed: as.Connector.getDefaultEncryptionEvent(),
 			},
 		})
 	}
@@ -281,6 +631,7 @@ func (as *ASIntent) CreateRoom(ctx context.Context, req *mautrix.ReqCreateRoom)
 		}
 		req.CreationContent["m.federate"] = false
 	}
+	as.filterCreateRequestForV12(ctx, req)
 	resp, err := as.Matrix.CreateRoom(ctx, req)
 	if err != nil {
 		return "", err
@@ -288,9 +639,53 @@ func (as *ASIntent) CreateRoom(ctx context.Context, req *mautrix.ReqCreateRoom)
 	return resp.RoomID, nil
 }
 
+func (as *ASIntent) MarkAsDM(ctx context.Context, roomID id.RoomID, withUser id.UserID) error {
+	if !as.Connector.Config.Matrix.SyncDirectChatList {
+		return nil
+	}
+	as.dmUpdateLock.Lock()
+	defer as.dmUpdateLock.Unlock()
+	cached, ok := as.directChatsCache[withUser]
+	if ok && slices.Contains(cached, roomID) {
+		return nil
+	}
+	var directChats event.DirectChatsEventContent
+	err := as.Matrix.GetAccountData(ctx, event.AccountDataDirectChats.Type, &directChats)
+	if err != nil {
+		return err
+	}
+	as.directChatsCache = directChats
+	rooms := directChats[withUser]
+	if slices.Contains(rooms, roomID) {
+		return nil
+	}
+	directChats[withUser] = append(rooms, roomID)
+	err = as.Matrix.SetAccountData(ctx, event.AccountDataDirectChats.Type, &directChats)
+	if err != nil {
+		if rooms == nil {
+			delete(directChats, withUser)
+		} else {
+			directChats[withUser] = rooms
+		}
+		return fmt.Errorf("failed to set direct chats account data: %w", err)
+	}
+	return nil
+}
+
 func (as *ASIntent) DeleteRoom(ctx context.Context, roomID id.RoomID, puppetsOnly bool) error {
+	if roomID == "" {
+		return nil
+	}
 	if as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureRoomYeeting) {
-		return as.Matrix.BeeperDeleteRoom(ctx, roomID)
+		err := as.Matrix.BeeperDeleteRoom(ctx, roomID)
+		if err != nil {
+			return err
+		}
+		err = as.Matrix.StateStore.ClearCachedMembers(ctx, roomID)
+		if err != nil {
+			zerolog.Ctx(ctx).Err(err).Msg("Failed to clear cached members while cleaning up portal")
+		}
+		return nil
 	}
 	members, err := as.Matrix.JoinedMembers(ctx, roomID)
 	if err != nil {
@@ -300,8 +695,7 @@ func (as *ASIntent) DeleteRoom(ctx context.Context, roomID id.RoomID, puppetsOnl
 		if member == as.Matrix.UserID {
 			continue
 		}
-		_, isGhost := as.Connector.ParseGhostMXID(member)
-		if isGhost {
+		if as.Connector.Bridge.IsGhostMXID(member) {
 			_, err = as.Connector.AS.Intent(member).LeaveRoom(ctx, roomID)
 			if err != nil {
 				zerolog.Ctx(ctx).Err(err).Stringer("user_id", member).Msg("Failed to leave room while cleaning up portal")
@@ -317,6 +711,10 @@ func (as *ASIntent) DeleteRoom(ctx context.Context, roomID id.RoomID, puppetsOnl
 	if err != nil {
 		zerolog.Ctx(ctx).Err(err).Msg("Failed to leave room while cleaning up portal")
 	}
+	err = as.Matrix.StateStore.ClearCachedMembers(ctx, roomID)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to clear cached members while cleaning up portal")
+	}
 	return nil
 }
 
@@ -349,7 +747,20 @@ func (as *ASIntent) TagRoom(ctx context.Context, roomID id.RoomID, tag event.Roo
 }
 
 func (as *ASIntent) MuteRoom(ctx context.Context, roomID id.RoomID, until time.Time) error {
-	if !until.IsZero() && until.Before(time.Now()) {
+	var mutedUntil int64
+	if until.Before(time.Now()) {
+		mutedUntil = 0
+	} else if until == event.MutedForever {
+		mutedUntil = -1
+	} else {
+		mutedUntil = until.UnixMilli()
+	}
+	if as.Connector.SpecVersions.Supports(mautrix.BeeperFeatureAccountDataMute) {
+		return as.Matrix.SetRoomAccountData(ctx, roomID, event.AccountDataBeeperMute.Type, &event.BeeperMuteEventContent{
+			MutedUntil: mutedUntil,
+		})
+	}
+	if mutedUntil == 0 {
 		err := as.Matrix.DeletePushRule(ctx, "global", pushrules.RoomRule, string(roomID))
 		// If the push rule doesn't exist, everything is fine
 		if errors.Is(err, mautrix.MNotFound) {
@@ -362,3 +773,23 @@ func (as *ASIntent) MuteRoom(ctx context.Context, roomID id.RoomID, until time.T
 		})
 	}
 }
+
+func (as *ASIntent) GetEvent(ctx context.Context, roomID id.RoomID, eventID id.EventID) (*event.Event, error) {
+	evt, err := as.Matrix.Client.GetEvent(ctx, roomID, eventID)
+	if err != nil {
+		return nil, err
+	}
+	err = evt.Content.ParseRaw(evt.Type)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Stringer("room_id", roomID).Stringer("event_id", eventID).Msg("failed to parse event content")
+	}
+
+	if evt.Type == event.EventEncrypted {
+		if as.Connector.Crypto == nil || as.Connector.Config.Encryption.DeleteKeys.RatchetOnDecrypt {
+			return nil, errors.New("can't decrypt the event")
+		}
+		return as.Connector.Crypto.Decrypt(ctx, evt)
+	}
+
+	return evt, nil
+}

bridgev2/matrix/matrix.go

@@ -17,8 +17,8 @@ import (
 	"go.mau.fi/util/jsontime"
 
 	"maunium.net/go/mautrix/appservice"
-	"maunium.net/go/mautrix/bridge/status"
 	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/status"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
@@ -27,6 +27,11 @@ func (br *Connector) handleRoomEvent(ctx context.Context, evt *event.Event) {
 	if br.shouldIgnoreEvent(evt) {
 		return
 	}
+	if !br.Config.Bridge.Permissions.Get(evt.Sender).SendEvents && evt.Type != event.StateMember {
+		zerolog.Ctx(ctx).Debug().Msg("Dropping event from user with no permission to send events")
+		br.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt))
+		return
+	}
 	if (evt.Type == event.EventMessage || evt.Type == event.EventSticker) && !evt.Mautrix.WasEncrypted && br.Config.Encryption.Require {
 		zerolog.Ctx(ctx).Warn().Msg("Dropping unencrypted event as encryption is configured to be required")
 		br.sendCryptoStatusError(ctx, evt, errMessageNotEncrypted, nil, 0, true)
@@ -63,6 +68,10 @@ func (br *Connector) handleEphemeralEvent(ctx context.Context, evt *event.Event)
 	case event.EphemeralEventTyping:
 		typingContent := evt.Content.AsTyping()
 		typingContent.UserIDs = slices.DeleteFunc(typingContent.UserIDs, br.shouldIgnoreEventFromUser)
+	case event.BeeperEphemeralEventAIStream:
+		if br.shouldIgnoreEvent(evt) {
+			return
+		}
 	}
 	br.Bridge.QueueMatrixEvent(ctx, evt)
 }
@@ -76,6 +85,11 @@ func (br *Connector) handleEncryptedEvent(ctx context.Context, evt *event.Event)
 		Str("event_id", evt.ID.String()).
 		Str("session_id", content.SessionID.String()).
 		Logger()
+	if !br.Config.Bridge.Permissions.Get(evt.Sender).SendEvents {
+		log.Debug().Msg("Dropping event from user with no permission to send events")
+		br.SendMessageStatus(ctx, &bridgev2.ErrNoPermissionToInteract, bridgev2.StatusEventInfoFromEvent(evt))
+		return
+	}
 	ctx = log.WithContext(ctx)
 	if br.Crypto == nil {
 		br.sendCryptoStatusError(ctx, evt, errNoCrypto, nil, 0, true)
@@ -87,17 +101,18 @@ func (br *Connector) handleEncryptedEvent(ctx context.Context, evt *event.Event)
 	decryptionStart := time.Now()
 	decrypted, err := br.Crypto.Decrypt(ctx, evt)
 	decryptionRetryCount := 0
+	var errorEventID id.EventID
 	if errors.Is(err, NoSessionFound) {
 		decryptionRetryCount = 1
 		log.Debug().
 			Int("wait_seconds", int(initialSessionWaitTimeout.Seconds())).
 			Msg("Couldn't find session, waiting for keys to arrive...")
-		go br.sendCryptoStatusError(ctx, evt, err, nil, 0, false)
+		go br.sendCryptoStatusError(ctx, evt, err, &errorEventID, 0, false)
 		if br.Crypto.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, initialSessionWaitTimeout) {
 			log.Debug().Msg("Got keys after waiting, trying to decrypt event again")
 			decrypted, err = br.Crypto.Decrypt(ctx, evt)
 		} else {
-			go br.waitLongerForSession(ctx, evt, decryptionStart)
+			go br.waitLongerForSession(ctx, evt, decryptionStart, &errorEventID)
 			return
 		}
 	}
@@ -106,18 +121,18 @@ func (br *Connector) handleEncryptedEvent(ctx context.Context, evt *event.Event)
 		go br.sendCryptoStatusError(ctx, evt, err, nil, decryptionRetryCount, true)
 		return
 	}
-	br.postDecrypt(ctx, evt, decrypted, decryptionRetryCount, nil, time.Since(decryptionStart))
+	br.postDecrypt(ctx, evt, decrypted, decryptionRetryCount, &errorEventID, time.Since(decryptionStart))
 }
 
-func (br *Connector) waitLongerForSession(ctx context.Context, evt *event.Event, decryptionStart time.Time) {
+func (br *Connector) waitLongerForSession(ctx context.Context, evt *event.Event, decryptionStart time.Time, errorEventID *id.EventID) {
 	log := zerolog.Ctx(ctx)
 	content := evt.Content.AsEncrypted()
 	log.Debug().
 		Int("wait_seconds", int(extendedSessionWaitTimeout.Seconds())).
 		Msg("Couldn't find session, requesting keys and waiting longer...")
 
+	//lint:ignore SA1019 RequestSession will gracefully request from all devices if DeviceID is blank
 	go br.Crypto.RequestSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, evt.Sender, content.DeviceID)
-	var errorEventID *id.EventID
 	go br.sendCryptoStatusError(ctx, evt, fmt.Errorf("%w. The bridge will retry for %d seconds", errNoDecryptionKeys, int(extendedSessionWaitTimeout.Seconds())), errorEventID, 1, false)
 
 	if !br.Crypto.WaitForSession(ctx, evt.RoomID, content.SenderKey, content.SessionID, extendedSessionWaitTimeout) {
@@ -142,7 +157,7 @@ type CommandProcessor interface {
 }
 
 func (br *Connector) sendSuccessCheckpoint(ctx context.Context, evt *event.Event, step status.MessageCheckpointStep, retryNum int) {
-	err := br.SendMessageCheckpoints([]*status.MessageCheckpoint{{
+	err := br.SendMessageCheckpoints(ctx, []*status.MessageCheckpoint{{
 		RoomID:      evt.RoomID,
 		EventID:     evt.ID,
 		EventType:   evt.Type,
@@ -165,18 +180,11 @@ func (br *Connector) sendBridgeCheckpoint(ctx context.Context, evt *event.Event)
 }
 
 func (br *Connector) shouldIgnoreEventFromUser(userID id.UserID) bool {
-	if userID == br.Bot.UserID {
-		return true
-	}
-	_, isGhost := br.ParseGhostMXID(userID)
-	if isGhost {
-		return true
-	}
-	return false
+	return userID == br.Bot.UserID || br.Bridge.IsGhostMXID(userID)
 }
 
 func (br *Connector) shouldIgnoreEvent(evt *event.Event) bool {
-	if br.shouldIgnoreEventFromUser(evt.Sender) {
+	if br.shouldIgnoreEventFromUser(evt.Sender) && evt.Type != event.StateTombstone {
 		return true
 	}
 	dpVal, ok := evt.Content.Raw[appservice.DoublePuppetKey]
@@ -227,7 +235,6 @@ func (br *Connector) postDecrypt(ctx context.Context, original, decrypted *event
 	go br.sendSuccessCheckpoint(ctx, decrypted, status.MsgStepDecrypted, retryCount)
 	decrypted.Mautrix.CheckpointSent = true
 	decrypted.Mautrix.DecryptionDuration = duration
-	decrypted.Mautrix.EventSource |= event.SourceDecrypted
 	br.EventProcessor.Dispatch(ctx, decrypted)
 	if errorEventID != nil && *errorEventID != "" {
 		_, _ = br.Bot.RedactEvent(ctx, decrypted.RoomID, *errorEventID)

bridgev2/matrix/mxmain/dberror.go

@@ -64,10 +64,14 @@ func (br *BridgeMain) LogDBUpgradeErrorAndExit(name string, err error, message s
 	if sqlError := (&sqlite3.Error{}); errors.As(err, sqlError) && sqlError.Code == sqlite3.ErrCorrupt {
 		os.Exit(18)
 	} else if errors.Is(err, dbutil.ErrForeignTables) {
-		br.Log.Info().Msg("You can use --ignore-foreign-tables to ignore this error")
 		br.Log.Info().Msg("See https://docs.mau.fi/faq/foreign-tables for more info")
 	} else if errors.Is(err, dbutil.ErrNotOwned) {
-		br.Log.Info().Msg("Sharing the same database with different programs is not supported")
+		var noe dbutil.NotOwnedError
+		if errors.As(err, &noe) && noe.Owner == br.Name {
+			br.Log.Info().Msg("The database appears to be on a very old pre-megabridge schema. Perhaps you need to run an older version of the bridge with migration support first?")
+		} else {
+			br.Log.Info().Msg("Sharing the same database with different programs is not supported")
+		}
 	} else if errors.Is(err, dbutil.ErrUnsupportedDatabaseVersion) {
 		br.Log.Info().Msg("Downgrading the bridge is not supported")
 	}

bridgev2/matrix/mxmain/envconfig.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2025 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package mxmain
+
+import (
+	"fmt"
+	"iter"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"go.mau.fi/util/random"
+)
+
+var randomParseFilePrefix = random.String(16) + "READFILE:"
+
+func parseEnv(prefix string) iter.Seq2[[]string, string] {
+	return func(yield func([]string, string) bool) {
+		for _, s := range os.Environ() {
+			if !strings.HasPrefix(s, prefix) {
+				continue
+			}
+			kv := strings.SplitN(s, "=", 2)
+			key := strings.TrimPrefix(kv[0], prefix)
+			value := kv[1]
+			if strings.HasSuffix(key, "_FILE") {
+				key = strings.TrimSuffix(key, "_FILE")
+				value = randomParseFilePrefix + value
+			}
+			key = strings.ToLower(key)
+			if !strings.ContainsRune(key, '.') {
+				key = strings.ReplaceAll(key, "__", ".")
+			}
+			if !yield(strings.Split(key, "."), value) {
+				return
+			}
+		}
+	}
+}
+
+func reflectYAMLFieldName(f *reflect.StructField) string {
+	parts := strings.SplitN(f.Tag.Get("yaml"), ",", 2)
+	fieldName := parts[0]
+	if fieldName == "-" && len(parts) == 1 {
+		return ""
+	}
+	if fieldName == "" {
+		return strings.ToLower(f.Name)
+	}
+	return fieldName
+}
+
+type reflectGetResult struct {
+	val           reflect.Value
+	valKind       reflect.Kind
+	remainingPath []string
+}
+
+func reflectGetYAML(rv reflect.Value, path []string) (*reflectGetResult, bool) {
+	if len(path) == 0 {
+		return &reflectGetResult{val: rv, valKind: rv.Kind()}, true
+	}
+	if rv.Kind() == reflect.Ptr {
+		rv = rv.Elem()
+	}
+	switch rv.Kind() {
+	case reflect.Map:
+		return &reflectGetResult{val: rv, remainingPath: path, valKind: rv.Type().Elem().Kind()}, true
+	case reflect.Struct:
+		fields := reflect.VisibleFields(rv.Type())
+		for _, field := range fields {
+			fieldName := reflectYAMLFieldName(&field)
+			if fieldName != "" && fieldName == path[0] {
+				return reflectGetYAML(rv.FieldByIndex(field.Index), path[1:])
+			}
+		}
+	default:
+	}
+	return nil, false
+}
+
+func reflectGetFromMainOrNetwork(main, network reflect.Value, path []string) (*reflectGetResult, bool) {
+	if len(path) > 0 && path[0] == "network" {
+		return reflectGetYAML(network, path[1:])
+	}
+	return reflectGetYAML(main, path)
+}
+
+func formatKeyString(key []string) string {
+	return strings.Join(key, "->")
+}
+
+func UpdateConfigFromEnv(cfg, networkData any, prefix string) error {
+	cfgVal := reflect.ValueOf(cfg)
+	networkVal := reflect.ValueOf(networkData)
+	for key, value := range parseEnv(prefix) {
+		field, ok := reflectGetFromMainOrNetwork(cfgVal, networkVal, key)
+		if !ok {
+			return fmt.Errorf("%s not found", formatKeyString(key))
+		}
+		if strings.HasPrefix(value, randomParseFilePrefix) {
+			filepath := strings.TrimPrefix(value, randomParseFilePrefix)
+			fileData, err := os.ReadFile(filepath)
+			if err != nil {
+				return fmt.Errorf("failed to read file %s for %s: %w", filepath, formatKeyString(key), err)
+			}
+			value = strings.TrimSpace(string(fileData))
+		}
+		var parsedVal any
+		var err error
+		switch field.valKind {
+		case reflect.String:
+			parsedVal = value
+		case reflect.Bool:
+			parsedVal, err = strconv.ParseBool(value)
+			if err != nil {
+				return fmt.Errorf("invalid value for %s: %w", formatKeyString(key), err)
+			}
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			parsedVal, err = strconv.ParseInt(value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("invalid value for %s: %w", formatKeyString(key), err)
+			}
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			parsedVal, err = strconv.ParseUint(value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("invalid value for %s: %w", formatKeyString(key), err)
+			}
+		case reflect.Float32, reflect.Float64:
+			parsedVal, err = strconv.ParseFloat(value, 64)
+			if err != nil {
+				return fmt.Errorf("invalid value for %s: %w", formatKeyString(key), err)
+			}
+		default:
+			return fmt.Errorf("unsupported type %s in %s", field.valKind, formatKeyString(key))
+		}
+		if field.val.Kind() == reflect.Ptr {
+			if field.val.IsNil() {
+				field.val.Set(reflect.New(field.val.Type().Elem()))
+			}
+			field.val = field.val.Elem()
+		}
+		if field.val.Kind() == reflect.Map {
+			key = key[:len(key)-len(field.remainingPath)]
+			mapKeyStr := strings.Join(field.remainingPath, ".")
+			key = append(key, mapKeyStr)
+			if field.val.Type().Key().Kind() != reflect.String {
+				return fmt.Errorf("unsupported map key type %s in %s", field.val.Type().Key().Kind(), formatKeyString(key))
+			}
+			field.val.SetMapIndex(reflect.ValueOf(mapKeyStr), reflect.ValueOf(parsedVal))
+		} else {
+			field.val.Set(reflect.ValueOf(parsedVal))
+		}
+	}
+	return nil
+}

bridgev2/matrix/mxmain/example-config.yaml

@@ -4,6 +4,84 @@ bridge:
     command_prefix: '$<<or .DefaultCommandPrefix (printf "!%s" .NetworkID)>>'
     # Should the bridge create a space for each login containing the rooms that account is in?
     personal_filtering_spaces: true
+    # Whether the bridge should set names and avatars explicitly for DM portals.
+    # This is only necessary when using clients that don't support MSC4171.
+    private_chat_portal_meta: true
+    # Should events be handled asynchronously within portal rooms?
+    # If true, events may end up being out of order, but slow events won't block other ones.
+    # This is not yet safe to use.
+    async_events: false
+    # Should every user have their own portals rather than sharing them?
+    # By default, users who are in the same group on the remote network will be
+    # in the same Matrix room bridged to that group. If this is set to true,
+    # every user will get their own Matrix room instead.
+    # SETTING THIS IS IRREVERSIBLE AND POTENTIALLY DESTRUCTIVE IF PORTALS ALREADY EXIST.
+    split_portals: false
+    # Should the bridge resend `m.bridge` events to all portals on startup?
+    resend_bridge_info: false
+    # Should `m.bridge` events be sent without a state key?
+    # By default, the bridge uses a unique key that won't conflict with other bridges.
+    no_bridge_info_state_key: false
+    # Should bridge connection status be sent to the management room as `m.notice` events?
+    # These contain the same data that can be posted to an external HTTP server using homeserver -> status_endpoint.
+    # Allowed values: none, errors, all
+    bridge_status_notices: errors
+    # How long after an unknown error should the bridge attempt a full reconnect?
+    # Must be at least 1 minute. The bridge will add an extra ±20% jitter to this value.
+    unknown_error_auto_reconnect: null
+    # Maximum number of times to do the auto-reconnect above.
+    # The counter is per login, but is never reset except on logout and restart.
+    unknown_error_max_auto_reconnects: 10
+
+    # Should leaving Matrix rooms be bridged as leaving groups on the remote network?
+    bridge_matrix_leave: false
+    # Should `m.notice` messages be bridged?
+    bridge_notices: false
+    # Should room tags only be synced when creating the portal? Tags mean things like favorite/pin and archive/low priority.
+    # Tags currently can't be synced back to the remote network, so a continuous sync means tagging from Matrix will be undone.
+    tag_only_on_create: true
+    # List of tags to allow bridging. If empty, no tags will be bridged.
+    only_bridge_tags: [m.favourite, m.lowpriority]
+    # Should room mute status only be synced when creating the portal?
+    # Like tags, mutes can't currently be synced back to the remote network.
+    mute_only_on_create: true
+    # Should the bridge check the db to ensure that incoming events haven't been handled before
+    deduplicate_matrix_messages: false
+    # Should cross-room reply metadata be bridged?
+    # Most Matrix clients don't support this and servers may reject such messages too.
+    cross_room_replies: false
+    # If a state event fails to bridge, should the bridge revert any state changes made by that event?
+    revert_failed_state_changes: false
+    # In portals with no relay set, should Matrix users be kicked if they're
+    # not logged into an account that's in the remote chat?
+    kick_matrix_users: true
+
+    # What should be done to portal rooms when a user logs out or is logged out?
+    # Permitted values:
+    #   nothing - Do nothing, let the user stay in the portals
+    #   kick - Remove the user from the portal rooms, but don't delete them
+    #   unbridge - Remove all ghosts in the room and disassociate it from the remote chat
+    #   delete - Remove all ghosts and users from the room (i.e. delete it)
+    cleanup_on_logout:
+        # Should cleanup on logout be enabled at all?
+        enabled: false
+        # Settings for manual logouts (explicitly initiated by the Matrix user)
+        manual:
+            # Action for private portals which will never be shared with other Matrix users.
+            private: nothing
+            # Action for portals with a relay user configured.
+            relayed: nothing
+            # Action for portals which may be shared, but don't currently have any other Matrix users.
+            shared_no_users: nothing
+            # Action for portals which have other logged-in Matrix users.
+            shared_has_users: nothing
+        # Settings for credentials being invalidated (initiated by the remote network, possibly through user action).
+        # Keys have the same meanings as in the manual section.
+        bad_credentials:
+            private: nothing
+            relayed: nothing
+            shared_no_users: nothing
+            shared_has_users: nothing
 
     # Settings for relay mode
     relay:
@@ -11,19 +89,34 @@ bridge:
         # authenticated user into a relaybot for that chat.
         enabled: false
         # Should only admins be allowed to set themselves as relay users?
+        # If true, non-admins can only set users listed in default_relays as relays in a room.
         admin_only: true
         # List of user login IDs which anyone can set as a relay, as long as the relay user is in the room.
         default_relays: []
         # The formats to use when sending messages via the relaybot.
+        # Available variables:
+        #   .Sender.UserID - The Matrix user ID of the sender.
+        #   .Sender.Displayname - The display name of the sender (if set).
+        #   .Sender.RequiresDisambiguation - Whether the sender's name may be confused with the name of another user in the room.
+        #   .Sender.DisambiguatedName - The disambiguated name of the sender. This will be the displayname if set,
+        #                               plus the user ID in parentheses if the displayname is not unique.
+        #                               If the displayname is not set, this is just the user ID.
+        #   .Message - The `formatted_body` field of the message.
+        #   .Caption - The `formatted_body` field of the message, if it's a caption. Otherwise an empty string.
+        #   .FileName - The name of the file being sent.
         message_formats:
-            m.text: "<b>{{ .Sender.Displayname }}</b>: {{ .Message }}"
-            m.notice: "<b>{{ .Sender.Displayname }}</b>: {{ .Message }}"
-            m.emote: "* <b>{{ .Sender.Displayname }}</b> {{ .Message }}"
-            m.file: "<b>{{ .Sender.Displayname }}</b> sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}"
-            m.image: "<b>{{ .Sender.Displayname }}</b> sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}"
-            m.audio: "<b>{{ .Sender.Displayname }}</b> sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}"
-            m.video: "<b>{{ .Sender.Displayname }}</b> sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}"
-            m.location: "<b>{{ .Sender.Displayname }}</b> sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}"
+            m.text: "<b>{{ .Sender.DisambiguatedName }}</b>: {{ .Message }}"
+            m.notice: "<b>{{ .Sender.DisambiguatedName }}</b>: {{ .Message }}"
+            m.emote: "* <b>{{ .Sender.DisambiguatedName }}</b> {{ .Message }}"
+            m.file: "<b>{{ .Sender.DisambiguatedName }}</b> sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}"
+            m.image: "<b>{{ .Sender.DisambiguatedName }}</b> sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}"
+            m.audio: "<b>{{ .Sender.DisambiguatedName }}</b> sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}"
+            m.video: "<b>{{ .Sender.DisambiguatedName }}</b> sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}"
+            m.location: "<b>{{ .Sender.DisambiguatedName }}</b> sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}"
+        # For networks that support per-message displaynames (i.e. Slack and Discord), the template for those names.
+        # This has all the Sender variables available under message_formats (but without the .Sender prefix).
+        # Note that you need to manually remove the displayname from message_formats above.
+        displayname_format: "{{ .DisambiguatedName }}"
 
     # Permissions for using the bridge.
     # Permitted values:
@@ -94,8 +187,12 @@ homeserver:
 # Changing these values requires regeneration of the registration (except when noted otherwise)
 appservice:
     # The address that the homeserver can use to connect to this appservice.
+    # Like the homeserver address, a local non-https address is recommended when the bridge is on the same machine.
+    # If the bridge is elsewhere, you must secure the connection yourself (e.g. with https or wireguard)
+    # If you want to use https, you need to use a reverse proxy. The bridge does not have TLS support built in.
     address: http://localhost:$<<or .DefaultPort 8008>>
     # A public address that external services can use to reach this appservice.
+    # This is only needed for things like public media. A reverse proxy is generally necessary when using this field.
     # This value doesn't affect the registration file.
     public_address: https://bridge.example.com
 
@@ -139,17 +236,31 @@ matrix:
     delivery_receipts: false
     # Whether the bridge should send error notices via m.notice events when a message fails to bridge.
     message_error_notices: true
-    sync_direct_chat_list: false
+    # Whether the bridge should update the m.direct account data event when double puppeting is enabled.
+    sync_direct_chat_list: true
     # Whether created rooms should have federation enabled. If false, created portal rooms
     # will never be federated. Changing this option requires recreating rooms.
     federate_rooms: true
+    # The threshold as bytes after which the bridge should roundtrip uploads via the disk
+    # rather than keeping the whole file in memory.
+    upload_file_threshold: 5242880
+    # Should the bridge set additional custom profile info for ghosts?
+    # This can make a lot of requests, as there's no batch profile update endpoint.
+    ghost_extra_profile_info: false
+
+# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors.
+analytics:
+    # API key to send with tracking requests. Tracking is disabled if this is null.
+    token: null
+    # Address to send tracking requests to.
+    url: https://api.segment.io/v1/track
+    # Optional user ID for tracking events. If null, defaults to using Matrix user ID.
+    user_id: null
 
 # Settings for provisioning API
 provisioning:
-    # Prefix for the provisioning API paths.
-    prefix: /_matrix/provision
     # Shared secret for authentication. If set to "generate" or null, a random secret will be generated,
-    # or if set to "disable", the provisioning API will be disabled.
+    # or if set to "disable", the provisioning API will be disabled. Must be at least 16 characters.
     shared_secret: generate
     # Whether to allow provisioning API requests to be authed using Matrix access tokens.
     # This follows the same rules as double puppeting to determine which server to contact to check the token,
@@ -157,6 +268,32 @@ provisioning:
     allow_matrix_auth: true
     # Enable debug API at /debug with provisioning authentication.
     debug_endpoints: false
+    # Enable session transfers between bridges. Note that this only validates Matrix or shared secret
+    # auth before passing live network client credentials down in the response.
+    enable_session_transfers: false
+
+# Some networks require publicly accessible media download links (e.g. for user avatars when using Discord webhooks).
+# These settings control whether the bridge will provide such public media access.
+public_media:
+    # Should public media be enabled at all?
+    # The public_address field under the appservice section MUST be set when enabling public media.
+    enabled: false
+    # A key for signing public media URLs.
+    # If set to "generate", a random key will be generated.
+    signing_key: generate
+    # Number of seconds that public media URLs are valid for.
+    # If set to 0, URLs will never expire.
+    expiry: 0
+    # Length of hash to use for public media URLs. Must be between 0 and 32.
+    hash_length: 32
+    # The path prefix for generated URLs. Note that this will NOT change the path where media is actually served.
+    # If you change this, you must configure your reverse proxy to rewrite the path accordingly.
+    path_prefix: /_mautrix/publicmedia
+    # Should the bridge store media metadata in the database in order to support encrypted media and generate shorter URLs?
+    # If false, the generated URLs will just have the MXC URI and a HMAC signature.
+    # The hash_length field will be used to decide the length of the generated URL.
+    # This also allows invalidating URLs by deleting the database entry.
+    use_database: false
 
 # Settings for converting remote media to custom mxc:// URIs instead of reuploading.
 # More details can be found at https://docs.mau.fi/bridges/go/discord/direct-media.html
@@ -180,6 +317,39 @@ direct_media:
     # This key is also used to sign the mxc:// URIs to ensure only the bridge can generate them.
     server_key: generate
 
+# Settings for backfilling messages.
+# Note that the exact way settings are applied depends on the network connector.
+# See https://docs.mau.fi/bridges/general/backfill.html for more details.
+backfill:
+    # Whether to do backfilling at all.
+    enabled: false
+    # Maximum number of messages to backfill in empty rooms.
+    max_initial_messages: 50
+    # Maximum number of missed messages to backfill after bridge restarts.
+    max_catchup_messages: 500
+    # If a backfilled chat is older than this number of hours,
+    # mark it as read even if it's unread on the remote network.
+    unread_hours_threshold: 720
+    # Settings for backfilling threads within other backfills.
+    threads:
+        # Maximum number of messages to backfill in a new thread.
+        max_initial_messages: 50
+    # Settings for the backwards backfill queue. This only applies when connecting to
+    # Beeper as standard Matrix servers don't support inserting messages into history.
+    queue:
+        # Should the backfill queue be enabled?
+        enabled: false
+        # Number of messages to backfill in one batch.
+        batch_size: 100
+        # Delay between batches in seconds.
+        batch_delay: 20
+        # Maximum number of batches to backfill per portal.
+        # If set to -1, all available messages will be backfilled.
+        max_batches: -1
+        # Optional network-specific overrides for max batches.
+        # Interpretation of this field depends on the network connector.
+        max_batches_override: {}
+
 # Settings for enabling double puppeting
 double_puppet:
     # Servers to always allow double puppeting from.
@@ -205,9 +375,21 @@ encryption:
     default: false
     # Whether to require all messages to be encrypted and drop any unencrypted messages.
     require: false
-    # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data.
+    # Whether to use MSC3202/MSC4203 instead of /sync long polling for receiving encryption-related data.
     # This option is not yet compatible with standard Matrix servers like Synapse and should not be used.
+    # Changing this option requires updating the appservice registration file.
     appservice: false
+    # Whether to use MSC4190 instead of appservice login to create the bridge bot device.
+    # Requires the homeserver to support MSC4190 and the device masquerading parts of MSC3202.
+    # Only relevant when using end-to-bridge encryption, required when using encryption with next-gen auth (MSC3861).
+    # Changing this option requires updating the appservice registration file.
+    msc4190: false
+    # Whether to encrypt reactions and reply metadata as per MSC4392.
+    msc4392: false
+    # Should the bridge bot generate a recovery key and cross-signing keys and verify itself?
+    # Note that without the latest version of MSC4190, this will fail if you reset the bridge database.
+    # The generated recovery key will be saved in the kv_store table under `recovery_key`.
+    self_sign: false
     # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
     # You must use a client that supports requesting keys from other users to use this feature.
     allow_key_sharing: true
@@ -270,6 +452,16 @@ encryption:
         # You should not enable this option unless you understand all the implications.
         disable_device_change_key_rotation: false
 
+# Prefix for environment variables. All variables with this prefix must map to valid config fields.
+# Nesting in variable names is represented with a dot (.).
+# If there are no dots in the name, two underscores (__) are replaced with a dot.
+#
+# e.g. if the prefix is set to `BRIDGE_`, then `BRIDGE_APPSERVICE__AS_TOKEN` will set appservice.as_token.
+# `BRIDGE_appservice.as_token` would work as well, but can't be set in a shell as easily.
+#
+# If this is null, reading config fields from environment will be disabled.
+env_config_prefix: null
+
 # Logging config. See https://github.com/tulir/zeroconfig for details.
 logging:
     min_level: debug

bridgev2/matrix/mxmain/legacymigrate.go

@@ -12,11 +12,37 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+
+	"github.com/rs/zerolog"
+	"go.mau.fi/util/dbutil"
+
+	"maunium.net/go/mautrix/appservice"
+	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/database"
+	"maunium.net/go/mautrix/bridgev2/matrix"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
 )
 
-func (br *BridgeMain) LegacyMigrateSimple(renameTablesQuery, copyDataQuery string, newDBVersion int) func(ctx context.Context) error {
+func (br *BridgeMain) LegacyMigrateWithAnotherUpgrader(renameTablesQuery, copyDataQuery string, newDBVersion int, otherTable dbutil.UpgradeTable, otherTableName string, otherNewVersion int) func(ctx context.Context) error {
 	return func(ctx context.Context) error {
-		_, err := br.DB.Exec(ctx, renameTablesQuery)
+		// Unique constraints must have globally unique names on postgres, and renaming the table doesn't rename them,
+		// so just drop the ones that may conflict with the new schema.
+		if br.DB.Dialect == dbutil.Postgres {
+			_, err := br.DB.Exec(ctx, "ALTER TABLE message DROP CONSTRAINT IF EXISTS message_mxid_unique")
+			if err != nil {
+				return fmt.Errorf("failed to drop potentially conflicting constraint on message: %w", err)
+			}
+			_, err = br.DB.Exec(ctx, "ALTER TABLE reaction DROP CONSTRAINT IF EXISTS reaction_mxid_unique")
+			if err != nil {
+				return fmt.Errorf("failed to drop potentially conflicting constraint on reaction: %w", err)
+			}
+		}
+		err := dbutil.DangerousInternalUpgradeVersionTable(ctx, br.DB)
+		if err != nil {
+			return err
+		}
+		_, err = br.DB.Exec(ctx, renameTablesQuery)
 		if err != nil {
 			return err
 		}
@@ -27,6 +53,22 @@ func (br *BridgeMain) LegacyMigrateSimple(renameTablesQuery, copyDataQuery strin
 		if upgradesTo < newDBVersion || compat > newDBVersion {
 			return fmt.Errorf("unexpected new database version (%d/c:%d, expected %d)", upgradesTo, compat, newDBVersion)
 		}
+		if otherTable != nil {
+			_, err = br.DB.Exec(ctx, fmt.Sprintf("CREATE TABLE %s (version INTEGER, compat INTEGER)", otherTableName))
+			if err != nil {
+				return err
+			}
+			otherUpgradesTo, otherCompat, err := otherTable[0].DangerouslyRun(ctx, br.DB)
+			if err != nil {
+				return err
+			} else if otherUpgradesTo < otherNewVersion || otherCompat > otherNewVersion {
+				return fmt.Errorf("unexpected new database version for %s (%d/c:%d, expected %d)", otherTableName, otherUpgradesTo, otherCompat, otherNewVersion)
+			}
+			_, err = br.DB.Exec(ctx, fmt.Sprintf("INSERT INTO %s (version, compat) VALUES ($1, $2)", otherTableName), otherUpgradesTo, otherCompat)
+			if err != nil {
+				return err
+			}
+		}
 		copyDataQuery, err = br.DB.Internals().FilterSQLUpgrade(bytes.Split([]byte(copyDataQuery), []byte("\n")))
 		if err != nil {
 			return err
@@ -35,11 +77,23 @@ func (br *BridgeMain) LegacyMigrateSimple(renameTablesQuery, copyDataQuery strin
 		if err != nil {
 			return err
 		}
-		_, err = br.DB.Exec(ctx, "UPDATE database_owner SET owner = $1 WHERE key = 0", br.DB.Owner)
+		_, err = br.DB.Exec(ctx, "DELETE FROM database_owner")
 		if err != nil {
 			return err
 		}
-		_, err = br.DB.Exec(ctx, "UPDATE version SET version = $1, compat = $2", upgradesTo, compat)
+		_, err = br.DB.Exec(ctx, "INSERT INTO database_owner (key, owner) VALUES (0, $1)", br.DB.Owner)
+		if err != nil {
+			return err
+		}
+		_, err = br.DB.Exec(ctx, "DELETE FROM version")
+		if err != nil {
+			return err
+		}
+		_, err = br.DB.Exec(ctx, "INSERT INTO version (version, compat) VALUES ($1, $2)", upgradesTo, compat)
+		if err != nil {
+			return err
+		}
+		_, err = br.DB.Exec(ctx, "CREATE TABLE database_was_migrated(empty INTEGER)")
 		if err != nil {
 			return err
 		}
@@ -48,7 +102,17 @@ func (br *BridgeMain) LegacyMigrateSimple(renameTablesQuery, copyDataQuery strin
 	}
 }
 
-func (br *BridgeMain) CheckLegacyDB(expectedVersion int, minBridgeVersion, firstMegaVersion string, migrator func(context.Context) error, transaction bool) {
+func (br *BridgeMain) LegacyMigrateSimple(renameTablesQuery, copyDataQuery string, newDBVersion int) func(ctx context.Context) error {
+	return br.LegacyMigrateWithAnotherUpgrader(renameTablesQuery, copyDataQuery, newDBVersion, nil, "", 0)
+}
+
+func (br *BridgeMain) CheckLegacyDB(
+	expectedVersion int,
+	minBridgeVersion,
+	firstMegaVersion string,
+	migrator func(context.Context) error,
+	transaction bool,
+) {
 	log := br.Log.With().Str("action", "migrate legacy db").Logger()
 	ctx := log.WithContext(context.Background())
 	exists, err := br.DB.TableExists(ctx, "database_owner")
@@ -59,7 +123,7 @@ func (br *BridgeMain) CheckLegacyDB(expectedVersion int, minBridgeVersion, first
 		return
 	}
 	var owner string
-	err = br.DB.QueryRow(ctx, "SELECT owner FROM database_owner WHERE key=0").Scan(&owner)
+	err = br.DB.QueryRow(ctx, "SELECT owner FROM database_owner LIMIT 1").Scan(&owner)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
 		log.Err(err).Msg("Failed to get database owner")
 		return
@@ -71,7 +135,10 @@ func (br *BridgeMain) CheckLegacyDB(expectedVersion int, minBridgeVersion, first
 	}
 	var dbVersion int
 	err = br.DB.QueryRow(ctx, "SELECT version FROM version").Scan(&dbVersion)
-	if dbVersion < expectedVersion {
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to get database version")
+		return
+	} else if dbVersion < expectedVersion {
 		log.Fatal().
 			Int("expected_version", expectedVersion).
 			Int("version", dbVersion).
@@ -96,3 +163,101 @@ func (br *BridgeMain) CheckLegacyDB(expectedVersion int, minBridgeVersion, first
 		log.Info().Msg("Successfully migrated legacy database")
 	}
 }
+
+func (br *BridgeMain) postMigrateDMPortal(ctx context.Context, portal *bridgev2.Portal) error {
+	otherUserID := portal.OtherUserID
+	if otherUserID == "" {
+		zerolog.Ctx(ctx).Warn().
+			Str("portal_id", string(portal.ID)).
+			Msg("DM portal has no other user ID")
+		return nil
+	}
+	ghost, err := br.Bridge.GetGhostByID(ctx, otherUserID)
+	if err != nil {
+		return fmt.Errorf("failed to get ghost for %s: %w", otherUserID, err)
+	}
+	mx := ghost.Intent.(*matrix.ASIntent).Matrix
+	err = br.Matrix.Bot.EnsureJoined(ctx, portal.MXID, appservice.EnsureJoinedParams{
+		BotOverride: mx.Client,
+	})
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).
+			Str("portal_id", string(portal.ID)).
+			Stringer("room_id", portal.MXID).
+			Msg("Failed to ensure bot is joined to DM")
+	}
+	pls, err := mx.PowerLevels(ctx, portal.MXID)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).
+			Str("portal_id", string(portal.ID)).
+			Stringer("room_id", portal.MXID).
+			Msg("Failed to get power levels in room")
+	} else {
+		userLevel := pls.GetUserLevel(mx.UserID)
+		pls.EnsureUserLevel(br.Matrix.Bot.UserID, userLevel)
+		if userLevel > 50 {
+			pls.SetUserLevel(mx.UserID, 50)
+		}
+		_, err = mx.SetPowerLevels(ctx, portal.MXID, pls)
+		if err != nil {
+			zerolog.Ctx(ctx).Err(err).
+				Str("portal_id", string(portal.ID)).
+				Stringer("room_id", portal.MXID).
+				Msg("Failed to set power levels")
+		}
+	}
+	portal.UpdateInfoFromGhost(ctx, ghost)
+	return nil
+}
+
+func (br *BridgeMain) PostMigrate(ctx context.Context) error {
+	log := br.Log.With().Str("action", "post-migrate").Logger()
+	wasMigrated, err := br.DB.TableExists(ctx, "database_was_migrated")
+	if err != nil {
+		return fmt.Errorf("failed to check if database_was_migrated table exists: %w", err)
+	} else if !wasMigrated {
+		return nil
+	}
+	log.Info().Msg("Doing post-migration updates to Matrix rooms")
+
+	portals, err := br.Bridge.GetAllPortalsWithMXID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get all portals: %w", err)
+	}
+	for _, portal := range portals {
+		log := log.With().
+			Stringer("room_id", portal.MXID).
+			Object("portal_key", portal.PortalKey).
+			Str("room_type", string(portal.RoomType)).
+			Logger()
+		log.Debug().Msg("Migrating portal")
+		if br.PostMigratePortal != nil {
+			err = br.PostMigratePortal(ctx, portal)
+			if err != nil {
+				log.Err(err).Msg("Failed to run post-migrate portal hook")
+				continue
+			}
+		} else {
+			switch portal.RoomType {
+			case database.RoomTypeDM:
+				err = br.postMigrateDMPortal(ctx, portal)
+				if err != nil {
+					return fmt.Errorf("failed to update DM portal %s: %w", portal.MXID, err)
+				}
+			}
+		}
+		_, err = br.Matrix.Bot.SendStateEvent(ctx, portal.MXID, event.StateElementFunctionalMembers, "", &event.ElementFunctionalMembersContent{
+			ServiceMembers: []id.UserID{br.Matrix.Bot.UserID},
+		})
+		if err != nil {
+			log.Warn().Err(err).Stringer("room_id", portal.MXID).Msg("Failed to set service members")
+		}
+	}
+
+	_, err = br.DB.Exec(ctx, "DROP TABLE database_was_migrated")
+	if err != nil {
+		return fmt.Errorf("failed to drop database_was_migrated table: %w", err)
+	}
+	log.Info().Msg("Post-migration updates complete")
+	return nil
+}

bridgev2/matrix/mxmain/main.go

@@ -8,6 +8,7 @@
 package mxmain
 
 import (
+	"context"
 	_ "embed"
 	"encoding/json"
 	"errors"
@@ -25,6 +26,7 @@ import (
 	"go.mau.fi/util/dbutil"
 	"go.mau.fi/util/exerrors"
 	"go.mau.fi/util/exzerolog"
+	"go.mau.fi/util/progver"
 	"gopkg.in/yaml.v3"
 	flag "maunium.net/go/mauflag"
 
@@ -61,11 +63,18 @@ type BridgeMain struct {
 	// git tag to see if the built version is the release or a dev build.
 	// You can either bump this right after a release or right before, as long as it matches on the release commit.
 	Version string
+	// SemCalVer defines whether this bridge uses a mix of semantic and calendar versioning,
+	// such that the Version field is YY.0M.patch, while git tags are major.YY0M.patch.
+	SemCalVer bool
 
 	// PostInit is a function that will be called after the bridge has been initialized but before it is started.
 	PostInit  func()
 	PostStart func()
 
+	// PostMigratePortal is a function that will be called during a legacy
+	// migration for each portal.
+	PostMigratePortal func(context.Context, *bridgev2.Portal) error
+
 	// Connector is the network connector for the bridge.
 	Connector bridgev2.NetworkConnector
 
@@ -81,11 +90,7 @@ type BridgeMain struct {
 	RegistrationPath string
 	SaveConfig       bool
 
-	baseVersion      string
-	commit           string
-	LinkifiedVersion string
-	VersionDesc      string
-	BuildTime        time.Time
+	ver progver.ProgramVersion
 
 	AdditionalShortFlags string
 	AdditionalLongFlags  string
@@ -94,14 +99,7 @@ type BridgeMain struct {
 }
 
 type VersionJSONOutput struct {
-	Name string
-	URL  string
-
-	Version          string
-	IsRelease        bool
-	Commit           string
-	FormattedVersion string
-	BuildTime        time.Time
+	progver.ProgramVersion
 
 	OS   string
 	Arch string
@@ -142,18 +140,11 @@ func (br *BridgeMain) PreInit() {
 		flag.PrintHelp()
 		os.Exit(0)
 	} else if *version {
-		fmt.Println(br.VersionDesc)
+		fmt.Println(br.ver.VersionDescription)
 		os.Exit(0)
 	} else if *versionJSON {
 		output := VersionJSONOutput{
-			URL:  br.URL,
-			Name: br.Name,
-
-			Version:          br.baseVersion,
-			IsRelease:        br.Version == br.baseVersion,
-			Commit:           br.commit,
-			FormattedVersion: br.Version,
-			BuildTime:        br.BuildTime,
+			ProgramVersion: br.ver,
 
 			OS:   runtime.GOOS,
 			Arch: runtime.GOARCH,
@@ -163,13 +154,20 @@ func (br *BridgeMain) PreInit() {
 		_ = json.NewEncoder(os.Stdout).Encode(output)
 		os.Exit(0)
 	} else if *writeExampleConfig {
-		if _, err = os.Stat(*configPath); !errors.Is(err, os.ErrNotExist) {
-			_, _ = fmt.Fprintln(os.Stderr, *configPath, "already exists, please remove it if you want to generate a new example")
-			os.Exit(1)
+		if *configPath != "-" && *configPath != "/dev/stdout" && *configPath != "/dev/stderr" {
+			if _, err = os.Stat(*configPath); !errors.Is(err, os.ErrNotExist) {
+				_, _ = fmt.Fprintln(os.Stderr, *configPath, "already exists, please remove it if you want to generate a new example")
+				os.Exit(1)
+			}
 		}
 		networkExample, _, _ := br.Connector.GetConfig()
-		exerrors.PanicIfNotNil(os.WriteFile(*configPath, []byte(br.makeFullExampleConfig(networkExample)), 0600))
-		fmt.Println("Wrote example config to", *configPath)
+		fullCfg := br.makeFullExampleConfig(networkExample)
+		if *configPath == "-" {
+			fmt.Print(fullCfg)
+		} else {
+			exerrors.PanicIfNotNil(os.WriteFile(*configPath, []byte(fullCfg), 0600))
+			fmt.Println("Wrote example config to", *configPath)
+		}
 		os.Exit(0)
 	}
 	br.LoadConfig()
@@ -228,8 +226,8 @@ func (br *BridgeMain) Init() {
 
 	br.Log.Info().
 		Str("name", br.Name).
-		Str("version", br.Version).
-		Time("built_at", br.BuildTime).
+		Str("version", br.ver.FormattedVersion).
+		Time("built_at", br.ver.BuildTime).
 		Str("go_version", runtime.Version()).
 		Msg("Initializing bridge")
 
@@ -243,7 +241,7 @@ func (br *BridgeMain) Init() {
 	br.Matrix.AS.DoublePuppetValue = br.Name
 	br.Bridge.Commands.(*commands.Processor).AddHandler(&commands.FullHandler{
 		Func: func(ce *commands.Event) {
-			ce.Reply("[%s](%s) %s (%s)", br.Name, br.URL, br.LinkifiedVersion, br.BuildTime.Format(time.RFC1123))
+			ce.Reply(br.ver.MarkdownDescription())
 		},
 		Name: "version",
 		Help: commands.HelpMeta{
@@ -259,6 +257,10 @@ func (br *BridgeMain) Init() {
 func (br *BridgeMain) initDB() {
 	br.Log.Debug().Msg("Initializing database connection")
 	dbConfig := br.Config.Database
+	if dbConfig.Type == "sqlite3" {
+		br.Log.WithLevel(zerolog.FatalLevel).Msg("Invalid database type sqlite3. Use sqlite3-fk-wal instead.")
+		os.Exit(14)
+	}
 	if (dbConfig.Type == "sqlite3-fk-wal" || dbConfig.Type == "litestream") && dbConfig.MaxOpenConns != 1 && !strings.Contains(dbConfig.URI, "_txlock=immediate") {
 		var fixedExampleURI string
 		if !strings.HasPrefix(dbConfig.URI, "file:") {
@@ -298,7 +300,7 @@ func (br *BridgeMain) validateConfig() error {
 	case br.Config.AppService.HSToken == "This value is generated when generating the registration":
 		return errors.New("appservice.hs_token not configured. Did you forget to generate the registration? ")
 	case br.Config.Database.URI == "postgres://user:password@host/database?sslmode=disable":
-		return errors.New("appservice.database not configured")
+		return errors.New("database.uri not configured")
 	case !br.Config.Bridge.Permissions.IsConfigured():
 		return errors.New("bridge.permissions not configured")
 	case !strings.Contains(br.Config.AppService.FormatUsername("1234567890"), "1234567890"):
@@ -351,13 +353,22 @@ func (br *BridgeMain) LoadConfig() {
 			os.Exit(10)
 		}
 	}
+	cfg.Bridge.Backfill = cfg.Backfill
+	if cfg.EnvConfigPrefix != "" {
+		err = UpdateConfigFromEnv(&cfg, networkData, cfg.EnvConfigPrefix)
+		if err != nil {
+			_, _ = fmt.Fprintln(os.Stderr, "Failed to parse environment variables:", err)
+			os.Exit(10)
+		}
+	}
 	br.Config = &cfg
 }
 
 // Start starts the bridge after everything has been initialized.
 // This is called by [Run] and does not need to be called manually.
 func (br *BridgeMain) Start() {
-	err := br.Bridge.Start()
+	ctx := br.Log.WithContext(context.Background())
+	err := br.Bridge.StartConnectors(ctx)
 	if err != nil {
 		var dbUpgradeErr bridgev2.DBUpgradeError
 		if errors.As(err, &dbUpgradeErr) {
@@ -366,6 +377,15 @@ func (br *BridgeMain) Start() {
 			br.Log.Fatal().Err(err).Msg("Failed to start bridge")
 		}
 	}
+	err = br.PostMigrate(ctx)
+	if err != nil {
+		br.Log.Fatal().Err(err).Msg("Failed to run post-migration updates")
+	}
+	err = br.Bridge.StartLogins(ctx)
+	if err != nil {
+		br.Log.Fatal().Err(err).Msg("Failed to start existing user logins")
+	}
+	br.Bridge.PostStart(ctx)
 	if br.PostStart != nil {
 		br.PostStart()
 	}
@@ -377,8 +397,10 @@ func (br *BridgeMain) WaitForInterrupt() int {
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	select {
 	case <-c:
+		br.Log.Info().Msg("Interrupt signal received from OS")
 		return 0
 	case exitCode := <-br.manualStop:
+		br.Log.Info().Msg("Internal stop signal received")
 		return exitCode
 	}
 }
@@ -392,7 +414,7 @@ func (br *BridgeMain) TriggerStop(exitCode int) {
 
 // Stop cleanly stops the bridge. This is called by [Run] and does not need to be called manually.
 func (br *BridgeMain) Stop() {
-	br.Bridge.Stop()
+	br.Bridge.StopWithTimeout(5 * time.Second)
 }
 
 // InitVersion formats the bridge version and build time nicely for things like
@@ -417,42 +439,12 @@ func (br *BridgeMain) Stop() {
 //
 // (to use both at the same time, simply merge the ldflags into one, `-ldflags "-X '...' -X ..."`)
 func (br *BridgeMain) InitVersion(tag, commit, rawBuildTime string) {
-	br.baseVersion = br.Version
-	if len(tag) > 0 && tag[0] == 'v' {
-		tag = tag[1:]
-	}
-	if tag != br.Version {
-		suffix := ""
-		if !strings.HasSuffix(br.Version, "+dev") {
-			suffix = "+dev"
-		}
-		if len(commit) > 8 {
-			br.Version = fmt.Sprintf("%s%s.%s", br.Version, suffix, commit[:8])
-		} else {
-			br.Version = fmt.Sprintf("%s%s.unknown", br.Version, suffix)
-		}
-	}
-
-	br.LinkifiedVersion = fmt.Sprintf("v%s", br.Version)
-	if tag == br.Version {
-		br.LinkifiedVersion = fmt.Sprintf("[v%s](%s/releases/v%s)", br.Version, br.URL, tag)
-	} else if len(commit) > 8 {
-		br.LinkifiedVersion = strings.Replace(br.LinkifiedVersion, commit[:8], fmt.Sprintf("[%s](%s/commit/%s)", commit[:8], br.URL, commit), 1)
-	}
-	var buildTime time.Time
-	if rawBuildTime != "unknown" {
-		buildTime, _ = time.Parse(time.RFC3339, rawBuildTime)
-	}
-	var builtWith string
-	if buildTime.IsZero() {
-		rawBuildTime = "unknown"
-		builtWith = runtime.Version()
-	} else {
-		rawBuildTime = buildTime.Format(time.RFC1123)
-		builtWith = fmt.Sprintf("built at %s with %s", rawBuildTime, runtime.Version())
-	}
-	mautrix.DefaultUserAgent = fmt.Sprintf("%s/%s %s", br.Name, br.Version, mautrix.DefaultUserAgent)
-	br.VersionDesc = fmt.Sprintf("%s %s (%s)", br.Name, br.Version, builtWith)
-	br.commit = commit
-	br.BuildTime = buildTime
+	br.ver = progver.ProgramVersion{
+		Name:        br.Name,
+		URL:         br.URL,
+		BaseVersion: br.Version,
+		SemCalVer:   br.SemCalVer,
+	}.Init(tag, commit, rawBuildTime)
+	mautrix.DefaultUserAgent = fmt.Sprintf("%s/%s %s", br.Name, br.ver.FormattedVersion, mautrix.DefaultUserAgent)
+	br.Version = br.ver.FormattedVersion
 }

bridgev2/matrix/no-crypto.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tulir Asokan
+// Copyright (c) 2024 Tulir Asokan
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -10,15 +10,13 @@ package matrix
 
 import (
 	"errors"
-
-	"maunium.net/go/mautrix/bridge"
 )
 
-func NewCryptoHelper(bridge *bridge.Bridge) bridge.Crypto {
-	if bridge.Config.Bridge.GetEncryptionConfig().Allow {
-		bridge.ZLog.Warn().Msg("Bridge built without end-to-bridge encryption, but encryption is enabled in config")
+func NewCryptoHelper(c *Connector) Crypto {
+	if c.Config.Encryption.Allow {
+		c.Log.Warn().Msg("Bridge built without end-to-bridge encryption, but encryption is enabled in config")
 	} else {
-		bridge.ZLog.Debug().Msg("Bridge built without end-to-bridge encryption")
+		c.Log.Debug().Msg("Bridge built without end-to-bridge encryption")
 	}
 	return nil
 }