support timestamped appservice messaging

Co-authored-by: strawberry <strawberry@puppygock.gay>
Signed-off-by: strawberry <strawberry@puppygock.gay>

Cargo.toml

@@ -19,12 +19,15 @@ license = "Apache-2.0"
 # See also `rust-toolchain.toml`
 readme = "README.md"
 repository = "https://github.com/girlbossceo/conduwuit"
-rust-version = "1.80.0"
+rust-version = "1.80.1"
 version = "0.4.6"
 
 [workspace.metadata.crane]
 name = "conduit"
 
+[workspace.dependencies.arrayvec]
+version = "0.7.4"
+
 [workspace.dependencies.const-str]
 version = "0.5.7"
 
@@ -66,7 +69,7 @@ version = "0.8.5"
 
 # Used for the http request / response body type for Ruma endpoints used with reqwest
 [workspace.dependencies.bytes]
-version = "1.6.1"
+version = "1.7.1"
 
 [workspace.dependencies.http-body-util]
 version = "0.1.1"
@@ -75,7 +78,7 @@ version = "0.1.1"
 version = "1.1.0"
 
 [workspace.dependencies.regex]
-version = "1.10.4"
+version = "1.10.6"
 
 [workspace.dependencies.axum]
 version = "0.7.5"
@@ -96,7 +99,7 @@ default-features = false
 features = ["typed-header", "tracing"]
 
 [workspace.dependencies.axum-server]
-version = "0.6.0"
+version = "0.7.1"
 default-features = false
 features = ["tls-rustls"]
 
@@ -104,7 +107,7 @@ features = ["tls-rustls"]
 version = "0.6.0"
 
 [workspace.dependencies.tower]
-version = "0.4.13"
+version = "0.5.0"
 default-features = false
 features = ["util"]
 
@@ -137,7 +140,7 @@ default-features = false
 features = ["rc"]
 
 [workspace.dependencies.serde_json]
-version = "1.0.121"
+version = "1.0.124"
 default-features = false
 features = ["raw_value"]
 
@@ -190,7 +193,7 @@ default-features = false
 
 # used for conduit's CLI and admin room command parsing
 [workspace.dependencies.clap]
-version = "4.5.9"
+version = "4.5.15"
 default-features = false
 features = [
 	"std",
@@ -304,7 +307,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://github.com/girlbossceo/ruwuma"
 #branch = "conduwuit-changes"
-rev = "c76e2873c1593a3308d4ba3e0e4a1db65acf8536"
+rev = "11cbd52af1039f3d4a0b96333acd1d1687ba01aa"
 features = [
     "compat",
     "rand",
@@ -428,11 +431,11 @@ features = [
 ]
 
 [workspace.dependencies.rustyline-async]
-version = "0.4.2"
+version = "0.4.3"
 default-features = false
 
 [workspace.dependencies.termimad]
-version = "0.29.4"
+version = "0.30.0"
 default-features = false
 
 [workspace.dependencies.checked_ops]
@@ -470,17 +473,11 @@ rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
 git = "https://github.com/girlbossceo/tracing"
 rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
 
-# fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114]
-# https://github.com/girlbossceo/axum-server/commit/8e3368d899079818934e61cc9c839abcbbcada8a
-[patch.crates-io.axum-server]
-git = "https://github.com/girlbossceo/axum-server"
-rev = "8e3368d899079818934e61cc9c839abcbbcada8a"
-
 # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50
 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b
 [patch.crates-io.rustyline-async]
 git = "https://github.com/girlbossceo/rustyline-async"
-rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50"
+rev = "9654cc84e19241f6e19021eb8e677892656f5071"
 
 #
 # Our crates

conduwuit-example.toml

@@ -224,9 +224,11 @@ registration_token = "change this token for something specific to your server"
 # No default.
 # forbidden_alias_names = []
 
-# List of forbidden server names that we will block all client room joins, incoming federated room directory requests, incoming federated invites for, and incoming federated joins. This check is applied on the room ID, room alias, sender server name, and sender user's server name.
+# List of forbidden server names that we will block incoming AND outgoing federation with, and block client room joins / remote user invites.
-# Basically "global" ACLs. For our user (client) checks, admin users are allowed.
+#
-# No default.
+# This check is applied on the room ID, room alias, sender server name, sender user's server name, inbound federation X-Matrix origin, and outbound federation handler.
+#
+# Basically "global" ACLs. No default.
 # forbidden_remote_server_names = []
 
 # List of forbidden server names that we will block all outgoing federated room directory requests for. Useful for preventing our users from wandering into bad servers or spaces.
@@ -514,6 +516,31 @@ allow_profile_lookup_federation_requests = true
 # Defaults to false as this uses more CPU when compressing.
 #rocksdb_bottommost_compression = false
 
+# Level of statistics collection. Some admin commands to display database statistics may require
+# this option to be set. Database performance may be impacted by higher settings.
+#
+# Option is a number ranging from 0 to 6:
+# 0 = No statistics.
+# 1 = No statistics in release mode (default).
+# 2 to 3 = Statistics with no performance impact.
+# 3 to 5 = Statistics with possible performance impact.
+# 6 = All statistics.
+#
+# Defaults to 1 (No statistics, except in debug-mode)
+#rocksdb_stats_level = 1
+
+# Database repair mode (for RocksDB SST corruption)
+#
+# Use this option when the server reports corruption while running or panics. If the server refuses
+# to start use the recovery mode options first. Corruption errors containing the acronym 'SST' which
+# occur after startup will likely require this option.
+#
+# - Backing up your database directory is recommended prior to running the repair.
+# - Disabling repair mode and restarting the server is recommended after running the repair.
+#
+# Defaults to false
+#rocksdb_repair = false
+
 # Database recovery mode (for RocksDB WAL corruption)
 #
 # Use this option when the server reports corruption and refuses to start. Set mode 2 (PointInTime)

debian/postinst

@@ -16,7 +16,6 @@ case "$1" in
         --home "$CONDUWUIT_DATABASE_PATH" \
         --disabled-login \
         --shell "/usr/sbin/nologin" \
-        --verbose \
         conduwuit
     fi
 

docs/SUMMARY.md

@@ -10,6 +10,7 @@
   - [Docker](deploying/docker.md)
   - [Arch Linux](deploying/arch-linux.md)
   - [Debian](deploying/debian.md)
+  - [FreeBSD](deploying/freebsd.md)
 - [TURN](turn.md)
 - [Appservices](appservices.md)
 - [Maintenance](maintenance.md)

docs/configuration.md

@@ -4,15 +4,35 @@ This chapter describes various ways to configure conduwuit.
 
 ## Basics
 
-Conduwuit uses a config file for the majority of the settings. Please refer to the
+conduwuit uses a config file for the majority of the settings, but also supports setting individual config options via commandline.
-[example config file](./configuration/examples.md#example-configuration) for all of those settings.
+
-The config file to use can either be specified on the command line when running conduwuit by specifying the
+Please refer to the [example config file](./configuration/examples.md#example-configuration) for all of those settings.
+
+The config file to use can be specified on the commandline when running conduwuit by specifying the
 `-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config
-file to used.
+file to used. Conduit's environment variables are supported for backwards compatibility.
+
+## Option commandline flag
+
+conduwuit supports setting individual config options in TOML format from the `-O` / `--option` flag. For example, you can set your server name via `-O server_name=\"example.com\"`.
+
+Note that the config is parsed as TOML, and shells like bash will remove quotes. So unfortunately it is required to escape quotes if the config option takes a string.
+This does not apply to options that take booleans or numbers:
+- `--option allow_registration=true` works ✅
+- `-O max_request_size=99999999` works ✅
+- `-O server_name=example.com` does not work ❌
+- `--option log=\"debug\"` works ✅
+- `--option server_name='"example.com'"` works ✅
+
 
 ## Environment variables
 
 All of the settings that are found in the config file can be specified by using environment variables.
 The environment variable names should be all caps and prefixed with `CONDUWUIT_`.
+
 For example, if the setting you are changing is `max_request_size`, then the environment variable to set is
 `CONDUWUIT_MAX_REQUEST_SIZE`.
+
+To modify config options not in the `[global]` context such as `[global.well_known]`, use the `__` suffix split: `CONDUWUIT_WELL_KNOWN__SERVER`
+
+Conduit's environment variables are supported for backwards compatibility (e.g. `CONDUIT_SERVER_NAME`).

docs/deploying/docker.md

@@ -17,9 +17,9 @@ OCI images for conduwuit are available in the registries listed below.
 | GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl]   | ![Image Size][shield-main]    | Stable main branch.   |
 | Docker Hub      | [docker.io/girlbossceo/conduwuit:main][dh]               | ![Image Size][shield-main]    | Stable main branch.   |
 
-[dh]: https://hub.docker.com/repository/docker/girlbossceo/conduwuit
+[dh]: https://hub.docker.com/r/girlbossceo/conduwuit
 [gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit
-[gl]: https://gitlab.com/conduwuit/conduwuit/container_registry/6351657
+[gl]: https://gitlab.com/conduwuit/conduwuit/container_registry/6369729
 [shield-latest]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest
 [shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main
 

docs/deploying/freebsd.md

@@ -0,0 +1,8 @@
+# conduwuit for FreeBSD
+
+conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on FreeBSD requires a specific environment variable to use the
+system prebuilt RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* work and will cause a build error or coredump.
+
+Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib`
+
+Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo build --release`

docs/deploying/generic.md

@@ -9,14 +9,12 @@
 
 You may simply download the binary that fits your machine. Run `uname -m` to see what you need.
 
-Prebuilt binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest).
+Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or `main` CI branch workflow artifact output. These also include Debian packages. These binaries have jemalloc and io_uring statically linked and included with them.
-
-The latest tagged release also includes the Debian packages.
 
 Alternatively, you may compile the binary yourself. We recommend using [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed
 reproducibiltiy and easiest to get a build environment and output going.
 
-Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang, installing liburing for io_uring and RocksDB, etc).
+Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang for RocksDB, installing liburing for io_uring and RocksDB, etc).
 
 ## Adding a conduwuit user
 

docs/troubleshooting.md

@@ -8,6 +8,17 @@
 >
 > If there are things like Compose file issues or Dockerhub image issues, those can still be mentioned as long as they're something we can fix.
 
+## General potential issues
+
+#### Potential DNS issues when using Docker
+
+Docker has issues with its default DNS setup that may cause DNS to not be properly functional when running conduwuit, resulting in federation issues.
+The symptoms of this have shown in excessively long room joins (30+ minutes) from very long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation.
+
+This is **not** a conduwuit issue, and is purely a Docker issue. It is not sustainable for heavy DNS activity which is normal for Matrix federation. The workarounds for this are:
+- Use DNS over TCP via the config option `query_over_tcp_only = true`
+- Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`)
+
 ## Rocksdb / database issues
 
 #### Direct IO
@@ -38,8 +49,6 @@ With this in mind:
 - Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored
 - If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database
 
-## Media
-
 ## Debugging
 
 Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`.

flake.lock

@@ -9,11 +9,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1720542474,
+        "lastModified": 1723558887,
-        "narHash": "sha256-aKjJ/4l2I9+wNGTaOGRsuS3M1+IoTibqgEMPDikXm04=",
+        "narHash": "sha256-FBaRmikNnCX+HQJXAfNJyiOl3sM1NrlxvcFwQDqwCyk=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "6139576a3ce6bb992e0f6c3022528ec233e45f00",
+        "rev": "26b9417bde03edc6280d1f7ce709cd619cdb72d4",
         "type": "github"
       },
       "original": {
@@ -27,15 +27,15 @@
       "inputs": {
         "devenv": "devenv",
         "flake-compat": "flake-compat_3",
-        "nixpkgs": "nixpkgs_3",
+        "git-hooks": "git-hooks",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "nixpkgs": "nixpkgs_3"
       },
       "locked": {
-        "lastModified": 1719923519,
+        "lastModified": 1723252655,
-        "narHash": "sha256-7Rhljj2fsklFRsu+eq7N683Z9qukmreMEj5C1GqCrSA=",
+        "narHash": "sha256-uOxcrNEMTOHvGHYyqDWnUpkhkY29l01KhjkgVQU2gXg=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "4e9e71f78b9500fa6210cf1eaa4d75bdbab777c3",
+        "rev": "bd558fd32f55d6df1c3d496bade90335a1a6044d",
         "type": "github"
       },
       "original": {
@@ -81,11 +81,11 @@
     "complement": {
       "flake": false,
       "locked": {
-        "lastModified": 1720637557,
+        "lastModified": 1722323564,
-        "narHash": "sha256-oZz6nCmFmdJZpC+K1iOG2KkzTI6rlAmndxANPDVU7X0=",
+        "narHash": "sha256-6w6/N8walz4Ayc9zu7iySqJRmGFukhkaICLn4dweAcA=",
         "owner": "matrix-org",
         "repo": "complement",
-        "rev": "0d14432e010482ea9e13a6f7c47c1533c0c9d62f",
+        "rev": "6e4426a9e63233f9821a4d2382bfed145244183f",
         "type": "github"
       },
       "original": {
@@ -123,11 +123,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720546058,
+        "lastModified": 1722960479,
-        "narHash": "sha256-iU2yVaPIZm5vMGdlT0+57vdB/aPq/V5oZFBRwYw+HBM=",
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "2d83156f23c43598cf44e152c33a59d3892f8b29",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
         "type": "github"
       },
       "original": {
@@ -151,15 +151,15 @@
         ],
         "pre-commit-hooks": [
           "cachix",
-          "pre-commit-hooks"
+          "git-hooks"
         ]
       },
       "locked": {
-        "lastModified": 1719759336,
+        "lastModified": 1723156315,
-        "narHash": "sha256-3a34VL/QnHprl5gMy9xlx6d8J+iNp+W88Ex8smkgH9M=",
+        "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "bb32aa986f2f695385e54428d0eaf7d05b31466e",
+        "rev": "ff5eb4f2accbcda963af67f1a1159e3f6c7f5f91",
         "type": "github"
       },
       "original": {
@@ -209,11 +209,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1720852044,
+        "lastModified": 1723530607,
-        "narHash": "sha256-3NBYz8VuXuKU+8ONd9NFafCNjPEGHIZQ2Mdoam1a4mY=",
+        "narHash": "sha256-FaXZZLLDW1D+pj7UgrIslDS8XjMMG3Pus5gAvUYWQS0=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "5087b12a595ee73131a944d922f24d81dae05725",
+        "rev": "296d44c440302980824c5f3b67e477cf0522e0c1",
         "type": "github"
       },
       "original": {
@@ -272,22 +272,6 @@
       }
     },
     "flake-compat_4": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1696426674,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-compat_5": {
       "flake": false,
       "locked": {
         "lastModified": 1696426674,
@@ -356,11 +340,38 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "cachix",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "cachix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable_2"
+      },
+      "locked": {
+        "lastModified": 1723202784,
+        "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "c7012d0c18567c889b948781bc74a501e92275d1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
           "cachix",
-          "pre-commit-hooks",
+          "git-hooks",
           "nixpkgs"
         ]
       },
@@ -381,11 +392,11 @@
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1720798442,
+        "lastModified": 1723553052,
-        "narHash": "sha256-gtPppAoksMLW4GuruQ36nf4EAqIA1Bs6V9Xcx8dBxrQ=",
+        "narHash": "sha256-sjqmYP/tvQKmsTD6FOd8m7aEja2hhEyyB3aTxVXEbuE=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "1d674f83b7d0f07553ac44d99a401b05853d9dbe",
+        "rev": "401b3e4bde2316d35e93b548269c72bbb64adc02",
         "type": "github"
       },
       "original": {
@@ -558,16 +569,16 @@
     },
     "nixpkgs-stable_2": {
       "locked": {
-        "lastModified": 1718811006,
+        "lastModified": 1720386169,
-        "narHash": "sha256-0Y8IrGhRmBmT7HHXlxxepg2t8j1X90++qRN3lukGaIk=",
+        "narHash": "sha256-NGKVY4PjzwAa4upkGtAMz1npHGoRzWotlSnVlqI40mo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "03d771e513ce90147b65fe922d87d3a0356fc125",
+        "rev": "194846768975b7ad2c4988bdb82572c00222c0d7",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -590,11 +601,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1719848872,
+        "lastModified": 1722813957,
-        "narHash": "sha256-H3+EC5cYuq+gQW8y0lSrrDZfH71LB4DAf+TDFyvwCNA=",
+        "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "00d80d13810dbfea8ab4ed1009b09100cca86ba8",
+        "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa",
         "type": "github"
       },
       "original": {
@@ -606,11 +617,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1720768451,
+        "lastModified": 1723362943,
-        "narHash": "sha256-EYekUHJE2gxeo2pM/zM9Wlqw1Uw2XTJXOSAO79ksc4Y=",
+        "narHash": "sha256-dFZRVSgmJkyM0bkPpaYRtG/kRMRTorUIDj8BxoOt1T4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9",
+        "rev": "a58bc8ad779655e790115244571758e8de055e3d",
         "type": "github"
       },
       "original": {
@@ -646,30 +657,6 @@
         "type": "github"
       }
     },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat_4",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "cachix",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": "nixpkgs-stable_2"
-      },
-      "locked": {
-        "lastModified": 1719259945,
-        "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=",
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "rocksdb": {
       "flake": false,
       "locked": {
@@ -694,7 +681,7 @@
         "complement": "complement",
         "crane": "crane_2",
         "fenix": "fenix",
-        "flake-compat": "flake-compat_5",
+        "flake-compat": "flake-compat_4",
         "flake-utils": "flake-utils_3",
         "liburing": "liburing",
         "nix-filter": "nix-filter",
@@ -705,11 +692,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1720717809,
+        "lastModified": 1723473250,
-        "narHash": "sha256-6I+fm+nTLF/iaj7ffiFGlSY7POmubwUaPA/Wq0Bm53M=",
+        "narHash": "sha256-Ls0e6R4FmGUFXZlUcm6ZQaVNJ4Yj/nua4SSctXIopao=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "ffbc5ad993d5cd2f3b8bcf9a511165470944ab91",
+        "rev": "32a86cb1dad2b208e8f36f1bb50c2e4806b0371f",
         "type": "github"
       },
       "original": {

flake.nix

@@ -24,7 +24,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-6eN/GKzjVSjEhGO9FhWObkRFaE1Jf+uqMSdQnb8lcB4=";
+        sha256 = "sha256-3jVIIf5XPnUU1CRaTyAiO0XHVbJl12MSx3eucTXCjtE=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
@@ -137,7 +137,11 @@
           # Useful for editing the book locally
           mdbook
 
+          # used for rust caching in CI to speed it up
           sccache
+
+          # needed so we can get rid of gcc and other unused deps that bloat OCI images
+          removeReferencesTo
         ])
         ++ scope.main.buildInputs
         ++ scope.main.propagatedBuildInputs

nix/pkgs/main/default.nix

@@ -7,6 +7,7 @@
 , liburing
 , pkgsBuildHost
 , rocksdb
+, removeReferencesTo
 , rust
 , rust-jemalloc-sys
 , stdenv
@@ -94,8 +95,8 @@ buildDepsOnlyEnv =
         else if stdenv.targetPlatform.isAarch64
         then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
         ++ lib.optionals stdenv.targetPlatform.isAarch64 [
-          # cortex-a55 == ARMv8.2-a
+          # cortex-a73 == ARMv8-A
-          "-DPORTABLE=armv8.2-a"
+          "-DPORTABLE=armv8-a"
         ]
         else old.cmakeFlags;
     });
@@ -128,7 +129,7 @@ buildPackageEnv = {
     + lib.optionalString stdenv.targetPlatform.isx86_64
       " -Ctarget-cpu=x86-64-v2"
     + lib.optionalString stdenv.targetPlatform.isAarch64
-      " -Ctarget-cpu=cortex-a55"; # cortex-a55 == ARMv8.2-a
+      " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A
 };
 
 
@@ -154,6 +155,7 @@ commonAttrs = {
     };
 
     dontStrip = profile == "dev" || profile == "test";
+    dontPatchELF = profile == "dev" || profile == "test";
 
     buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
 
@@ -168,6 +170,9 @@ commonAttrs = {
       # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
       # rebuilds of bindgen and its depedents.
       jq
+
+      # needed so we can get rid of gcc and other unused deps that bloat OCI images
+      removeReferencesTo
   ]
   ++ lib.optionals stdenv.isDarwin [
       # https://github.com/NixOS/nixpkgs/issues/206242
@@ -177,6 +182,13 @@ commonAttrs = {
       # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
       pkgsBuildHost.darwin.apple_sdk.frameworks.Security
     ];
+
+    # for some reason gcc and other weird deps are added to OCI images and bloats it up
+    #
+    # <https://github.com/input-output-hk/haskell.nix/issues/829>
+    postInstall = with pkgsBuildHost; ''
+        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' +
+    '';
  };
 in
 

rust-toolchain.toml

@@ -11,7 +11,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.80.0"
+channel = "1.80.1"
 components = [
     # For rust-analyzer
     "rust-src",

src/admin/command.rs

@@ -1,6 +1,9 @@
-use service::Services;
+use std::time::SystemTime;
+
+use conduit_service::Services;
 
 pub(crate) struct Command<'a> {
 	pub(crate) services: &'a Services,
 	pub(crate) body: &'a [&'a str],
+	pub(crate) timer: SystemTime,
 }

src/admin/debug/commands.rs

@@ -1,16 +1,12 @@
 use std::{
 	collections::{BTreeMap, HashMap},
 	fmt::Write,
-	sync::{Arc, Mutex},
+	sync::Arc,
 	time::{Instant, SystemTime},
 };
 
 use api::client::validate_and_add_event_id;
-use conduit::{
+use conduit::{debug, debug_error, err, info, trace, utils, warn, Error, PduEvent, Result};
-	debug, debug_error, err, info, log,
-	log::{capture, Capture},
-	utils, warn, Error, PduEvent, Result,
-};
 use ruma::{
 	api::{client::error::ErrorKind, federation::event::get_room_state},
 	events::room::message::RoomMessageEventContent,
@@ -149,23 +145,32 @@ pub(super) async fn get_remote_pdu_list(
 		.filter_map(|pdu| EventId::parse(pdu).ok())
 		.collect::<Vec<_>>();
 
+	let mut failed_count: usize = 0;
+	let mut success_count: usize = 0;
+
 	for pdu in list {
 		if force {
 			if let Err(e) = self.get_remote_pdu(Box::from(pdu), server.clone()).await {
+				failed_count = failed_count.saturating_add(1);
 				self.services
 					.admin
 					.send_message(RoomMessageEventContent::text_plain(format!(
 						"Failed to get remote PDU, ignoring error: {e}"
 					)))
 					.await;
-				warn!(%e, "Failed to get remote PDU, ignoring error");
+				warn!("Failed to get remote PDU, ignoring error: {e}");
+			} else {
+				success_count = success_count.saturating_add(1);
 			}
 		} else {
 			self.get_remote_pdu(Box::from(pdu), server.clone()).await?;
+			success_count = success_count.saturating_add(1);
 		}
 	}
 
-	Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs."))
+	Ok(RoomMessageEventContent::text_plain(format!(
+		"Fetched {success_count} remote PDUs successfully with {failed_count} failures"
+	)))
 }
 
 #[admin_command]
@@ -204,7 +209,7 @@ pub(super) async fn get_remote_pdu(
 				Error::BadRequest(ErrorKind::Unknown, "Received response from server but failed to parse PDU")
 			})?;
 
-			debug!("Attempting to parse PDU: {:?}", &response.pdu);
+			trace!("Attempting to parse PDU: {:?}", &response.pdu);
 			let parsed_pdu = {
 				let parsed_result = self
 					.services
@@ -708,30 +713,14 @@ pub(super) async fn resolve_true_destination(
 		));
 	}
 
-	let filter: &capture::Filter = &|data| {
-		data.level() <= log::Level::DEBUG
-			&& data.mod_name().starts_with("conduit")
-			&& matches!(data.span_name(), "actual" | "well-known" | "srv")
-	};
-
-	let state = &self.services.server.log.capture;
-	let logs = Arc::new(Mutex::new(String::new()));
-	let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone()));
-
-	let capture_scope = capture.start();
 	let actual = self
 		.services
 		.resolver
 		.resolve_actual_dest(&server_name, !no_cache)
 		.await?;
-	drop(capture_scope);
 
-	let msg = format!(
+	let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host,);
-		"{}\nDestination: {}\nHostname URI: {}",
+
-		logs.lock().expect("locked"),
-		actual.dest,
-		actual.host,
-	);
 	Ok(RoomMessageEventContent::text_markdown(msg))
 }
 
@@ -819,3 +808,24 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result<RoomMessageE
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
 }
+
+#[admin_command]
+pub(super) async fn database_stats(
+	&self, property: Option<String>, map: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned());
+	let map_name = map.as_ref().map_or(utils::string::EMPTY, String::as_str);
+
+	let mut out = String::new();
+	for (name, map) in self.services.db.iter_maps() {
+		if !map_name.is_empty() && *map_name != *name {
+			continue;
+		}
+
+		let res = map.property(&property)?;
+		let res = res.trim();
+		writeln!(out, "##### {name}:\n```\n{res}\n```")?;
+	}
+
+	Ok(RoomMessageEventContent::notice_markdown(out))
+}

src/admin/debug/mod.rs

@@ -184,6 +184,14 @@ pub(super) enum DebugCommand {
 		names: bool,
 	},
 
+	/// - Get database statistics
+	DatabaseStats {
+		property: Option<String>,
+
+		#[arg(short, long, alias("column"))]
+		map: Option<String>,
+	},
+
 	/// - Developer test stubs
 	#[command(subcommand)]
 	#[allow(non_snake_case)]

src/admin/handler.rs

@@ -1,7 +1,21 @@
-use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant};
+use std::{
+	panic::AssertUnwindSafe,
+	sync::{Arc, Mutex},
+	time::SystemTime,
+};
 
 use clap::{CommandFactory, Parser};
-use conduit::{checked, error, trace, utils::string::common_prefix, Error, Result};
+use conduit::{
+	debug, error,
+	log::{
+		capture,
+		capture::Capture,
+		fmt::{markdown_table, markdown_table_head},
+	},
+	trace,
+	utils::string::{collect_stream, common_prefix},
+	Error, Result,
+};
 use futures_util::future::FutureExt;
 use ruma::{
 	events::{
@@ -11,9 +25,10 @@ use ruma::{
 	OwnedEventId,
 };
 use service::{
-	admin::{CommandInput, CommandOutput, CommandResult, HandlerResult},
+	admin::{CommandInput, CommandOutput, HandlerFuture, HandlerResult},
 	Services,
 };
+use tracing::Level;
 
 use crate::{admin, admin::AdminCommand, Command};
 
@@ -21,12 +36,12 @@ use crate::{admin, admin::AdminCommand, Command};
 pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) }
 
 #[must_use]
-pub(super) fn handle(services: Arc<Services>, command: CommandInput) -> HandlerResult {
+pub(super) fn handle(services: Arc<Services>, command: CommandInput) -> HandlerFuture {
 	Box::pin(handle_command(services, command))
 }
 
 #[tracing::instrument(skip_all, name = "admin")]
-async fn handle_command(services: Arc<Services>, command: CommandInput) -> CommandResult {
+async fn handle_command(services: Arc<Services>, command: CommandInput) -> HandlerResult {
 	AssertUnwindSafe(Box::pin(process_command(services, &command)))
 		.catch_unwind()
 		.await
@@ -34,13 +49,24 @@ async fn handle_command(services: Arc<Services>, command: CommandInput) -> Comma
 		.or_else(|error| handle_panic(&error, command))
 }
 
-async fn process_command(services: Arc<Services>, command: &CommandInput) -> CommandOutput {
+async fn process_command(services: Arc<Services>, input: &CommandInput) -> CommandOutput {
-	process(services, &command.command)
+	let (command, args, body) = match parse(&services, input) {
+		Err(error) => return error,
+		Ok(parsed) => parsed,
+	};
+
+	let context = Command {
+		services: &services,
+		body: &body,
+		timer: SystemTime::now(),
+	};
+
+	process(&context, command, &args)
 		.await
-		.and_then(|content| reply(content, command.reply_id.clone()))
+		.and_then(|content| reply(content, input.reply_id.clone()))
 }
 
-fn handle_panic(error: &Error, command: CommandInput) -> CommandResult {
+fn handle_panic(error: &Error, command: CommandInput) -> HandlerResult {
 	let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺";
 	let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}");
 	let content = RoomMessageEventContent::notice_markdown(msg);
@@ -59,69 +85,61 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option<OwnedEventId>) -
 }
 
 // Parse and process a message from the admin room
-async fn process(services: Arc<Services>, msg: &str) -> CommandOutput {
+async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> CommandOutput {
-	let lines = msg.lines().filter(|l| !l.trim().is_empty());
+	let filter: &capture::Filter =
-	let command = lines
+		&|data| data.level() <= Level::DEBUG && data.our_modules() && data.scope.contains(&"admin");
-		.clone()
+	let logs = Arc::new(Mutex::new(
-		.next()
+		collect_stream(|s| markdown_table_head(s)).expect("markdown table header"),
-		.expect("each string has at least one line");
+	));
-	let (parsed, body) = match parse_command(command) {
+
-		Ok(parsed) => parsed,
+	let capture = Capture::new(
-		Err(error) => {
+		&context.services.server.log.capture,
-			let server_name = services.globals.server_name();
+		Some(filter),
-			let message = error.replace("server.name", server_name.as_str());
+		capture::fmt(markdown_table, logs.clone()),
-			return Some(RoomMessageEventContent::notice_markdown(message));
+	);
-		},
+
+	let capture_scope = capture.start();
+	let result = Box::pin(admin::process(command, context)).await;
+	drop(capture_scope);
+
+	debug!(
+		ok = result.is_ok(),
+		elapsed = ?context.timer.elapsed(),
+		command = ?args,
+		"command processed"
+	);
+
+	let logs = logs.lock().expect("locked");
+	let output = match result {
+		Err(error) => format!("{logs}\nEncountered an error while handling the command:\n```\n{error:#?}\n```"),
+		Ok(reply) => format!("{logs}\n{}", reply.body()), //TODO: content is recreated to add logs
 	};
 
-	let body = parse_body(AdminCommand::command(), &body, lines.skip(1).collect()).expect("trailing body parsed");
+	Some(RoomMessageEventContent::notice_markdown(output))
-	let context = Command {
-		services: &services,
-		body: &body,
-	};
-	let timer = Instant::now();
-	let result = Box::pin(admin::process(parsed, &context)).await;
-	let elapsed = timer.elapsed();
-	conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}");
-	match result {
-		Ok(reply) => Some(reply),
-		Err(error) => Some(RoomMessageEventContent::notice_markdown(format!(
-			"Encountered an error while handling the command:\n```\n{error:#?}\n```"
-		))),
-	}
 }
 
 // Parse chat messages from the admin room into an AdminCommand object
-fn parse_command(command_line: &str) -> Result<(AdminCommand, Vec<String>), String> {
+fn parse<'a>(
-	let argv = parse_line(command_line);
+	services: &Arc<Services>, input: &'a CommandInput,
-	let com = AdminCommand::try_parse_from(&argv).map_err(|error| error.to_string())?;
+) -> Result<(AdminCommand, Vec<String>, Vec<&'a str>), CommandOutput> {
-	Ok((com, argv))
+	let lines = input.command.lines().filter(|line| !line.trim().is_empty());
-}
+	let command_line = lines.clone().next().expect("command missing first line");
-
+	let body = lines.skip(1).collect();
-fn parse_body<'a>(mut cmd: clap::Command, body: &'a [String], lines: Vec<&'a str>) -> Result<Vec<&'a str>> {
+	match parse_command(command_line) {
-	let mut start = 1;
+		Ok((command, args)) => Ok((command, args, body)),
-	'token: for token in body.iter().skip(1) {
+		Err(error) => {
-		let cmd_ = cmd.clone();
+			let message = error
-		for sub in cmd_.get_subcommands() {
+				.to_string()
-			if sub.get_name() == *token {
+				.replace("server.name", services.globals.server_name().as_str());
-				start = checked!(start + 1)?;
+			Err(Some(RoomMessageEventContent::notice_markdown(message)))
-				cmd = sub.clone();
+		},
-				continue 'token;
 	}
 }
 
-		// positional arguments have to be skipped too
+fn parse_command(line: &str) -> Result<(AdminCommand, Vec<String>)> {
-		let num_posargs = cmd_.get_positionals().count();
+	let argv = parse_line(line);
-		start = checked!(start + num_posargs)?;
+	let command = AdminCommand::try_parse_from(&argv)?;
-		break;
+	Ok((command, argv))
-	}
-
-	Ok(body
-		.iter()
-		.skip(start)
-		.map(String::as_str)
-		.chain(lines)
-		.collect::<Vec<&'a str>>())
 }
 
 fn complete_command(mut cmd: clap::Command, line: &str) -> String {

src/admin/room/moderation.rs

@@ -547,7 +547,7 @@ async fn list_banned_rooms(&self) -> Result<RoomMessageEventContent> {
 			rooms.reverse();
 
 			let output_plain = format!(
-				"Rooms Banned ({}):\n```\n{}```",
+				"Rooms Banned ({}):\n```\n{}\n```",
 				rooms.len(),
 				rooms
 					.iter()

src/api/client/account.rs

@@ -240,7 +240,7 @@ pub(crate) async fn register_route(
 
 	// If `new_user_displayname_suffix` is set, registration will push whatever
 	// content is set to the user's display name with a space before it
-	if !services.globals.new_user_displayname_suffix().is_empty() {
+	if !services.globals.new_user_displayname_suffix().is_empty() && body.appservice_info.is_none() {
 		write!(displayname, " {}", services.globals.config.new_user_displayname_suffix)
 			.expect("should be able to write to string buffer");
 	}
@@ -286,9 +286,13 @@ pub(crate) async fn register_route(
 	let token = utils::random_string(TOKEN_LENGTH);
 
 	// Create device for this account
-	services
+	services.users.create_device(
-		.users
+		&user_id,
-		.create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?;
+		&device_id,
+		&token,
+		body.initial_device_display_name.clone(),
+		Some(client.to_string()),
+	)?;
 
 	debug_info!(%user_id, %device_id, "User account was created");
 

src/api/client/context.rs

@@ -21,11 +21,13 @@ pub(crate) async fn get_context_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
 
+	// some clients, at least element, seem to require knowledge of redundant
+	// members for "inline" profiles on the timeline to work properly
 	let (lazy_load_enabled, lazy_load_send_redundant) = match &body.filter.lazy_load_options {
 		LazyLoadOptions::Enabled {
 			include_redundant_members,
 		} => (true, *include_redundant_members),
-		LazyLoadOptions::Disabled => (false, false),
+		LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
 	};
 
 	let mut lazy_loaded = HashSet::new();

src/api/client/media.rs

@@ -5,7 +5,7 @@ use std::time::Duration;
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduit::{
-	debug_warn, err, error,
+	debug_info, debug_warn, err, info,
 	utils::{
 		self,
 		content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename},
@@ -13,12 +13,8 @@ use conduit::{
 	},
 	warn, Err, Error, Result,
 };
-use ruma::api::client::{
+use ruma::api::client::media::{
-	error::{ErrorKind, RetryAfter},
+	create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview,
-	media::{
-		create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config,
-		get_media_preview,
-	},
 };
 use service::{
 	media::{FileMeta, MXC_LENGTH},
@@ -70,35 +66,22 @@ pub(crate) async fn get_media_preview_route(
 
 	let url = &body.url;
 	if !services.media.url_preview_allowed(url) {
-		return Err!(Request(Forbidden(
+		debug_info!(%sender_user, %url, "URL is not allowed to be previewed");
-			warn!(%sender_user, %url, "URL is not allowed to be previewed")
+		return Err!(Request(Forbidden("URL is not allowed to be previewed")));
-		)));
 	}
 
 	match services.media.get_url_preview(url).await {
 		Ok(preview) => {
 			let res = serde_json::value::to_raw_value(&preview).map_err(|e| {
-				error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}");
+				warn!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}");
-				Error::BadRequest(
+				err!(Request(Unknown("Failed to generate a URL preview")))
-					ErrorKind::LimitExceeded {
-						retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))),
-					},
-					"Failed to generate a URL preview, try again later.",
-				)
 			})?;
 
 			Ok(get_media_preview::v3::Response::from_raw_value(res))
 		},
 		Err(e) => {
-			warn!(%sender_user, "Failed to generate a URL preview: {e}");
+			info!(%sender_user, "Failed to generate a URL preview: {e}");
-			// there doesn't seem to be an agreed-upon error code in the spec.
+			Err!(Request(Unknown("Failed to generate a URL preview")))
-			// the only response codes in the preview_url spec page are 200 and 429.
-			Err(Error::BadRequest(
-				ErrorKind::LimitExceeded {
-					retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))),
-				},
-				"Failed to generate a URL preview, try again later.",
-			))
 		},
 	}
 }

src/api/client/membership.rs

@@ -403,6 +403,7 @@ pub(crate) async fn kick_user_route(
 				unsigned: None,
 				state_key: Some(body.user_id.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,
@@ -465,6 +466,7 @@ pub(crate) async fn ban_user_route(
 				unsigned: None,
 				state_key: Some(body.user_id.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,
@@ -512,6 +514,7 @@ pub(crate) async fn unban_user_route(
 				unsigned: None,
 				state_key: Some(body.user_id.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,
@@ -662,6 +665,12 @@ pub async fn join_room_by_id_helper(
 ) -> Result<join_room_by_id::v3::Response> {
 	let state_lock = services.rooms.state.mutex.lock(room_id).await;
 
+	let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false);
+
+	if matches!(services.rooms.state_accessor.guest_can_join(room_id), Ok(false)) && user_is_guest {
+		return Err!(Request(Forbidden("Guests are not allowed to join this room")));
+	}
+
 	if matches!(services.rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) {
 		debug_warn!("{sender_user} is already joined in {room_id}");
 		return Ok(join_room_by_id::v3::Response {
@@ -1094,6 +1103,7 @@ async fn join_room_by_id_helper_local(
 				unsigned: None,
 				state_key: Some(sender_user.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			room_id,
@@ -1406,6 +1416,7 @@ pub(crate) async fn invite_helper(
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				sender_user,
 				room_id,
@@ -1517,6 +1528,7 @@ pub(crate) async fn invite_helper(
 				unsigned: None,
 				state_key: Some(user_id.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			room_id,
@@ -1634,6 +1646,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId,
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				user_id,
 				room_id,

src/api/client/message.rs

@@ -84,6 +84,11 @@ pub(crate) async fn send_message_event_route(
 				unsigned: Some(unsigned),
 				state_key: None,
 				redacts: None,
+				timestamp: if body.appservice_info.is_some() {
+					body.timestamp
+				} else {
+					None
+				},
 			},
 			sender_user,
 			&body.room_id,

src/api/client/profile.rs

@@ -257,6 +257,12 @@ pub(crate) async fn get_profile_route(
 pub async fn update_displayname(
 	services: &Services, user_id: OwnedUserId, displayname: Option<String>, all_joined_rooms: Vec<OwnedRoomId>,
 ) -> Result<()> {
+	let current_display_name = services.users.displayname(&user_id).unwrap_or_default();
+
+	if displayname == current_display_name {
+		return Ok(());
+	}
+
 	services
 		.users
 		.set_displayname(&user_id, displayname.clone())
@@ -289,6 +295,7 @@ pub async fn update_displayname(
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				room_id,
 			))
@@ -305,6 +312,13 @@ pub async fn update_avatar_url(
 	services: &Services, user_id: OwnedUserId, avatar_url: Option<OwnedMxcUri>, blurhash: Option<String>,
 	all_joined_rooms: Vec<OwnedRoomId>,
 ) -> Result<()> {
+	let current_avatar_url = services.users.avatar_url(&user_id).unwrap_or_default();
+	let current_blurhash = services.users.blurhash(&user_id).unwrap_or_default();
+
+	if current_avatar_url == avatar_url && current_blurhash == blurhash {
+		return Ok(());
+	}
+
 	services
 		.users
 		.set_avatar_url(&user_id, avatar_url.clone())
@@ -342,6 +356,7 @@ pub async fn update_avatar_url(
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				room_id,
 			))

src/api/client/redact.rs

@@ -34,6 +34,7 @@ pub(crate) async fn redact_event_route(
 				unsigned: None,
 				state_key: None,
 				redacts: Some(body.event_id.into()),
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,

src/api/client/room.rs

@@ -1,7 +1,7 @@
 use std::{cmp::max, collections::BTreeMap};
 
 use axum::extract::State;
-use conduit::{debug_info, debug_warn};
+use conduit::{debug_info, debug_warn, err};
 use ruma::{
 	api::client::{
 		error::ErrorKind,
@@ -197,6 +197,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -225,6 +226,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(sender_user.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -262,6 +264,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -285,6 +288,7 @@ pub(crate) async fn create_room_route(
 					unsigned: None,
 					state_key: Some(String::new()),
 					redacts: None,
+					timestamp: None,
 				},
 				sender_user,
 				&room_id,
@@ -311,6 +315,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -330,6 +335,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -352,6 +358,7 @@ pub(crate) async fn create_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&room_id,
@@ -405,6 +412,7 @@ pub(crate) async fn create_room_route(
 					unsigned: None,
 					state_key: Some(String::new()),
 					redacts: None,
+					timestamp: None,
 				},
 				sender_user,
 				&room_id,
@@ -427,6 +435,7 @@ pub(crate) async fn create_room_route(
 					unsigned: None,
 					state_key: Some(String::new()),
 					redacts: None,
+					timestamp: None,
 				},
 				sender_user,
 				&room_id,
@@ -475,10 +484,7 @@ pub(crate) async fn get_room_event_route(
 		.rooms
 		.timeline
 		.get_pdu(&body.event_id)?
-		.ok_or_else(|| {
+		.ok_or_else(|| err!(Request(NotFound("Event {} not found.", &body.event_id))))?;
-			warn!("Event not found, event ID: {:?}", &body.event_id);
-			Error::BadRequest(ErrorKind::NotFound, "Event not found.")
-		})?;
 
 	if !services
 		.rooms
@@ -584,6 +590,7 @@ pub(crate) async fn upgrade_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,
@@ -675,6 +682,7 @@ pub(crate) async fn upgrade_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&replacement_room,
@@ -703,6 +711,7 @@ pub(crate) async fn upgrade_room_route(
 				unsigned: None,
 				state_key: Some(sender_user.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&replacement_room,
@@ -731,6 +740,7 @@ pub(crate) async fn upgrade_room_route(
 					unsigned: None,
 					state_key: Some(String::new()),
 					redacts: None,
+					timestamp: None,
 				},
 				sender_user,
 				&replacement_room,
@@ -789,6 +799,7 @@ pub(crate) async fn upgrade_room_route(
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			sender_user,
 			&body.room_id,

src/api/client/session.rs

@@ -1,4 +1,5 @@
 use axum::extract::State;
+use axum_client_ip::InsecureClientIp;
 use ruma::{
 	api::client::{
 		error::ErrorKind,
@@ -33,8 +34,9 @@ struct Claims {
 ///
 /// Get the supported login types of this server. One of these should be used as
 /// the `type` field when logging in.
+#[tracing::instrument(skip_all, fields(%client), name = "login")]
 pub(crate) async fn get_login_types_route(
-	_body: Ruma<get_login_types::v3::Request>,
+	InsecureClientIp(client): InsecureClientIp, _body: Ruma<get_login_types::v3::Request>,
 ) -> Result<get_login_types::v3::Response> {
 	Ok(get_login_types::v3::Response::new(vec![
 		get_login_types::v3::LoginType::Password(PasswordLoginType::default()),
@@ -56,8 +58,9 @@ pub(crate) async fn get_login_types_route(
 /// Note: You can use [`GET
 /// /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see
 /// supported login types.
+#[tracing::instrument(skip_all, fields(%client), name = "login")]
 pub(crate) async fn login_route(
-	State(services): State<crate::State>, body: Ruma<login::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp, body: Ruma<login::v3::Request>,
 ) -> Result<login::v3::Response> {
 	// Validate login method
 	// TODO: Other login methods
@@ -176,9 +179,13 @@ pub(crate) async fn login_route(
 	if device_exists {
 		services.users.set_token(&user_id, &device_id, &token)?;
 	} else {
-		services
+		services.users.create_device(
-			.users
+			&user_id,
-			.create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?;
+			&device_id,
+			&token,
+			body.initial_device_display_name.clone(),
+			Some(client.to_string()),
+		)?;
 	}
 
 	// send client well-known if specified so the client knows to reconfigure itself
@@ -214,8 +221,9 @@ pub(crate) async fn login_route(
 ///   last seen ts)
 /// - Forgets to-device events
 /// - Triggers device list updates
+#[tracing::instrument(skip_all, fields(%client), name = "logout")]
 pub(crate) async fn logout_route(
-	State(services): State<crate::State>, body: Ruma<logout::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp, body: Ruma<logout::v3::Request>,
 ) -> Result<logout::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
@@ -241,8 +249,10 @@ pub(crate) async fn logout_route(
 /// Note: This is equivalent to calling [`GET
 /// /_matrix/client/r0/logout`](fn.logout_route.html) from each device of this
 /// user.
+#[tracing::instrument(skip_all, fields(%client), name = "logout")]
 pub(crate) async fn logout_all_route(
-	State(services): State<crate::State>, body: Ruma<logout_all::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<logout_all::v3::Request>,
 ) -> Result<logout_all::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 

src/api/client/state.rs

@@ -43,6 +43,11 @@ pub(crate) async fn send_state_event_for_key_route(
 			&body.event_type,
 			&body.body.body,
 			body.state_key.clone(),
+			if body.appservice_info.is_some() {
+				body.timestamp
+			} else {
+				None
+			},
 		)
 		.await?
 		.into(),
@@ -172,7 +177,7 @@ pub(crate) async fn get_state_events_for_empty_key_route(
 
 async fn send_state_event_for_key_helper(
 	services: &Services, sender: &UserId, room_id: &RoomId, event_type: &StateEventType,
-	json: &Raw<AnyStateEventContent>, state_key: String,
+	json: &Raw<AnyStateEventContent>, state_key: String, timestamp: Option<ruma::MilliSecondsSinceUnixEpoch>,
 ) -> Result<Arc<EventId>> {
 	allowed_to_send_state_event(services, room_id, event_type, json).await?;
 	let state_lock = services.rooms.state.mutex.lock(room_id).await;
@@ -186,6 +191,7 @@ async fn send_state_event_for_key_helper(
 				unsigned: None,
 				state_key: Some(state_key),
 				redacts: None,
+				timestamp,
 			},
 			sender,
 			room_id,

src/api/client/sync.rs

@@ -106,11 +106,13 @@ pub(crate) async fn sync_events_route(
 			.unwrap_or_default(),
 	};
 
+	// some clients, at least element, seem to require knowledge of redundant
+	// members for "inline" profiles on the timeline to work properly
 	let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options {
 		LazyLoadOptions::Enabled {
-			include_redundant_members: redundant,
+			include_redundant_members,
-		} => (true, redundant),
+		} => (true, include_redundant_members),
-		LazyLoadOptions::Disabled => (false, false),
+		LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
 	};
 
 	let full_state = body.full_state;

src/api/client/voip.rs

@@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime};
 
 use axum::extract::State;
 use base64::{engine::general_purpose, Engine as _};
-use conduit::utils;
+use conduit::{utils, Err};
 use hmac::{Hmac, Mac};
 use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId};
 use sha1::Sha1;
@@ -19,6 +19,11 @@ type HmacSha1 = Hmac<Sha1>;
 pub(crate) async fn turn_server_route(
 	State(services): State<crate::State>, body: Ruma<get_turn_server_info::v3::Request>,
 ) -> Result<get_turn_server_info::v3::Response> {
+	// MSC4166: return M_NOT_FOUND 404 if no TURN URIs are specified in any way
+	if services.server.config.turn_uris.is_empty() {
+		return Err!(Request(NotFound("Not Found")));
+	}
+
 	let turn_secret = services.globals.turn_secret().clone();
 
 	let (username, password) = if !turn_secret.is_empty() {

src/api/router/auth.rs

@@ -6,7 +6,7 @@ use axum_extra::{
 	typed_header::TypedHeaderRejectionReason,
 	TypedHeader,
 };
-use conduit::{warn, Err, Error, Result};
+use conduit::{debug_info, warn, Err, Error, Result};
 use http::uri::PathAndQuery;
 use ruma::{
 	api::{client::error::ErrorKind, AuthScheme, Metadata},
@@ -185,7 +185,7 @@ fn auth_appservice(services: &Services, request: &Request, info: Box<Registratio
 async fn auth_server(
 	services: &Services, request: &mut Request, json_body: &Option<CanonicalJsonValue>,
 ) -> Result<Auth> {
-	if !services.globals.allow_federation() {
+	if !services.server.config.allow_federation {
 		return Err!(Config("allow_federation", "Federation is disabled."));
 	}
 
@@ -206,6 +206,17 @@ async fn auth_server(
 		})?;
 
 	let origin = &x_matrix.origin;
+
+	if services
+		.server
+		.config
+		.forbidden_remote_server_names
+		.contains(origin)
+	{
+		debug_info!("Refusing to accept inbound federation request to {origin}");
+		return Err!(Request(Forbidden("Federation with this homeserver is not allowed.")));
+	}
+
 	let signatures =
 		BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig.to_string()))]);
 	let signatures = BTreeMap::from_iter([(

src/api/server/make_join.rs

@@ -139,6 +139,7 @@ pub(crate) async fn create_join_event_template_route(
 			unsigned: None,
 			state_key: Some(body.user_id.to_string()),
 			redacts: None,
+			timestamp: None,
 		},
 		&body.user_id,
 		&body.room_id,

src/api/server/make_leave.rs

@@ -57,6 +57,7 @@ pub(crate) async fn create_leave_event_template_route(
 			unsigned: None,
 			state_key: Some(body.user_id.to_string()),
 			redacts: None,
+			timestamp: None,
 		},
 		&body.user_id,
 		&body.room_id,

src/core/Cargo.toml

@@ -51,11 +51,13 @@ sha256_media = []
 
 [dependencies]
 argon2.workspace = true
+arrayvec.workspace = true
 axum.workspace = true
 bytes.workspace = true
 cargo_toml.workspace = true
 checked_ops.workspace = true
 chrono.workspace = true
+clap.workspace = true
 conduit-macros.workspace = true
 const-str.workspace = true
 ctor.workspace = true

src/core/config/mod.rs

@@ -236,6 +236,8 @@ pub struct Config {
 	pub rocksdb_compaction_ioprio_idle: bool,
 	#[serde(default = "true_fn")]
 	pub rocksdb_compaction: bool,
+	#[serde(default = "default_rocksdb_stats_level")]
+	pub rocksdb_stats_level: u8,
 
 	pub emergency_password: Option<String>,
 
@@ -718,6 +720,7 @@ impl fmt::Display for Config {
 			&self.rocksdb_compaction_ioprio_idle.to_string(),
 		);
 		line("RocksDB Compaction enabled", &self.rocksdb_compaction.to_string());
+		line("RocksDB Statistics level", &self.rocksdb_stats_level.to_string());
 		line("Media integrity checks on startup", &self.media_startup_check.to_string());
 		line("Media compatibility filesystem links", &self.media_compat_file_link.to_string());
 		line("Prevent Media Downloads From", {
@@ -1002,6 +1005,8 @@ fn default_rocksdb_compression_level() -> i32 { 32767 }
 #[allow(clippy::doc_markdown)]
 fn default_rocksdb_bottommost_compression_level() -> i32 { 32767 }
 
+fn default_rocksdb_stats_level() -> u8 { 1 }
+
 // I know, it's a great name
 #[must_use]
 pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V10 }

src/core/error/mod.rs

@@ -62,6 +62,8 @@ pub enum Error {
 	TomlSerError(#[from] toml::ser::Error),
 	#[error("{0}")]
 	TomlDeError(#[from] toml::de::Error),
+	#[error("{0}")]
+	Clap(#[from] clap::error::Error),
 
 	// ruma
 	#[error("{0}")]

src/core/info/mod.rs

@@ -6,3 +6,6 @@ pub mod rustc;
 pub mod version;
 
 pub use conduit_macros::rustc_flags_capture;
+
+pub const MODULE_ROOT: &str = const_str::split!(std::module_path!(), "::")[0];
+pub const CRATE_PREFIX: &str = const_str::split!(MODULE_ROOT, '_')[0];

src/core/log/capture/data.rs

@@ -2,15 +2,20 @@ use tracing::Level;
 use tracing_core::{span::Current, Event};
 
 use super::{layer::Value, Layer};
+use crate::{info, utils::string::EMPTY};
 
 pub struct Data<'a> {
 	pub layer: &'a Layer,
 	pub event: &'a Event<'a>,
 	pub current: &'a Current,
-	pub values: Option<&'a mut [Value]>,
+	pub values: &'a [Value],
+	pub scope: &'a [&'static str],
 }
 
 impl Data<'_> {
+	#[must_use]
+	pub fn our_modules(&self) -> bool { self.mod_name().starts_with(info::CRATE_PREFIX) }
+
 	#[must_use]
 	pub fn level(&self) -> Level { *self.event.metadata().level() }
 
@@ -18,15 +23,13 @@ impl Data<'_> {
 	pub fn mod_name(&self) -> &str { self.event.metadata().module_path().unwrap_or_default() }
 
 	#[must_use]
-	pub fn span_name(&self) -> &str { self.current.metadata().map_or("", |s| s.name()) }
+	pub fn span_name(&self) -> &str { self.current.metadata().map_or(EMPTY, |s| s.name()) }
 
 	#[must_use]
 	pub fn message(&self) -> &str {
 		self.values
-			.as_ref()
-			.expect("values are not composed for a filter")
 			.iter()
 			.find(|(k, _)| *k == "message")
-			.map_or("", |(_, v)| v.as_str())
+			.map_or(EMPTY, |(_, v)| v.as_str())
 	}
 }

src/core/log/capture/layer.rs

@@ -1,21 +1,25 @@
 use std::{fmt, sync::Arc};
 
+use arrayvec::ArrayVec;
 use tracing::field::{Field, Visit};
 use tracing_core::{Event, Subscriber};
 use tracing_subscriber::{layer::Context, registry::LookupSpan};
 
 use super::{Capture, Data, State};
 
-pub type Value = (&'static str, String);
-
 pub struct Layer {
 	state: Arc<State>,
 }
 
 struct Visitor {
-	values: Vec<Value>,
+	values: Values,
 }
 
+type Values = ArrayVec<Value, 32>;
+pub type Value = (&'static str, String);
+
+type ScopeNames = ArrayVec<&'static str, 32>;
+
 impl Layer {
 	#[inline]
 	pub fn new(state: &Arc<State>) -> Self {
@@ -51,8 +55,9 @@ fn handle<S>(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context<
 where
 	S: Subscriber + for<'a> LookupSpan<'a>,
 {
+	let names = ScopeNames::new();
 	let mut visitor = Visitor {
-		values: Vec::new(),
+		values: Values::new(),
 	};
 	event.record(&mut visitor);
 
@@ -61,7 +66,8 @@ where
 		layer,
 		event,
 		current: &ctx.current_span(),
-		values: Some(&mut visitor.values),
+		values: &visitor.values,
+		scope: &names,
 	});
 }
 
@@ -69,12 +75,21 @@ fn filter<S>(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context<
 where
 	S: Subscriber + for<'a> LookupSpan<'a>,
 {
+	let values = Values::new();
+	let mut names = ScopeNames::new();
+	if let Some(scope) = ctx.event_scope(event) {
+		for span in scope {
+			names.push(span.name());
+		}
+	}
+
 	capture.filter.as_ref().map_or(true, |filter| {
 		filter(Data {
 			layer,
 			event,
 			current: &ctx.current_span(),
-			values: None,
+			values: &values,
+			scope: &names,
 		})
 	})
 }

src/core/log/fmt.rs

@@ -5,7 +5,7 @@ use crate::Result;
 
 pub fn html<S>(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()>
 where
-	S: Write,
+	S: Write + ?Sized,
 {
 	let color = color::code_tag(level);
 	let level = level.as_str().to_uppercase();
@@ -19,7 +19,7 @@ where
 
 pub fn markdown<S>(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()>
 where
-	S: Write,
+	S: Write + ?Sized,
 {
 	let level = level.as_str().to_uppercase();
 	writeln!(out, "`{level:>5}` `{span:^12}` `{msg}`")?;
@@ -29,19 +29,19 @@ where
 
 pub fn markdown_table<S>(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()>
 where
-	S: Write,
+	S: Write + ?Sized,
 {
 	let level = level.as_str().to_uppercase();
-	writeln!(out, "| `{level:>5}` | `{span:^12}` | `{msg} |")?;
+	writeln!(out, "| {level:>5} | {span:^12} | {msg} |")?;
 
 	Ok(())
 }
 
 pub fn markdown_table_head<S>(out: &mut S) -> Result<()>
 where
-	S: Write,
+	S: Write + ?Sized,
 {
-	write!(out, "| level | span | message |\n|------:|:----:|:--------|\n")?;
+	write!(out, "| level | span | message |\n| ------: | :-----: | :------- |\n")?;
 
 	Ok(())
 }

src/core/pdu/builder.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, sync::Arc};
 
-use ruma::{events::TimelineEventType, EventId};
+use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch};
 use serde::Deserialize;
 use serde_json::value::RawValue as RawJsonValue;
 
@@ -13,4 +13,8 @@ pub struct PduBuilder {
 	pub unsigned: Option<BTreeMap<String, serde_json::Value>>,
 	pub state_key: Option<String>,
 	pub redacts: Option<Arc<EventId>>,
+	/// For timestamped messaging, should only be used for appservices
+	///
+	/// Will be set to current time if None
+	pub timestamp: Option<MilliSecondsSinceUnixEpoch>,
 }

src/core/utils/string.rs

@@ -30,6 +30,16 @@ macro_rules! is_format {
 	};
 }
 
+#[inline]
+pub fn collect_stream<F>(func: F) -> Result<String>
+where
+	F: FnOnce(&mut dyn std::fmt::Write) -> Result<()>,
+{
+	let mut out = String::new();
+	func(&mut out)?;
+	Ok(out)
+}
+
 #[inline]
 #[must_use]
 pub fn camel_to_snake_string(s: &str) -> String {

src/database/database.rs

@@ -2,7 +2,12 @@ use std::{ops::Index, sync::Arc};
 
 use conduit::{Result, Server};
 
-use crate::{cork::Cork, maps, maps::Maps, Engine, Map};
+use crate::{
+	cork::Cork,
+	maps,
+	maps::{Maps, MapsKey, MapsVal},
+	Engine, Map,
+};
 
 pub struct Database {
 	pub db: Arc<Engine>,
@@ -30,6 +35,9 @@ impl Database {
 	#[inline]
 	#[must_use]
 	pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) }
+
+	#[inline]
+	pub fn iter_maps(&self) -> impl Iterator<Item = (&MapsKey, &MapsVal)> + '_ { self.map.iter() }
 }
 
 impl Index<&str> for Database {

src/database/engine.rs

@@ -1,5 +1,6 @@
 use std::{
 	collections::{BTreeSet, HashMap},
+	ffi::CStr,
 	fmt::Write,
 	path::PathBuf,
 	sync::{atomic::AtomicU32, Arc, Mutex, RwLock},
@@ -9,7 +10,8 @@ use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err,
 use rocksdb::{
 	backup::{BackupEngine, BackupEngineOptions},
 	perf::get_memory_usage_stats,
-	BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, MultiThreaded, Options,
+	AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env,
+	MultiThreaded, Options,
 };
 
 use crate::{
@@ -240,6 +242,20 @@ impl Engine {
 			},
 		}
 	}
+
+	/// Query for database property by null-terminated name which is expected to
+	/// have a result with an integer representation. This is intended for
+	/// low-overhead programmatic use.
+	pub(crate) fn property_integer(&self, cf: &impl AsColumnFamilyRef, name: &CStr) -> Result<u64> {
+		result(self.db.property_int_value_cf(cf, name))
+			.and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok))
+	}
+
+	/// Query for database property by name receiving the result in a string.
+	pub(crate) fn property(&self, cf: &impl AsColumnFamilyRef, name: &str) -> Result<String> {
+		result(self.db.property_value_cf(cf, name))
+			.and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok))
+	}
 }
 
 pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> {

src/database/map.rs

@@ -1,4 +1,4 @@
-use std::{future::Future, mem::size_of, pin::Pin, sync::Arc};
+use std::{ffi::CStr, future::Future, mem::size_of, pin::Pin, sync::Arc};
 
 use conduit::{utils, Result};
 use rocksdb::{
@@ -189,6 +189,10 @@ impl Map {
 		self.watchers.watch(prefix)
 	}
 
+	pub fn property_integer(&self, name: &CStr) -> Result<u64> { self.db.property_integer(&self.cf(), name) }
+
+	pub fn property(&self, name: &str) -> Result<String> { self.db.property(&self.cf(), name) }
+
 	#[inline]
 	pub fn name(&self) -> &str { &self.name }
 

src/database/maps.rs

@@ -4,7 +4,9 @@ use conduit::Result;
 
 use crate::{Engine, Map};
 
-pub type Maps = BTreeMap<String, Arc<Map>>;
+pub type Maps = BTreeMap<MapsKey, MapsVal>;
+pub(crate) type MapsVal = Arc<Map>;
+pub(crate) type MapsKey = String;
 
 pub(crate) fn open(db: &Arc<Engine>) -> Result<Maps> { open_list(db, MAPS) }
 

src/database/opts.rs

@@ -2,8 +2,8 @@ use std::{cmp, collections::HashMap};
 
 use conduit::{utils, Config};
 use rocksdb::{
-	BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options,
+	statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env,
-	UniversalCompactOptions, UniversalCompactionStopStyle,
+	LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle,
 };
 
 /// Create database-wide options suitable for opening the database. This also
@@ -13,6 +13,11 @@ use rocksdb::{
 /// through cf_options().
 pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Options {
 	const MIN_PARALLELISM: usize = 2;
+	const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) {
+		StatsLevel::ExceptDetailedTimers
+	} else {
+		StatsLevel::DisableAll
+	};
 
 	let mut opts = Options::default();
 
@@ -68,8 +73,18 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_
 	set_compression_defaults(&mut opts, config);
 
 	// Misc
-	opts.set_disable_auto_compactions(!config.rocksdb_compaction);
 	opts.create_if_missing(true);
+	opts.set_disable_auto_compactions(!config.rocksdb_compaction);
+
+	opts.set_statistics_level(match config.rocksdb_stats_level {
+		0 => StatsLevel::DisableAll,
+		1 => DEFAULT_STATS_LEVEL,
+		2 => StatsLevel::ExceptHistogramOrTimers,
+		3 => StatsLevel::ExceptTimers,
+		4 => StatsLevel::ExceptDetailedTimers,
+		5 => StatsLevel::ExceptTimeForMutex,
+		6_u8..=u8::MAX => StatsLevel::All,
+	});
 
 	// Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords
 	//

src/service/admin/create.rs

@@ -71,6 +71,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -99,6 +100,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(server_user.to_string()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -124,6 +126,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -143,6 +146,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -162,6 +166,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -181,6 +186,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -201,6 +207,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -221,6 +228,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -245,6 +253,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,
@@ -271,6 +280,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 				unsigned: None,
 				state_key: Some(String::new()),
 				redacts: None,
+				timestamp: None,
 			},
 			server_user,
 			&room_id,

src/service/admin/grant.rs

@@ -50,6 +50,7 @@ impl super::Service {
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				server_user,
 				&room_id,
@@ -75,6 +76,7 @@ impl super::Service {
 					unsigned: None,
 					state_key: Some(user_id.to_string()),
 					redacts: None,
+					timestamp: None,
 				},
 				user_id,
 				&room_id,
@@ -100,6 +102,7 @@ impl super::Service {
 					unsigned: None,
 					state_key: Some(String::new()),
 					redacts: None,
+					timestamp: None,
 				},
 				server_user,
 				&room_id,
@@ -119,6 +122,7 @@ impl super::Service {
                 unsigned: None,
                 state_key: None,
                 redacts: None,
+                timestamp: None,
             },
             server_user,
             &room_id,

src/service/admin/mod.rs

@@ -51,9 +51,9 @@ pub struct CommandInput {
 }
 
 pub type Completer = fn(&str) -> String;
-pub type Handler = fn(Arc<crate::Services>, CommandInput) -> HandlerResult;
+pub type Handler = fn(Arc<crate::Services>, CommandInput) -> HandlerFuture;
-pub type HandlerResult = Pin<Box<dyn Future<Output = CommandResult> + Send>>;
+pub type HandlerFuture = Pin<Box<dyn Future<Output = HandlerResult> + Send>>;
-pub type CommandResult = Result<CommandOutput, Error>;
+pub type HandlerResult = Result<CommandOutput>;
 pub type CommandOutput = Option<RoomMessageEventContent>;
 
 const COMMAND_QUEUE_LIMIT: usize = 512;
@@ -173,7 +173,7 @@ impl Service {
 		}
 	}
 
-	async fn process_command(&self, command: CommandInput) -> CommandResult {
+	async fn process_command(&self, command: CommandInput) -> HandlerResult {
 		let Some(services) = self
 			.services
 			.services
@@ -260,6 +260,7 @@ impl Service {
 			unsigned: None,
 			state_key: None,
 			redacts: None,
+			timestamp: None,
 		};
 
 		if let Err(e) = self
@@ -289,6 +290,7 @@ impl Service {
 			unsigned: None,
 			state_key: None,
 			redacts: None,
+			timestamp: None,
 		};
 
 		self.services

src/service/presence/data.rs

@@ -61,6 +61,22 @@ impl Data {
 			Some(ref presence) => presence.1.content.presence != *presence_state,
 		};
 
+		let status_msg_changed = match last_presence {
+			None => true,
+			Some(ref last_presence) => {
+				let old_msg = last_presence
+					.1
+					.content
+					.status_msg
+					.clone()
+					.unwrap_or_default();
+
+				let new_msg = status_msg.clone().unwrap_or_default();
+
+				new_msg != old_msg
+			},
+		};
+
 		let now = utils::millis_since_unix_epoch();
 		let last_last_active_ts = match last_presence {
 			None => 0,
@@ -72,10 +88,10 @@ impl Data {
 			Some(last_active_ago) => now.saturating_sub(last_active_ago.into()),
 		};
 
-		// tighten for state flicker?
+		// TODO: tighten for state flicker?
-		if !state_changed && last_active_ts <= last_last_active_ts {
+		if !status_msg_changed && !state_changed && last_active_ts < last_last_active_ts {
 			debug_warn!(
-				"presence spam {:?} last_active_ts:{:?} <= {:?}",
+				"presence spam {:?} last_active_ts:{:?} < {:?}",
 				user_id,
 				last_active_ts,
 				last_last_active_ts

src/service/rooms/state_accessor/mod.rs

@@ -350,6 +350,7 @@ impl Service {
 			unsigned: None,
 			state_key: Some(target_user.into()),
 			redacts: None,
+			timestamp: None,
 		};
 
 		Ok(self

src/service/rooms/timeline/mod.rs

@@ -638,6 +638,7 @@ impl Service {
 			unsigned,
 			state_key,
 			redacts,
+			timestamp,
 		} = pdu_builder;
 
 		let prev_events: Vec<_> = self
@@ -705,9 +706,14 @@ impl Service {
 			room_id: room_id.to_owned(),
 			sender: sender.to_owned(),
 			origin: None,
-			origin_server_ts: utils::millis_since_unix_epoch()
+			origin_server_ts: timestamp.map_or_else(
+				|| {
+					utils::millis_since_unix_epoch()
 						.try_into()
-				.expect("time is valid"),
+						.expect("u64 fits into UInt")
+				},
+				|ts| ts.get(),
+			),
 			kind: event_type,
 			content,
 			state_key,

src/service/sending/send.rs

@@ -1,7 +1,8 @@
 use std::{fmt::Debug, mem};
 
 use conduit::{
-	debug, debug_error, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result,
+	debug, debug_error, debug_info, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error,
+	Result,
 };
 use http::{header::AUTHORIZATION, HeaderValue};
 use ipaddress::IPAddress;
@@ -31,6 +32,16 @@ impl super::Service {
 			return Err!(Config("allow_federation", "Federation is disabled."));
 		}
 
+		if self
+			.server
+			.config
+			.forbidden_remote_server_names
+			.contains(&dest.to_owned())
+		{
+			debug_info!("Refusing to send outbound federation request to {dest}");
+			return Err!(Request(Forbidden("Federation with this homeserver is not allowed.")));
+		}
+
 		let actual = self.services.resolver.get_actual_dest(dest).await?;
 		let request = self.prepare::<T>(dest, &actual, req).await?;
 		self.execute::<T>(dest, &actual, request, client).await

src/service/users/data.rs

@@ -246,6 +246,7 @@ impl Data {
 	/// Adds a new device to a user.
 	pub(super) fn create_device(
 		&self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option<String>,
+		client_ip: Option<String>,
 	) -> Result<()> {
 		// This method should never be called for nonexistent users. We shouldn't assert
 		// though...
@@ -266,7 +267,7 @@ impl Data {
 			&serde_json::to_vec(&Device {
 				device_id: device_id.into(),
 				display_name: initial_device_display_name,
-				last_seen_ip: None, // TODO
+				last_seen_ip: client_ip,
 				last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()),
 			})
 			.expect("Device::to_string never fails."),

src/service/users/mod.rs

@@ -328,9 +328,10 @@ impl Service {
 	/// Adds a new device to a user.
 	pub fn create_device(
 		&self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option<String>,
+		client_ip: Option<String>,
 	) -> Result<()> {
 		self.db
-			.create_device(user_id, device_id, token, initial_device_display_name)
+			.create_device(user_id, device_id, token, initial_device_display_name, client_ip)
 	}
 
 	/// Removes a device from a user.