implement MSC4166, returning M_NOT_FOUND if no TURN URIs are set

Signed-off-by: strawberry <strawberry@puppygock.gay>

Cargo.lock

@@ -394,9 +394,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
 
 [[package]]
 name = "bytemuck"
-version = "1.16.3"
+version = "1.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83"
+checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
 
 [[package]]
 name = "byteorder"
@@ -412,9 +412,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
 
 [[package]]
 name = "bytes"
-version = "1.7.1"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
+checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"
 
 [[package]]
 name = "bzip2-sys"
@@ -439,9 +439,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.7"
+version = "1.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc"
+checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
 dependencies = [
  "jobserver",
  "libc",
@@ -499,9 +499,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.13"
+version = "4.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -509,9 +509,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.13"
+version = "4.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa"
 dependencies = [
  "anstyle",
  "clap_lex",
@@ -519,9 +519,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.13"
+version = "4.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
@@ -608,6 +608,7 @@ dependencies = [
  "http",
  "http-body-util",
  "hyper",
+ "image",
  "ipaddress",
  "jsonwebtoken",
  "log",
@@ -620,6 +621,7 @@ dependencies = [
  "sha-1",
  "tokio",
  "tracing",
+ "webpage",
 ]
 
 [[package]]
@@ -755,7 +757,6 @@ dependencies = [
  "tokio",
  "tracing",
  "url",
- "webpage",
 ]
 
 [[package]]
@@ -1339,7 +1340,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "http",
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "slab",
  "tokio",
  "tokio-util",
@@ -1708,9 +1709,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.3.0"
+version = "2.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.5",
@@ -2247,7 +2248,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
 dependencies = [
  "futures-core",
  "futures-sink",
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "js-sys",
  "once_cell",
  "pin-project-lite",
@@ -2294,7 +2295,7 @@ dependencies = [
  "glob",
  "once_cell",
  "opentelemetry",
- "ordered-float 4.2.2",
+ "ordered-float 4.2.1",
  "percent-encoding",
  "rand",
  "thiserror",
@@ -2313,9 +2314,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "4.2.2"
+version = "4.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
+checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be"
 dependencies = [
  "num-traits",
 ]
@@ -2542,12 +2543,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.18"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
-dependencies = [
- "zerocopy",
-]
 
 [[package]]
 name = "precomputed-hash"
@@ -2927,7 +2925,7 @@ dependencies = [
  "bytes",
  "form_urlencoded",
  "http",
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "js_int",
  "konst",
  "percent-encoding",
@@ -2953,7 +2951,7 @@ version = "0.28.1"
 source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536"
 dependencies = [
  "as_variant",
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "js_int",
  "js_option",
  "percent-encoding",
@@ -3477,7 +3475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5"
 dependencies = [
  "form_urlencoded",
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "itoa",
  "ryu",
  "serde",
@@ -3542,7 +3540,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "itoa",
  "ryu",
  "serde",
@@ -3609,9 +3607,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-mio"
-version = "0.2.4"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
+checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
 dependencies = [
  "libc",
  "mio 0.8.11",
@@ -4012,9 +4010,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-socks"
-version = "0.5.2"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f"
+checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0"
 dependencies = [
  "either",
  "futures-util",
@@ -4048,21 +4046,21 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.19"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
 dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.20",
+ "toml_edit 0.22.17",
 ]
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.8"
+version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
 dependencies = [
  "serde",
 ]
@@ -4073,22 +4071,22 @@ version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
 dependencies = [
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "toml_datetime",
  "winnow 0.5.40",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.20"
+version = "0.22.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
+checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
 dependencies = [
- "indexmap 2.3.0",
+ "indexmap 2.2.6",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.18",
+ "winnow 0.6.16",
 ]
 
 [[package]]
@@ -4769,9 +4767,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.18"
+version = "0.6.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
+checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c"
 dependencies = [
  "memchr",
 ]
@@ -4813,27 +4811,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
 
-[[package]]
-name = "zerocopy"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
-dependencies = [
- "byteorder",
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
-]
-
 [[package]]
 name = "zeroize"
 version = "1.8.1"

Cargo.toml

@@ -569,17 +569,7 @@ lto = "fat"
 [profile.release-max-perf.build-override]
 inherits = "release-max-perf"
 opt-level = 0
-codegen-units = 32
 #rustflags = [
-#	'-Crelocation-model=pic',
-#	'-Ctarget-feature=-crt-static',
-#	'-Clink-arg=-Wl,--no-gc-sections',
-#]
-
-[profile.release-max-perf.package.conduit_macros]
-inherits = "release-max-perf.build-override"
-#rustflags = [
-#	'-Crelocation-model=pic',
 #	'-Ctarget-feature=-crt-static',
 #]
 

conduwuit-example.toml

@@ -224,9 +224,11 @@ registration_token = "change this token for something specific to your server"
 # No default.
 # forbidden_alias_names = []
 
-# List of forbidden server names that we will block all client room joins, incoming federated room directory requests, incoming federated invites for, and incoming federated joins. This check is applied on the room ID, room alias, sender server name, and sender user's server name.
+# List of forbidden server names that we will block incoming AND outgoing federation with, and block client room joins / remote user invites.
-# Basically "global" ACLs. For our user (client) checks, admin users are allowed.
+#
-# No default.
+# This check is applied on the room ID, room alias, sender server name, sender user's server name, inbound federation X-Matrix origin, and outbound federation handler.
+#
+# Basically "global" ACLs. No default.
 # forbidden_remote_server_names = []
 
 # List of forbidden server names that we will block all outgoing federated room directory requests for. Useful for preventing our users from wandering into bad servers or spaces.

debian/postinst

@@ -16,7 +16,6 @@ case "$1" in
         --home "$CONDUWUIT_DATABASE_PATH" \
         --disabled-login \
         --shell "/usr/sbin/nologin" \
-        --verbose \
         conduwuit
     fi
 

docs/SUMMARY.md

@@ -10,6 +10,7 @@
   - [Docker](deploying/docker.md)
   - [Arch Linux](deploying/arch-linux.md)
   - [Debian](deploying/debian.md)
+  - [FreeBSD](deploying/freebsd.md)
 - [TURN](turn.md)
 - [Appservices](appservices.md)
 - [Maintenance](maintenance.md)

docs/configuration.md

@@ -4,15 +4,35 @@ This chapter describes various ways to configure conduwuit.
 
 ## Basics
 
-Conduwuit uses a config file for the majority of the settings. Please refer to the
+conduwuit uses a config file for the majority of the settings, but also supports setting individual config options via commandline.
-[example config file](./configuration/examples.md#example-configuration) for all of those settings.
+
-The config file to use can either be specified on the command line when running conduwuit by specifying the
+Please refer to the [example config file](./configuration/examples.md#example-configuration) for all of those settings.
+
+The config file to use can be specified on the commandline when running conduwuit by specifying the
 `-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config
-file to used.
+file to used. Conduit's environment variables are supported for backwards compatibility.
+
+## Option commandline flag
+
+conduwuit supports setting individual config options in TOML format from the `-O` / `--option` flag. For example, you can set your server name via `-O server_name=\"example.com\"`.
+
+Note that the config is parsed as TOML, and shells like bash will remove quotes. So unfortunately it is required to escape quotes if the config option takes a string.
+This does not apply to options that take booleans or numbers:
+- `--option allow_registration=true` works ✅
+- `-O max_request_size=99999999` works ✅
+- `-O server_name=example.com` does not work ❌
+- `--option log=\"debug\"` works ✅
+- `--option server_name='"example.com'"` works ✅
+
 
 ## Environment variables
 
 All of the settings that are found in the config file can be specified by using environment variables.
 The environment variable names should be all caps and prefixed with `CONDUWUIT_`.
+
 For example, if the setting you are changing is `max_request_size`, then the environment variable to set is
 `CONDUWUIT_MAX_REQUEST_SIZE`.
+
+To modify config options not in the `[global]` context such as `[global.well_known]`, use the `__` suffix split: `CONDUWUIT_WELL_KNOWN__SERVER`
+
+Conduit's environment variables are supported for backwards compatibility (e.g. `CONDUIT_SERVER_NAME`).

docs/deploying/freebsd.md

@@ -0,0 +1,8 @@
+# conduwuit for FreeBSD
+
+conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on FreeBSD requires a specific environment variable to use the
+system prebuilt RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* work and will cause a build error or coredump.
+
+Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib`
+
+Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo build --release`

docs/deploying/generic.md

@@ -9,14 +9,12 @@
 
 You may simply download the binary that fits your machine. Run `uname -m` to see what you need.
 
-Prebuilt binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest).
+Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or `main` CI branch workflow artifact output. These also include Debian packages. These binaries have jemalloc and io_uring statically linked and included with them.
-
-The latest tagged release also includes the Debian packages.
 
 Alternatively, you may compile the binary yourself. We recommend using [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed
 reproducibiltiy and easiest to get a build environment and output going.
 
-Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang, installing liburing for io_uring and RocksDB, etc).
+Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang for RocksDB, installing liburing for io_uring and RocksDB, etc).
 
 ## Adding a conduwuit user
 

docs/troubleshooting.md

@@ -8,6 +8,17 @@
 >
 > If there are things like Compose file issues or Dockerhub image issues, those can still be mentioned as long as they're something we can fix.
 
+## General potential issues
+
+#### Potential DNS issues when using Docker
+
+Docker has issues with its default DNS setup that may cause DNS to not be properly functional when running conduwuit, resulting in federation issues.
+The symptoms of this have shown in excessively long room joins (30+ minutes) from very long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation.
+
+This is **not** a conduwuit issue, and is purely a Docker issue. It is not sustainable for heavy DNS activity which is normal for Matrix federation. The workarounds for this are:
+- Use DNS over TCP via the config option `query_over_tcp_only = true`
+- Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`)
+
 ## Rocksdb / database issues
 
 #### Direct IO
@@ -38,8 +49,6 @@ With this in mind:
 - Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored
 - If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database
 
-## Media
-
 ## Debugging
 
 Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`.

flake.nix

@@ -137,7 +137,11 @@
           # Useful for editing the book locally
           mdbook
 
+          # used for rust caching in CI to speed it up
           sccache
+
+          # needed so we can get rid of gcc and other unused deps that bloat OCI images
+          removeReferencesTo
         ])
         ++ scope.main.buildInputs
         ++ scope.main.propagatedBuildInputs

nix/pkgs/main/default.nix

@@ -7,6 +7,7 @@
 , liburing
 , pkgsBuildHost
 , rocksdb
+, removeReferencesTo
 , rust
 , rust-jemalloc-sys
 , stdenv
@@ -94,8 +95,8 @@ buildDepsOnlyEnv =
         else if stdenv.targetPlatform.isAarch64
         then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
         ++ lib.optionals stdenv.targetPlatform.isAarch64 [
-          # cortex-a55 == ARMv8.2-a
+          # cortex-a73 == ARMv8-A
-          "-DPORTABLE=armv8.2-a"
+          "-DPORTABLE=armv8-a"
         ]
         else old.cmakeFlags;
     });
@@ -128,7 +129,7 @@ buildPackageEnv = {
     + lib.optionalString stdenv.targetPlatform.isx86_64
       " -Ctarget-cpu=x86-64-v2"
     + lib.optionalString stdenv.targetPlatform.isAarch64
-      " -Ctarget-cpu=cortex-a55"; # cortex-a55 == ARMv8.2-a
+      " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A
 };
 
 
@@ -154,6 +155,7 @@ commonAttrs = {
     };
 
     dontStrip = profile == "dev" || profile == "test";
+    dontPatchELF = profile == "dev" || profile == "test";
 
     buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
 
@@ -168,6 +170,9 @@ commonAttrs = {
       # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
       # rebuilds of bindgen and its depedents.
       jq
+
+      # needed so we can get rid of gcc and other unused deps that bloat OCI images
+      removeReferencesTo
   ]
   ++ lib.optionals stdenv.isDarwin [
       # https://github.com/NixOS/nixpkgs/issues/206242
@@ -177,6 +182,13 @@ commonAttrs = {
       # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
       pkgsBuildHost.darwin.apple_sdk.frameworks.Security
     ];
+
+    # for some reason gcc and other weird deps are added to OCI images and bloats it up
+    #
+    # <https://github.com/input-output-hk/haskell.nix/issues/829>
+    postInstall = with pkgsBuildHost; ''
+        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' +
+    '';
  };
 in
 

src/admin/debug/commands.rs

@@ -7,7 +7,7 @@ use std::{
 
 use api::client::validate_and_add_event_id;
 use conduit::{
-	debug, debug_error, err, info, log,
+	debug, info, log,
 	log::{capture, Capture},
 	utils, warn, Error, PduEvent, Result,
 };
@@ -149,23 +149,32 @@ pub(super) async fn get_remote_pdu_list(
 		.filter_map(|pdu| EventId::parse(pdu).ok())
 		.collect::<Vec<_>>();
 
+	let mut failed_count: usize = 0;
+	let mut success_count: usize = 0;
+
 	for pdu in list {
 		if force {
 			if let Err(e) = self.get_remote_pdu(Box::from(pdu), server.clone()).await {
+				failed_count = failed_count.saturating_add(1);
 				self.services
 					.admin
 					.send_message(RoomMessageEventContent::text_plain(format!(
 						"Failed to get remote PDU, ignoring error: {e}"
 					)))
 					.await;
-				warn!(%e, "Failed to get remote PDU, ignoring error");
+				warn!("Failed to get remote PDU, ignoring error: {e}");
+			} else {
+				success_count = success_count.saturating_add(1);
 			}
 		} else {
 			self.get_remote_pdu(Box::from(pdu), server.clone()).await?;
+			success_count = success_count.saturating_add(1);
 		}
 	}
 
-	Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs."))
+	Ok(RoomMessageEventContent::text_plain(format!(
+		"Fetched {success_count} remote PDUs successfully with {failed_count} failures"
+	)))
 }
 
 #[admin_command]
@@ -229,7 +238,8 @@ pub(super) async fn get_remote_pdu(
 
 			debug!("Attempting to fetch homeserver signing keys for {server}");
 			self.services
-				.server_keys
+				.rooms
+				.event_handler
 				.fetch_required_signing_keys(parsed_pdu.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map)
 				.await
 				.unwrap_or_else(|e| {
@@ -442,7 +452,8 @@ pub(super) async fn verify_json(&self) -> Result<RoomMessageEventContent> {
 			let pub_key_map = RwLock::new(BTreeMap::new());
 
 			self.services
-				.server_keys
+				.rooms
+				.event_handler
 				.fetch_required_signing_keys([&value], &pub_key_map)
 				.await?;
 
@@ -560,7 +571,8 @@ pub(super) async fn force_set_room_state_from_server(
 
 	info!("Fetching required signing keys for all the state events we got");
 	self.services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map)
 		.await?;
 
@@ -575,8 +587,8 @@ pub(super) async fn force_set_room_state_from_server(
 		};
 
 		let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| {
-			debug_error!("Invalid PDU in fetching remote room state PDUs response: {value:#?}");
+			warn!("Invalid PDU in fetching remote room state PDUs response: {} {:?}", e, value);
-			err!(BadServerResponse(debug_error!("Invalid PDU in send_join response: {e:?}")))
+			Error::BadServerResponse("Invalid PDU in send_join response.")
 		})?;
 
 		self.services
@@ -679,7 +691,8 @@ pub(super) async fn get_verify_keys(
 	let signature_ids: Vec<String> = Vec::new();
 	let keys = self
 		.services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_signing_keys_for_server(&server_name, signature_ids)
 		.await?;
 

src/api/Cargo.toml

@@ -47,6 +47,7 @@ hmac.workspace = true
 http.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true
+image.workspace = true
 ipaddress.workspace = true
 jsonwebtoken.workspace = true
 log.workspace = true
@@ -59,6 +60,7 @@ serde.workspace = true
 sha-1.workspace = true
 tokio.workspace = true
 tracing.workspace = true
+webpage.workspace = true
 
 [lints]
 workspace = true

src/api/client/account.rs

@@ -286,9 +286,13 @@ pub(crate) async fn register_route(
 	let token = utils::random_string(TOKEN_LENGTH);
 
 	// Create device for this account
-	services
+	services.users.create_device(
-		.users
+		&user_id,
-		.create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?;
+		&device_id,
+		&token,
+		body.initial_device_display_name.clone(),
+		Some(client.to_string()),
+	)?;
 
 	debug_info!(%user_id, %device_id, "User account was created");
 

src/api/client/context.rs

@@ -21,11 +21,13 @@ pub(crate) async fn get_context_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
 
+	// some clients, at least element, seem to require knowledge of redundant
+	// members for "inline" profiles on the timeline to work properly
 	let (lazy_load_enabled, lazy_load_send_redundant) = match &body.filter.lazy_load_options {
 		LazyLoadOptions::Enabled {
 			include_redundant_members,
 		} => (true, *include_redundant_members),
-		LazyLoadOptions::Disabled => (false, false),
+		LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
 	};
 
 	let mut lazy_loaded = HashSet::new();

src/api/client/keys.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use axum::extract::State;
-use conduit::{utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result};
+use conduit::{debug, utils, utils::math::continue_exponential_backoff_secs, Error, Result};
 use futures_util::{stream::FuturesUnordered, StreamExt};
 use ruma::{
 	api::{
@@ -362,7 +362,8 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool + Send>(
 				const MIN: u64 = 5 * 60;
 				const MAX: u64 = 60 * 60 * 24;
 				if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) {
-					return (server, Err!(BadServerResponse("bad query from {server:?}, still backing off")));
+					debug!("Backing off query from {server:?}");
+					return (server, Err(Error::BadServerResponse("bad query, still backing off")));
 				}
 			}
 

src/api/client/media.rs

@@ -1,11 +1,11 @@
 #![allow(deprecated)]
 
-use std::time::Duration;
+use std::{io::Cursor, time::Duration};
 
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduit::{
-	debug_warn, err, error,
+	debug, debug_info, debug_warn, err, info,
 	utils::{
 		self,
 		content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename},
@@ -13,20 +13,27 @@ use conduit::{
 	},
 	warn, Err, Error, Result,
 };
+use image::io::Reader as ImgReader;
+use ipaddress::IPAddress;
+use reqwest::Url;
 use ruma::api::client::{
-	error::{ErrorKind, RetryAfter},
+	error::ErrorKind,
 	media::{
 		create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config,
 		get_media_preview,
 	},
 };
 use service::{
-	media::{FileMeta, MXC_LENGTH},
+	media::{FileMeta, UrlPreviewData},
 	Services,
 };
+use webpage::HTML;
 
 use crate::{Ruma, RumaResponse};
 
+/// generated MXC ID (`media-id`) length
+const MXC_LENGTH: usize = 32;
+
 /// Cache control for immutable objects
 const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable";
 
@@ -69,36 +76,23 @@ pub(crate) async fn get_media_preview_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
 	let url = &body.url;
-	if !services.media.url_preview_allowed(url) {
+	if !url_preview_allowed(&services, url) {
-		return Err!(Request(Forbidden(
+		debug_info!(%sender_user, "URL is not allowed to be previewed: {url}");
-			warn!(%sender_user, %url, "URL is not allowed to be previewed")
+		return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed"));
-		)));
 	}
 
-	match services.media.get_url_preview(url).await {
+	match get_url_preview(&services, url).await {
 		Ok(preview) => {
 			let res = serde_json::value::to_raw_value(&preview).map_err(|e| {
-				error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}");
+				warn!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}");
-				Error::BadRequest(
+				err!(Request(Unknown("Failed to generate a URL preview")))
-					ErrorKind::LimitExceeded {
-						retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))),
-					},
-					"Failed to generate a URL preview, try again later.",
-				)
 			})?;
 
 			Ok(get_media_preview::v3::Response::from_raw_value(res))
 		},
 		Err(e) => {
-			warn!(%sender_user, "Failed to generate a URL preview: {e}");
+			info!(%sender_user, "Failed to generate a URL preview: {e}");
-			// there doesn't seem to be an agreed-upon error code in the spec.
+			Err!(Request(Unknown("Failed to generate a URL preview")))
-			// the only response codes in the preview_url spec page are 200 and 429.
-			Err(Error::BadRequest(
-				ErrorKind::LimitExceeded {
-					retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))),
-				},
-				"Failed to generate a URL preview, try again later.",
-			))
 		},
 	}
 }
@@ -222,7 +216,10 @@ pub(crate) async fn get_content_route(
 			body.timeout_ms,
 		)
 		.await
-		.map_err(|e| err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))))?;
+		.map_err(|e| {
+			debug_warn!("Fetching media `{}` failed: {:?}", mxc, e);
+			Error::BadRequest(ErrorKind::NotFound, "Remote media error.")
+		})?;
 
 		let content_disposition = Some(make_content_disposition(
 			&response.content_type,
@@ -238,7 +235,7 @@ pub(crate) async fn get_content_route(
 			cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()),
 		})
 	} else {
-		Err!(Request(NotFound("Media not found.")))
+		Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
 	}
 }
 
@@ -325,10 +322,13 @@ pub(crate) async fn get_content_as_filename_route(
 					cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()),
 				})
 			},
-			Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))),
+			Err(e) => {
+				debug_warn!("Fetching media `{}` failed: {:?}", mxc, e);
+				Err(Error::BadRequest(ErrorKind::NotFound, "Remote media error."))
+			},
 		}
 	} else {
-		Err!(Request(NotFound("Media not found.")))
+		Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
 	}
 }
 
@@ -379,10 +379,10 @@ pub(crate) async fn get_content_thumbnail_route(
 			&mxc,
 			body.width
 				.try_into()
-				.map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?,
+				.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
 			body.height
 				.try_into()
-				.map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?,
+				.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Height is invalid."))?,
 		)
 		.await?
 	{
@@ -405,7 +405,7 @@ pub(crate) async fn get_content_thumbnail_route(
 			// we'll lie to the client and say the blocked server's media was not found and
 			// log. the client has no way of telling anyways so this is a security bonus.
 			debug_warn!("Received request for media `{}` on blocklisted server", mxc);
-			return Err!(Request(NotFound("Media not found.")));
+			return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."));
 		}
 
 		match services
@@ -454,10 +454,13 @@ pub(crate) async fn get_content_thumbnail_route(
 					content_disposition,
 				})
 			},
-			Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))),
+			Err(e) => {
+				debug_warn!("Fetching media `{}` failed: {:?}", mxc, e);
+				Err(Error::BadRequest(ErrorKind::NotFound, "Remote media error."))
+			},
 		}
 	} else {
-		Err!(Request(NotFound("Media not found.")))
+		Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
 	}
 }
 
@@ -495,7 +498,7 @@ async fn get_remote_content(
 		// we'll lie to the client and say the blocked server's media was not found and
 		// log. the client has no way of telling anyways so this is a security bonus.
 		debug_warn!("Received request for media `{mxc}` on blocklisted server");
-		return Err!(Request(NotFound("Media not found.")));
+		return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."));
 	}
 
 	let content_response = services
@@ -537,3 +540,222 @@ async fn get_remote_content(
 		cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()),
 	})
 }
+
+async fn download_image(services: &Services, client: &reqwest::Client, url: &str) -> Result<UrlPreviewData> {
+	let image = client.get(url).send().await?.bytes().await?;
+	let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH));
+
+	services
+		.media
+		.create(None, &mxc, None, None, &image)
+		.await?;
+
+	let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() {
+		Err(_) => (None, None),
+		Ok(reader) => match reader.into_dimensions() {
+			Err(_) => (None, None),
+			Ok((width, height)) => (Some(width), Some(height)),
+		},
+	};
+
+	Ok(UrlPreviewData {
+		image: Some(mxc),
+		image_size: Some(image.len()),
+		image_width: width,
+		image_height: height,
+		..Default::default()
+	})
+}
+
+async fn download_html(services: &Services, client: &reqwest::Client, url: &str) -> Result<UrlPreviewData> {
+	let mut response = client.get(url).send().await?;
+
+	let mut bytes: Vec<u8> = Vec::new();
+	while let Some(chunk) = response.chunk().await? {
+		bytes.extend_from_slice(&chunk);
+		if bytes.len() > services.globals.url_preview_max_spider_size() {
+			debug!(
+				"Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \
+				 response body and assuming our necessary data is in this range.",
+				url,
+				services.globals.url_preview_max_spider_size()
+			);
+			break;
+		}
+	}
+	let body = String::from_utf8_lossy(&bytes);
+	let Ok(html) = HTML::from_string(body.to_string(), Some(url.to_owned())) else {
+		return Err(Error::BadRequest(ErrorKind::Unknown, "Failed to parse HTML"));
+	};
+
+	let mut data = match html.opengraph.images.first() {
+		None => UrlPreviewData::default(),
+		Some(obj) => download_image(services, client, &obj.url).await?,
+	};
+
+	let props = html.opengraph.properties;
+
+	/* use OpenGraph title/description, but fall back to HTML if not available */
+	data.title = props.get("title").cloned().or(html.title);
+	data.description = props.get("description").cloned().or(html.description);
+
+	Ok(data)
+}
+
+async fn request_url_preview(services: &Services, url: &str) -> Result<UrlPreviewData> {
+	if let Ok(ip) = IPAddress::parse(url) {
+		if !services.globals.valid_cidr_range(&ip) {
+			return Err(Error::BadServerResponse("Requesting from this address is forbidden"));
+		}
+	}
+
+	let client = &services.client.url_preview;
+	let response = client.head(url).send().await?;
+
+	if let Some(remote_addr) = response.remote_addr() {
+		if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
+			if !services.globals.valid_cidr_range(&ip) {
+				return Err(Error::BadServerResponse("Requesting from this address is forbidden"));
+			}
+		}
+	}
+
+	let Some(content_type) = response
+		.headers()
+		.get(reqwest::header::CONTENT_TYPE)
+		.and_then(|x| x.to_str().ok())
+	else {
+		return Err(Error::BadRequest(ErrorKind::Unknown, "Unknown Content-Type"));
+	};
+	let data = match content_type {
+		html if html.starts_with("text/html") => download_html(services, client, url).await?,
+		img if img.starts_with("image/") => download_image(services, client, url).await?,
+		_ => return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported Content-Type")),
+	};
+
+	services.media.set_url_preview(url, &data).await?;
+
+	Ok(data)
+}
+
+async fn get_url_preview(services: &Services, url: &str) -> Result<UrlPreviewData> {
+	if let Some(preview) = services.media.get_url_preview(url).await {
+		return Ok(preview);
+	}
+
+	// ensure that only one request is made per URL
+	let _request_lock = services.media.url_preview_mutex.lock(url).await;
+
+	match services.media.get_url_preview(url).await {
+		Some(preview) => Ok(preview),
+		None => request_url_preview(services, url).await,
+	}
+}
+
+fn url_preview_allowed(services: &Services, url_str: &str) -> bool {
+	let url: Url = match Url::parse(url_str) {
+		Ok(u) => u,
+		Err(e) => {
+			warn!("Failed to parse URL from a str: {}", e);
+			return false;
+		},
+	};
+
+	if ["http", "https"]
+		.iter()
+		.all(|&scheme| scheme != url.scheme().to_lowercase())
+	{
+		debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
+		return false;
+	}
+
+	let host = match url.host_str() {
+		None => {
+			debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
+			return false;
+		},
+		Some(h) => h.to_owned(),
+	};
+
+	let allowlist_domain_contains = services.globals.url_preview_domain_contains_allowlist();
+	let allowlist_domain_explicit = services.globals.url_preview_domain_explicit_allowlist();
+	let denylist_domain_explicit = services.globals.url_preview_domain_explicit_denylist();
+	let allowlist_url_contains = services.globals.url_preview_url_contains_allowlist();
+
+	if allowlist_domain_contains.contains(&"*".to_owned())
+		|| allowlist_domain_explicit.contains(&"*".to_owned())
+		|| allowlist_url_contains.contains(&"*".to_owned())
+	{
+		debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url);
+		return true;
+	}
+
+	if !host.is_empty() {
+		if denylist_domain_explicit.contains(&host) {
+			debug!(
+				"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
+				&host
+			);
+			return false;
+		}
+
+		if allowlist_domain_explicit.contains(&host) {
+			debug!("Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", &host);
+			return true;
+		}
+
+		if allowlist_domain_contains
+			.iter()
+			.any(|domain_s| domain_s.contains(&host.clone()))
+		{
+			debug!("Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", &host);
+			return true;
+		}
+
+		if allowlist_url_contains
+			.iter()
+			.any(|url_s| url.to_string().contains(&url_s.to_string()))
+		{
+			debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host);
+			return true;
+		}
+
+		// check root domain if available and if user has root domain checks
+		if services.globals.url_preview_check_root_domain() {
+			debug!("Checking root domain");
+			match host.split_once('.') {
+				None => return false,
+				Some((_, root_domain)) => {
+					if denylist_domain_explicit.contains(&root_domain.to_owned()) {
+						debug!(
+							"Root domain {} is not allowed by url_preview_domain_explicit_denylist (check 1/3)",
+							&root_domain
+						);
+						return true;
+					}
+
+					if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
+						debug!(
+							"Root domain {} is allowed by url_preview_domain_explicit_allowlist (check 2/3)",
+							&root_domain
+						);
+						return true;
+					}
+
+					if allowlist_domain_contains
+						.iter()
+						.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
+					{
+						debug!(
+							"Root domain {} is allowed by url_preview_domain_contains_allowlist (check 3/3)",
+							&root_domain
+						);
+						return true;
+					}
+				},
+			}
+		}
+	}
+
+	false
+}

src/api/client/membership.rs

@@ -8,11 +8,11 @@ use std::{
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduit::{
-	debug, debug_error, debug_warn, err, error, info,
+	debug, debug_warn, error, info,
 	pdu::{gen_event_id_canonical_json, PduBuilder},
 	trace, utils,
 	utils::math::continue_exponential_backoff_secs,
-	warn, Err, Error, PduEvent, Result,
+	warn, Error, PduEvent, Result,
 };
 use ruma::{
 	api::{
@@ -705,11 +705,11 @@ async fn join_room_by_id_helper_remote(
 		{
 			room_version
 		},
-		_ => return Err!(BadServerResponse("Room version is not supported")),
+		_ => return Err(Error::BadServerResponse("Room version is not supported")),
 	};
 
 	let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get())
-		.map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?;
+		.map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?;
 
 	let join_authorized_via_users_server = join_event_stub
 		.get("content")
@@ -876,14 +876,15 @@ async fn join_room_by_id_helper_remote(
 
 	info!("Parsing join event");
 	let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone())
-		.map_err(|e| err!(BadServerResponse("Invalid join event PDU: {e:?}")))?;
+		.map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;
 
 	let mut state = HashMap::new();
 	let pub_key_map = RwLock::new(BTreeMap::new());
 
 	info!("Fetching join signing keys");
 	services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map)
 		.await?;
 
@@ -899,8 +900,8 @@ async fn join_room_by_id_helper_remote(
 		};
 
 		let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| {
-			debug_warn!("Invalid PDU in send_join response: {value:#?}");
+			warn!("Invalid PDU in send_join response: {} {:?}", e, value);
-			err!(BadServerResponse("Invalid PDU in send_join response: {e:?}"))
+			Error::BadServerResponse("Invalid PDU in send_join response.")
 		})?;
 
 		services.rooms.outlier.add_pdu_outlier(&event_id, &value)?;
@@ -1122,10 +1123,10 @@ async fn join_room_by_id_helper_local(
 			{
 				room_version_id
 			},
-			_ => return Err!(BadServerResponse("Room version is not supported")),
+			_ => return Err(Error::BadServerResponse("Room version is not supported")),
 		};
 		let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get())
-			.map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?;
+			.map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?;
 		let join_authorized_via_users_server = join_event_stub
 			.get("content")
 			.map(|s| {
@@ -1229,7 +1230,8 @@ async fn join_room_by_id_helper_local(
 			drop(state_lock);
 			let pub_key_map = RwLock::new(BTreeMap::new());
 			services
-				.server_keys
+				.rooms
+				.event_handler
 				.fetch_required_signing_keys([&signed_value], &pub_key_map)
 				.await?;
 			services
@@ -1250,7 +1252,7 @@ async fn join_room_by_id_helper_local(
 async fn make_join_request(
 	services: &Services, sender_user: &UserId, room_id: &RoomId, servers: &[OwnedServerName],
 ) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> {
-	let mut make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining."));
+	let mut make_join_response_and_server = Err(Error::BadServerResponse("No server available to assist in joining."));
 
 	let mut make_join_counter: u16 = 0;
 	let mut incompatible_room_version_count: u8 = 0;
@@ -1291,7 +1293,8 @@ async fn make_join_request(
 					"15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or M_UNSUPPORTED_ROOM_VERSION, \
 					 assuming that Conduwuit does not support the room {room_id}: {e}"
 				);
-				make_join_response_and_server = Err!(BadServerResponse("Room version is not supported by Conduwuit"));
+				make_join_response_and_server =
+					Err(Error::BadServerResponse("Room version is not supported by Conduwuit"));
 				return make_join_response_and_server;
 			}
 
@@ -1299,7 +1302,8 @@ async fn make_join_request(
 				warn!(
 					"50 servers failed to provide valid make_join response, assuming no server can assist in joining."
 				);
-				make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining."));
+				make_join_response_and_server =
+					Err(Error::BadServerResponse("No server available to assist in joining."));
 				return make_join_response_and_server;
 			}
 		}
@@ -1319,8 +1323,8 @@ pub async fn validate_and_add_event_id(
 	pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
 ) -> Result<(OwnedEventId, CanonicalJsonObject)> {
 	let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
-		debug_error!("Invalid PDU in server response: {pdu:#?}");
+		error!("Invalid PDU in server response: {:?}: {:?}", pdu, e);
-		err!(BadServerResponse("Invalid PDU in server response: {e:?}"))
+		Error::BadServerResponse("Invalid PDU in server response")
 	})?;
 	let event_id = EventId::parse(format!(
 		"${}",
@@ -1356,15 +1360,15 @@ pub async fn validate_and_add_event_id(
 		const MIN: u64 = 60 * 5;
 		const MAX: u64 = 60 * 60 * 24;
 		if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) {
-			return Err!(BadServerResponse("bad event {event_id:?}, still backing off"));
+			debug!("Backing off from {event_id}");
+			return Err(Error::BadServerResponse("bad event, still backing off"));
 		}
 	}
 
 	if let Err(e) = ruma::signatures::verify_event(&*pub_key_map.read().await, &value, room_version) {
-		debug_error!("Event {event_id} failed verification {pdu:#?}");
+		warn!("Event {} failed verification {:?} {}", event_id, pdu, e);
-		let e = Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}")));
 		back_off(event_id).await;
-		return e;
+		return Err(Error::BadServerResponse("Event failed verification."));
 	}
 
 	value.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned()));
@@ -1470,7 +1474,8 @@ pub(crate) async fn invite_helper(
 		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?;
 
 		services
-			.server_keys
+			.rooms
+			.event_handler
 			.fetch_required_signing_keys([&value], &pub_key_map)
 			.await?;
 
@@ -1646,7 +1651,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId,
 }
 
 async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<()> {
-	let mut make_leave_response_and_server = Err!(BadServerResponse("No server available to assist in leaving."));
+	let mut make_leave_response_and_server = Err(Error::BadServerResponse("No server available to assist in leaving."));
 
 	let invite_state = services
 		.rooms
@@ -1703,11 +1708,11 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room
 		{
 			version
 		},
-		_ => return Err!(BadServerResponse("Room version is not supported")),
+		_ => return Err(Error::BadServerResponse("Room version is not supported")),
 	};
 
 	let mut leave_event_stub = serde_json::from_str::<CanonicalJsonObject>(make_leave_response.event.get())
-		.map_err(|e| err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")))?;
+		.map_err(|_| Error::BadServerResponse("Invalid make_leave event json received from server."))?;
 
 	// TODO: Is origin needed?
 	leave_event_stub.insert(

src/api/client/room.rs

@@ -1,7 +1,7 @@
 use std::{cmp::max, collections::BTreeMap};
 
 use axum::extract::State;
-use conduit::{debug_info, debug_warn};
+use conduit::{debug_info, debug_warn, err};
 use ruma::{
 	api::client::{
 		error::ErrorKind,
@@ -20,6 +20,7 @@ use ruma::{
 			tombstone::RoomTombstoneEventContent,
 			topic::RoomTopicEventContent,
 		},
+		space::child::SpaceChildEventContent,
 		StateEventType, TimelineEventType,
 	},
 	int,
@@ -475,10 +476,7 @@ pub(crate) async fn get_room_event_route(
 		.rooms
 		.timeline
 		.get_pdu(&body.event_id)?
-		.ok_or_else(|| {
+		.ok_or_else(|| err!(Request(NotFound("Event {} not found.", &body.event_id))))?;
-			warn!("Event not found, event ID: {:?}", &body.event_id);
-			Error::BadRequest(ErrorKind::NotFound, "Event not found.")
-		})?;
 
 	if !services
 		.rooms
@@ -621,22 +619,14 @@ pub(crate) async fn upgrade_room_route(
 			V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => {
 				create_event_content.insert(
 					"creator".into(),
-					json!(&sender_user).try_into().map_err(|e| {
+					json!(&sender_user)
-						info!("Error forming creation event: {e}");
+						.try_into()
-						Error::BadRequest(ErrorKind::BadJson, "Error forming creation event")
+						.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
-					})?,
 				);
 			},
-			V11 => {
-				// "creator" key no longer exists in V11 rooms
-				create_event_content.remove("creator");
-			},
 			_ => {
-				warn!("Unexpected or unsupported room version {}", body.new_version);
+				// "creator" key no longer exists in V11+ rooms
-				return Err(Error::BadRequest(
+				create_event_content.remove("creator");
-					ErrorKind::BadJson,
-					"Unexpected or unsupported room version found",
-				));
 			},
 		}
 	}
@@ -645,26 +635,100 @@ pub(crate) async fn upgrade_room_route(
 		"room_version".into(),
 		json!(&body.new_version)
 			.try_into()
-			.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?,
+			.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
 	);
 	create_event_content.insert(
 		"predecessor".into(),
 		json!(predecessor)
 			.try_into()
-			.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?,
+			.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
 	);
 
-	// Validate creation event content
+	// if the room was a space:
-	if serde_json::from_str::<CanonicalJsonObject>(
+	// - migrate m.space.child and/or m.space.parent
-		to_raw_value(&create_event_content)
+	// - add space type to replacement room m.room.create
-			.expect("Error forming creation event")
+	//
-			.get(),
+	// as apart of MSC4168
-	)
+	if services
-	.is_err()
+		.rooms
+		.state_accessor
+		.get_room_type(&body.room_id)
+		.unwrap_or(None)
+		.is_some_and(|room_type| room_type == ruma::room::RoomType::Space)
 	{
-		return Err(Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"));
+		create_event_content.insert(
+			"type".into(),
+			json!(ruma::room::RoomType::Space)
+				.try_into()
+				.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
+		);
+
+		if let Some(event_content) = services
+			.rooms
+			.state_accessor
+			.room_state_get(&body.room_id, &StateEventType::SpaceChild, body.room_id.as_str())
+			.unwrap_or(None)
+		{
+			// space contents that are empty are not apart of the space
+			if !event_content.content.get().is_empty() {
+				// remove all but us from via
+				let mut new_content: SpaceChildEventContent = serde_json::from_str(event_content.content.get())
+					.map_err(|_| err!(Database(error!("Invalid m.space.child content in database"))))?;
+				new_content.via = vec![services.globals.config.server_name.clone()];
+
+				services
+					.rooms
+					.timeline
+					.build_and_append_pdu(
+						PduBuilder {
+							event_type: StateEventType::SpaceChild.to_string().into(),
+							content: to_raw_value(&new_content).expect("we just created it"),
+							unsigned: None,
+							state_key: Some(replacement_room.to_string()),
+							redacts: None,
+						},
+						sender_user,
+						&replacement_room,
+						&state_lock,
+					)
+					.await?;
+			}
 		}
 
+		if let Some(event_content) = services
+			.rooms
+			.state_accessor
+			.room_state_get(&body.room_id, &StateEventType::SpaceParent, body.room_id.as_str())
+			.unwrap_or(None)
+		{
+			// space contents that are empty are not apart of the space
+			if !event_content.content.get().is_empty() {
+				// remove all but us from via
+				let mut new_content: SpaceChildEventContent = serde_json::from_str(event_content.content.get())
+					.map_err(|_| err!(Database(error!("Invalid m.space.child content in database"))))?;
+				new_content.via = vec![services.globals.config.server_name.clone()];
+
+				services
+					.rooms
+					.timeline
+					.build_and_append_pdu(
+						PduBuilder {
+							event_type: StateEventType::SpaceParent.to_string().into(),
+							content: to_raw_value(&new_content).expect("we just created it"),
+							unsigned: None,
+							state_key: Some(replacement_room.to_string()),
+							redacts: None,
+						},
+						sender_user,
+						&replacement_room,
+						&state_lock,
+					)
+					.await?;
+			}
+		}
+	}
+
+	// m.room.create
 	services
 		.rooms
 		.timeline
@@ -758,11 +822,11 @@ pub(crate) async fn upgrade_room_route(
 			.rooms
 			.state_accessor
 			.room_state_get(&body.room_id, &StateEventType::RoomPowerLevels, "")?
-			.ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
+			.ok_or_else(|| err!(Database(error!("Found room without m.room.create event."))))?
 			.content
 			.get(),
 	)
-	.map_err(|_| Error::bad_database("Invalid room event in database."))?;
+	.map_err(|_| err!(Database(error!("Invalid room event in database."))))?;
 
 	// Setting events_default and invite to the greater of 50 and users_default + 1
 	let new_level = max(
@@ -770,9 +834,7 @@ pub(crate) async fn upgrade_room_route(
 		power_levels_event_content
 			.users_default
 			.checked_add(int!(1))
-			.ok_or_else(|| {
+			.ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?,
-				Error::BadRequest(ErrorKind::BadJson, "users_default power levels event content is not valid")
-			})?,
 	);
 	power_levels_event_content.events_default = new_level;
 	power_levels_event_content.invite = new_level;
@@ -836,7 +898,7 @@ fn default_power_levels_content(
 
 	if let Some(power_level_content_override) = power_level_content_override {
 		let json: JsonObject = serde_json::from_str(power_level_content_override.json().get())
-			.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override."))?;
+			.map_err(|_| err!(Request(BadJson("Invalid power_level_content_override"))))?;
 
 		for (key, value) in json {
 			power_levels_content[key] = value;

src/api/client/session.rs

@@ -1,4 +1,5 @@
 use axum::extract::State;
+use axum_client_ip::InsecureClientIp;
 use ruma::{
 	api::client::{
 		error::ErrorKind,
@@ -56,8 +57,9 @@ pub(crate) async fn get_login_types_route(
 /// Note: You can use [`GET
 /// /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see
 /// supported login types.
+#[tracing::instrument(skip_all, fields(%client), name = "login")]
 pub(crate) async fn login_route(
-	State(services): State<crate::State>, body: Ruma<login::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp, body: Ruma<login::v3::Request>,
 ) -> Result<login::v3::Response> {
 	// Validate login method
 	// TODO: Other login methods
@@ -176,9 +178,13 @@ pub(crate) async fn login_route(
 	if device_exists {
 		services.users.set_token(&user_id, &device_id, &token)?;
 	} else {
-		services
+		services.users.create_device(
-			.users
+			&user_id,
-			.create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?;
+			&device_id,
+			&token,
+			body.initial_device_display_name.clone(),
+			Some(client.to_string()),
+		)?;
 	}
 
 	// send client well-known if specified so the client knows to reconfigure itself
@@ -214,8 +220,9 @@ pub(crate) async fn login_route(
 ///   last seen ts)
 /// - Forgets to-device events
 /// - Triggers device list updates
+#[tracing::instrument(skip_all, fields(%client), name = "logout")]
 pub(crate) async fn logout_route(
-	State(services): State<crate::State>, body: Ruma<logout::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp, body: Ruma<logout::v3::Request>,
 ) -> Result<logout::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
@@ -241,8 +248,10 @@ pub(crate) async fn logout_route(
 /// Note: This is equivalent to calling [`GET
 /// /_matrix/client/r0/logout`](fn.logout_route.html) from each device of this
 /// user.
+#[tracing::instrument(skip_all, fields(%client), name = "logout")]
 pub(crate) async fn logout_all_route(
-	State(services): State<crate::State>, body: Ruma<logout_all::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<logout_all::v3::Request>,
 ) -> Result<logout_all::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 

src/api/client/sync.rs

@@ -106,11 +106,13 @@ pub(crate) async fn sync_events_route(
 			.unwrap_or_default(),
 	};
 
+	// some clients, at least element, seem to require knowledge of redundant
+	// members for "inline" profiles on the timeline to work properly
 	let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options {
 		LazyLoadOptions::Enabled {
-			include_redundant_members: redundant,
+			include_redundant_members,
-		} => (true, redundant),
+		} => (true, include_redundant_members),
-		LazyLoadOptions::Disabled => (false, false),
+		LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
 	};
 
 	let full_state = body.full_state;

src/api/client/voip.rs

@@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime};
 
 use axum::extract::State;
 use base64::{engine::general_purpose, Engine as _};
-use conduit::utils;
+use conduit::{utils, Err};
 use hmac::{Hmac, Mac};
 use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId};
 use sha1::Sha1;
@@ -19,6 +19,11 @@ type HmacSha1 = Hmac<Sha1>;
 pub(crate) async fn turn_server_route(
 	State(services): State<crate::State>, body: Ruma<get_turn_server_info::v3::Request>,
 ) -> Result<get_turn_server_info::v3::Response> {
+	// MSC4166: return M_NOT_FOUND 404 if no TURN URIs are specified in any way
+	if services.server.config.turn_uris.is_empty() {
+		return Err!(Request(NotFound("Not Found")));
+	}
+
 	let turn_secret = services.globals.turn_secret().clone();
 
 	let (username, password) = if !turn_secret.is_empty() {

src/api/mod.rs

@@ -10,7 +10,8 @@ extern crate conduit_service as service;
 pub(crate) use conduit::{debug_info, pdu::PduEvent, utils, Error, Result};
 pub(crate) use service::services;
 
-pub(crate) use self::router::{Ruma, RumaResponse, State};
+pub use crate::router::State;
+pub(crate) use crate::router::{Ruma, RumaResponse};
 
 conduit::mod_ctor! {}
 conduit::mod_dtor! {}

src/api/router.rs

@@ -3,7 +3,8 @@ mod auth;
 mod handler;
 mod request;
 mod response;
-pub mod state;
+
+use std::sync::Arc;
 
 use axum::{
 	response::IntoResponse,
@@ -14,9 +15,11 @@ use conduit::{err, Server};
 use http::Uri;
 
 use self::handler::RouterExt;
-pub(super) use self::{args::Args as Ruma, response::RumaResponse, state::State};
+pub(super) use self::{args::Args as Ruma, response::RumaResponse};
 use crate::{client, server};
 
+pub type State = Arc<service::Services>;
+
 pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 	let config = &server.config;
 	let router = router

src/api/router/auth.rs

@@ -6,7 +6,7 @@ use axum_extra::{
 	typed_header::TypedHeaderRejectionReason,
 	TypedHeader,
 };
-use conduit::{warn, Err, Error, Result};
+use conduit::{debug_info, warn, Err, Error, Result};
 use http::uri::PathAndQuery;
 use ruma::{
 	api::{client::error::ErrorKind, AuthScheme, Metadata},
@@ -185,7 +185,7 @@ fn auth_appservice(services: &Services, request: &Request, info: Box<Registratio
 async fn auth_server(
 	services: &Services, request: &mut Request, json_body: &Option<CanonicalJsonValue>,
 ) -> Result<Auth> {
-	if !services.globals.allow_federation() {
+	if !services.server.config.allow_federation {
 		return Err!(Config("allow_federation", "Federation is disabled."));
 	}
 
@@ -206,6 +206,17 @@ async fn auth_server(
 		})?;
 
 	let origin = &x_matrix.origin;
+
+	if services
+		.server
+		.config
+		.forbidden_remote_server_names
+		.contains(origin)
+	{
+		debug_info!("Refusing to accept inbound federation request to {origin}");
+		return Err!(Request(Forbidden("Federation with this homeserver is not allowed.")));
+	}
+
 	let signatures =
 		BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig.to_string()))]);
 	let signatures = BTreeMap::from_iter([(
@@ -250,7 +261,8 @@ async fn auth_server(
 	};
 
 	let keys_result = services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_signing_keys_for_server(origin, vec![x_matrix.key.to_string()])
 		.await;
 

src/api/router/state.rs

@@ -1,79 +0,0 @@
-use std::{ops::Deref, sync::Arc};
-
-use conduit_service::Services;
-
-#[derive(Clone, Copy)]
-pub struct State {
-	services: *const Services,
-}
-
-pub struct Guard {
-	services: Arc<Services>,
-}
-
-pub fn create(services: Arc<Services>) -> (State, Guard) {
-	let state = State {
-		services: Arc::into_raw(services.clone()),
-	};
-
-	let guard = Guard {
-		services,
-	};
-
-	(state, guard)
-}
-
-impl Drop for Guard {
-	fn drop(&mut self) {
-		let ptr = Arc::as_ptr(&self.services);
-		// SAFETY: Parity with Arc::into_raw() called in create(). This revivifies the
-		// Arc lost to State so it can be dropped, otherwise Services will leak.
-		let arc = unsafe { Arc::from_raw(ptr) };
-		debug_assert!(
-			Arc::strong_count(&arc) > 1,
-			"Services usually has more than one reference and is not dropped here"
-		);
-	}
-}
-
-impl Deref for State {
-	type Target = Services;
-
-	fn deref(&self) -> &Self::Target {
-		deref(&self.services).expect("dereferenced Services pointer in State must not be null")
-	}
-}
-
-/// SAFETY: State is a thin wrapper containing a raw const pointer to Services
-/// in lieu of an Arc. Services is internally threadsafe. If State contains
-/// additional fields this notice should be reevaluated.
-unsafe impl Send for State {}
-
-/// SAFETY: State is a thin wrapper containing a raw const pointer to Services
-/// in lieu of an Arc. Services is internally threadsafe. If State contains
-/// additional fields this notice should be reevaluated.
-unsafe impl Sync for State {}
-
-fn deref(services: &*const Services) -> Option<&Services> {
-	// SAFETY: We replaced Arc<Services> with *const Services in State. This is
-	// worth about 10 clones (20 reference count updates) for each request handled.
-	// Though this is not an incredibly large quantity, it's woefully unnecessary
-	// given the context as explained below; though it is not currently known to be
-	// a performance bottleneck, the front-line position justifies preempting it.
-	//
-	// Services is created prior to the axum/tower stack and Router, and prior
-	// to serving any requests through the handlers in this crate. It is then
-	// dropped only after all requests have completed, the listening sockets
-	// have been closed, axum/tower has been dropped. Thus Services is
-	// expected to live at least as long as any instance of State, making the
-	// constant updates to the prior Arc unnecessary to keep Services alive.
-	//
-	// Nevertheless if it is possible to accomplish this by annotating State
-	// with a lifetime to hold a reference (and be aware I have made a
-	// significant effort trying to make this work) this unsafety may not be
-	// necessary. It is either very difficult or impossible to get a
-	// lifetime'ed reference through Router / RumaHandler; though it is
-	// possible to pass a reference through axum's `with_state()` in trivial
-	// configurations as the only requirement of a State is Clone.
-	unsafe { services.as_ref() }
-}

src/api/server/send.rs

@@ -105,7 +105,8 @@ async fn handle_pdus(
 	let pub_key_map = RwLock::new(BTreeMap::new());
 	if !parsed_pdus.is_empty() {
 		services
-			.server_keys
+			.rooms
+			.event_handler
 			.fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map)
 			.await
 			.unwrap_or_else(|e| warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}"));

src/api/server/send_join.rs

@@ -138,7 +138,8 @@ async fn create_join_event(
 	}
 
 	services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_required_signing_keys([&value], &pub_key_map)
 		.await?;
 

src/api/server/send_leave.rs

@@ -150,7 +150,8 @@ async fn create_leave_event(
 	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?;
 
 	services
-		.server_keys
+		.rooms
+		.event_handler
 		.fetch_required_signing_keys([&value], &pub_key_map)
 		.await?;
 

src/core/config/mod.rs

@@ -287,7 +287,7 @@ pub struct Config {
 
 	#[serde(default = "true_fn")]
 	pub media_startup_check: bool,
-	#[serde(default)]
+	#[serde(default = "true_fn")]
 	pub media_compat_file_link: bool,
 	#[serde(default = "Vec::new")]
 	pub prevent_media_downloads_from: Vec<OwnedServerName>,

src/core/debug.rs

@@ -14,7 +14,7 @@ pub use crate::utils::debug::*;
 #[macro_export]
 macro_rules! debug_event {
 	( $level:expr, $($x:tt)+ ) => {
-		if $crate::debug::logging() {
+		if cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) {
 			::tracing::event!( $level, $($x)+ )
 		} else {
 			::tracing::debug!( $($x)+ )
@@ -88,7 +88,3 @@ pub fn panic_str(p: &Box<dyn Any + Send>) -> &'static str { p.downcast_ref::<&st
 #[inline(always)]
 #[must_use]
 pub fn type_name<T>(_: &T) -> &'static str { std::any::type_name::<T>() }
-
-#[must_use]
-#[inline]
-pub const fn logging() -> bool { cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) }

src/core/error/err.rs

@@ -41,149 +41,60 @@ macro_rules! Err {
 
 #[macro_export]
 macro_rules! err {
-	(Request(Forbidden($level:ident!($($args:tt)+)))) => {{
+	(Config($item:literal, $($args:expr),*)) => {{
-		let mut buf = String::new();
+		$crate::error!(config = %$item, $($args),*);
+		$crate::error::Error::Config($item, $crate::format_maybe!($($args),*))
+	}};
+
+	(Request(Forbidden($level:ident!($($args:expr),*)))) => {{
+		$crate::$level!($($args),*);
 		$crate::error::Error::Request(
 			::ruma::api::client::error::ErrorKind::forbidden(),
-			$crate::err_log!(buf, $level, $($args)+),
+			$crate::format_maybe!($($args),*),
 			::http::StatusCode::BAD_REQUEST
 		)
 	}};
 
-	(Request(Forbidden($($args:tt)+))) => {
+	(Request(Forbidden($($args:expr),*))) => {
 		$crate::error::Error::Request(
 			::ruma::api::client::error::ErrorKind::forbidden(),
-			$crate::format_maybe!($($args)+),
+			$crate::format_maybe!($($args),*),
 			::http::StatusCode::BAD_REQUEST
 		)
 	};
 
-	(Request($variant:ident($level:ident!($($args:tt)+)))) => {{
+	(Request($variant:ident($level:ident!($($args:expr),*)))) => {{
-		let mut buf = String::new();
+		$crate::$level!($($args),*);
 		$crate::error::Error::Request(
 			::ruma::api::client::error::ErrorKind::$variant,
-			$crate::err_log!(buf, $level, $($args)+),
+			$crate::format_maybe!($($args),*),
 			::http::StatusCode::BAD_REQUEST
 		)
 	}};
 
-	(Request($variant:ident($($args:tt)+))) => {
+	(Request($variant:ident($($args:expr),*))) => {
 		$crate::error::Error::Request(
 			::ruma::api::client::error::ErrorKind::$variant,
-			$crate::format_maybe!($($args)+),
+			$crate::format_maybe!($($args),*),
 			::http::StatusCode::BAD_REQUEST
 		)
 	};
 
-	(Config($item:literal, $($args:tt)+)) => {{
+	($variant:ident($level:ident!($($args:expr),*))) => {{
-		let mut buf = String::new();
+		$crate::$level!($($args),*);
-		$crate::error::Error::Config($item, $crate::err_log!(buf, error, config = %$item, $($args)+))
+		$crate::error::Error::$variant($crate::format_maybe!($($args),*))
 	}};
 
-	($variant:ident($level:ident!($($args:tt)+))) => {{
+	($variant:ident($($args:expr),*)) => {
-		let mut buf = String::new();
+		$crate::error::Error::$variant($crate::format_maybe!($($args),*))
-		$crate::error::Error::$variant($crate::err_log!(buf, $level, $($args)+))
+	};
+
+	($level:ident!($($args:expr),*)) => {{
+		$crate::$level!($($args),*);
+		$crate::error::Error::Err($crate::format_maybe!($($args),*))
 	}};
 
-	($variant:ident($($args:tt)+)) => {
+	($($args:expr),*) => {
-		$crate::error::Error::$variant($crate::format_maybe!($($args)+))
+		$crate::error::Error::Err($crate::format_maybe!($($args),*))
-	};
-
-	($level:ident!($($args:tt)+)) => {{
-		let mut buf = String::new();
-		$crate::error::Error::Err($crate::err_log!(buf, $level, $($args)+))
-	}};
-
-	($($args:tt)+) => {
-		$crate::error::Error::Err($crate::format_maybe!($($args)+))
-	};
-}
-
-/// A trinity of integration between tracing, logging, and Error. This is a
-/// customization of tracing::event! with the primary purpose of sharing the
-/// error string, fieldset parsing and formatting. An added benefit is that we
-/// can share the same callsite metadata for the source of our Error and the
-/// associated logging and tracing event dispatches.
-#[macro_export]
-macro_rules! err_log {
-	($out:ident, $level:ident, $($fields:tt)+) => {{
-		use std::{fmt, fmt::Write};
-
-		use ::tracing::{
-			callsite, callsite2, level_enabled, metadata, valueset, Callsite, Event, __macro_support,
-			__tracing_log,
-			field::{Field, ValueSet, Visit},
-			Level,
-		};
-
-		const LEVEL: Level = $crate::err_lev!($level);
-		static __CALLSITE: callsite::DefaultCallsite = callsite2! {
-			name: std::concat! {
-				"event ",
-				std::file!(),
-				":",
-				std::line!(),
-			},
-			kind: metadata::Kind::EVENT,
-			target: std::module_path!(),
-			level: LEVEL,
-			fields: $($fields)+,
-		};
-
-		let visit = &mut |vs: ValueSet<'_>| {
-			struct Visitor<'a>(&'a mut String);
-			impl Visit for Visitor<'_> {
-				fn record_debug(&mut self, field: &Field, val: &dyn fmt::Debug) {
-					if field.name() == "message" {
-						write!(self.0, "{:?}", val).expect("stream error");
-					} else {
-						write!(self.0, " {}={:?}", field.name(), val).expect("stream error");
-					}
-				}
-			}
-
-			let meta = __CALLSITE.metadata();
-			let enabled = level_enabled!(LEVEL) && {
-				let interest = __CALLSITE.interest();
-				!interest.is_never() && __macro_support::__is_enabled(meta, interest)
-			};
-
-			if enabled {
-				Event::dispatch(meta, &vs);
-			}
-
-			__tracing_log!(LEVEL, __CALLSITE, &vs);
-			vs.record(&mut Visitor(&mut $out));
-		};
-
-		(visit)(valueset!(__CALLSITE.metadata().fields(), $($fields)+));
-		($out).into()
-	}}
-}
-
-#[macro_export]
-macro_rules! err_lev {
-	(debug_warn) => {
-		if $crate::debug::logging() {
-			::tracing::Level::WARN
-		} else {
-			::tracing::Level::DEBUG
-		}
-	};
-
-	(debug_error) => {
-		if $crate::debug::logging() {
-			::tracing::Level::ERROR
-		} else {
-			::tracing::Level::DEBUG
-		}
-	};
-
-	(warn) => {
-		::tracing::Level::WARN
-	};
-
-	(error) => {
-		::tracing::Level::ERROR
 	};
 }

src/core/error/mod.rs

@@ -5,7 +5,8 @@ mod response;
 
 use std::{any::Any, borrow::Cow, convert::Infallible, fmt};
 
-pub use self::log::*;
+pub use log::*;
+
 use crate::error;
 
 #[derive(thiserror::Error)]
@@ -86,13 +87,13 @@ pub enum Error {
 	// conduwuit
 	#[error("Arithmetic operation failed: {0}")]
 	Arithmetic(Cow<'static, str>),
-	#[error("{0}")]
-	BadServerResponse(Cow<'static, str>),
 	#[error("There was a problem with the '{0}' directive in your configuration: {1}")]
 	Config(&'static str, Cow<'static, str>),
 	#[error("{0}")]
 	Database(Cow<'static, str>),
 	#[error("{0}")]
+	BadServerResponse(&'static str),
+	#[error("{0}")]
 	Conflict(&'static str), // This is only needed for when a room alias already exists
 
 	// unique / untyped

src/core/mod.rs

@@ -16,7 +16,7 @@ pub use error::Error;
 pub use info::{rustc_flags_capture, version, version::version};
 pub use pdu::{PduBuilder, PduCount, PduEvent};
 pub use server::Server;
-pub use utils::{ctor, dtor, implement};
+pub use utils::{ctor, dtor};
 
 pub use crate as conduit_core;
 

src/core/pdu/mod.rs

@@ -23,7 +23,7 @@ use serde_json::{
 	value::{to_raw_value, RawValue as RawJsonValue},
 };
 
-use crate::{err, warn, Error};
+use crate::{warn, Error};
 
 #[derive(Deserialize)]
 struct ExtractRedactedBecause {
@@ -396,8 +396,10 @@ impl Ord for PduEvent {
 pub fn gen_event_id_canonical_json(
 	pdu: &RawJsonValue, room_version_id: &RoomVersionId,
 ) -> crate::Result<(OwnedEventId, CanonicalJsonObject)> {
-	let value: CanonicalJsonObject = serde_json::from_str(pdu.get())
+	let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
-		.map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?;
+		warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
+		Error::BadServerResponse("Invalid PDU in server response")
+	})?;
 
 	let event_id = format!(
 		"${}",

src/core/utils/mod.rs

@@ -17,7 +17,6 @@ use std::cmp::{self, Ordering};
 
 pub use ::ctor::{ctor, dtor};
 pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8};
-pub use conduit_macros::implement;
 pub use debug::slice_truncated as debug_slice_truncated;
 pub use hash::calculate_hash;
 pub use html::Escape as HtmlEscape;

src/core/utils/string.rs

@@ -8,12 +8,12 @@ pub const EMPTY: &str = "";
 /// arguments are provided the first is assumed to be a format string.
 #[macro_export]
 macro_rules! format_maybe {
-	($s:literal $(,)?) => {
+	($s:literal) => {
 		if $crate::is_format!($s) { std::format!($s).into() } else { $s.into() }
 	};
 
-	($s:literal, $($args:tt)+) => {
+	($($args:expr),*) => {
-		std::format!($s, $($args)+).into()
+		std::format!($($args),*).into()
 	};
 }
 
@@ -24,10 +24,6 @@ macro_rules! is_format {
 	($s:literal) => {
 		::const_str::contains!($s, "{") && ::const_str::contains!($s, "}")
 	};
-
-	($($s:tt)+) => {
-		false
-	};
 }
 
 #[inline]

src/router/layers.rs

@@ -6,7 +6,6 @@ use axum::{
 };
 use axum_client_ip::SecureClientIpSource;
 use conduit::{error, Result, Server};
-use conduit_api::router::state::Guard;
 use conduit_service::Services;
 use http::{
 	header::{self, HeaderName},
@@ -36,7 +35,7 @@ const CONDUWUIT_CSP: &[&str] = &[
 
 const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"];
 
-pub(crate) fn build(services: &Arc<Services>) -> Result<(Router, Guard)> {
+pub(crate) fn build(services: &Arc<Services>) -> Result<Router> {
 	let server = &services.server;
 	let layers = ServiceBuilder::new();
 
@@ -48,7 +47,7 @@ pub(crate) fn build(services: &Arc<Services>) -> Result<(Router, Guard)> {
 
 	let layers = layers
 		.sensitive_headers([header::AUTHORIZATION])
-		.layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::spawn))
+		.layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::spawn))
 		.layer(
 			TraceLayer::new_for_http()
 				.make_span_with(tracing_span::<_>)
@@ -56,7 +55,7 @@ pub(crate) fn build(services: &Arc<Services>) -> Result<(Router, Guard)> {
 				.on_request(DefaultOnRequest::new().level(Level::TRACE))
 				.on_response(DefaultOnResponse::new().level(Level::DEBUG)),
 		)
-		.layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::handle))
+		.layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::handle))
 		.layer(SecureClientIpSource::ConnectInfo.into_extension())
 		.layer(SetResponseHeaderLayer::if_not_present(
 			HeaderName::from_static("origin-agent-cluster"), // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin-Agent-Cluster
@@ -86,8 +85,7 @@ pub(crate) fn build(services: &Arc<Services>) -> Result<(Router, Guard)> {
 		.layer(body_limit_layer(server))
 		.layer(CatchPanicLayer::custom(catch_panic));
 
-	let (router, guard) = router::build(services);
+	Ok(router::build(services).layer(layers))
-	Ok((router.layer(layers), guard))
 }
 
 #[cfg(any(feature = "zstd_compression", feature = "gzip_compression", feature = "brotli_compression"))]

src/router/request.rs

@@ -4,15 +4,13 @@ use axum::{
 	extract::State,
 	response::{IntoResponse, Response},
 };
-use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result};
+use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result, Server};
-use conduit_service::Services;
 use http::{Method, StatusCode, Uri};
 
 #[tracing::instrument(skip_all, level = "debug")]
 pub(crate) async fn spawn(
-	State(services): State<Arc<Services>>, req: http::Request<axum::body::Body>, next: axum::middleware::Next,
+	State(server): State<Arc<Server>>, req: http::Request<axum::body::Body>, next: axum::middleware::Next,
 ) -> Result<Response, StatusCode> {
-	let server = &services.server;
 	if !server.running() {
 		debug_warn!("unavailable pending shutdown");
 		return Err(StatusCode::SERVICE_UNAVAILABLE);
@@ -36,9 +34,8 @@ pub(crate) async fn spawn(
 
 #[tracing::instrument(skip_all, name = "handle")]
 pub(crate) async fn handle(
-	State(services): State<Arc<Services>>, req: http::Request<axum::body::Body>, next: axum::middleware::Next,
+	State(server): State<Arc<Server>>, req: http::Request<axum::body::Body>, next: axum::middleware::Next,
 ) -> Result<Response, StatusCode> {
-	let server = &services.server;
 	if !server.running() {
 		debug_warn!(
 			method = %req.method(),

src/router/router.rs

@@ -2,20 +2,19 @@ use std::sync::Arc;
 
 use axum::{response::IntoResponse, routing::get, Router};
 use conduit::Error;
-use conduit_api::router::{state, state::Guard};
+use conduit_api::State;
 use conduit_service::Services;
 use http::{StatusCode, Uri};
 use ruma::api::client::error::ErrorKind;
 
-pub(crate) fn build(services: &Arc<Services>) -> (Router, Guard) {
+pub(crate) fn build(services: &Arc<Services>) -> Router {
-	let router = Router::<state::State>::new();
+	let router = Router::<State>::new();
-	let (state, guard) = state::create(services.clone());
+	let state = services.clone();
-	let router = conduit_api::router::build(router, &services.server)
+
+	conduit_api::router::build(router, &services.server)
 		.route("/", get(it_works))
 		.fallback(not_found)
-		.with_state(state);
+		.with_state(state)
-
-	(router, guard)
 }
 
 async fn not_found(_uri: Uri) -> impl IntoResponse {

src/router/serve/mod.rs

@@ -18,7 +18,7 @@ pub(super) async fn serve(
 	let server = &services.server;
 	let config = &server.config;
 	let addrs = config.get_bind_addrs();
-	let (app, _guard) = layers::build(&services)?;
+	let app = layers::build(&services)?;
 
 	if cfg!(unix) && config.unix_socket_path.is_some() {
 		unix::serve(server, app, shutdown).await

src/service/Cargo.toml

@@ -69,7 +69,6 @@ termimad.optional = true
 tokio.workspace = true
 tracing.workspace = true
 url.workspace = true
-webpage.workspace = true
 
 [lints]
 workspace = true

src/service/globals/migrations.rs

@@ -743,7 +743,6 @@ async fn db_lt_13(services: &Services) -> Result<()> {
 /// again.
 async fn migrate_sha256_media(services: &Services) -> Result<()> {
 	let db = &services.db;
-	let config = &services.server.config;
 
 	warn!("Migrating legacy base64 file names to sha256 file names");
 	let mediaid_file = &db["mediaid_file"];
@@ -760,11 +759,9 @@ async fn migrate_sha256_media(services: &Services) -> Result<()> {
 	for (old_path, path) in changes {
 		if old_path.exists() {
 			tokio::fs::rename(&old_path, &path).await?;
-			if config.media_compat_file_link {
 			tokio::fs::symlink(&path, &old_path).await?;
 		}
 	}
-	}
 
 	// Apply fix from when sha256_media was backward-incompat and bumped the schema
 	// version from 13 to 14. For users satisfying these conditions we can go back.
@@ -825,14 +822,8 @@ async fn handle_media_check(
 
 	let (mediaid_file, mediaid_user) = dbs;
 
-	let new_exists = files.contains(new_path);
 	let old_exists = files.contains(old_path);
-	let old_is_symlink = || async {
+	let new_exists = files.contains(new_path);
-		tokio::fs::symlink_metadata(old_path)
-			.await
-			.map_or(false, |md| md.is_symlink())
-	};
-
 	if !old_exists && !new_exists {
 		error!(
 			media_id = ?encode_key(key), ?new_path, ?old_path,
@@ -858,29 +849,10 @@ async fn handle_media_check(
 			"Legacy media found without sha256 migration. Fixing..."
 		);
 
-		debug_assert!(
-			old_is_symlink().await,
-			"Legacy media not expected to be a symlink without an existing sha256 migration."
-		);
-
 		tokio::fs::rename(&old_path, &new_path).await?;
 		tokio::fs::symlink(&new_path, &old_path).await?;
 	}
 
-	if !config.media_compat_file_link && old_exists && old_is_symlink().await {
-		debug_warn!(
-			media_id = ?encode_key(key), ?new_path, ?old_path,
-			"Legacy link found but compat disabled. Cleansing symlink..."
-		);
-
-		debug_assert!(
-			new_exists,
-			"sha256 migration into new file expected prior to cleaning legacy symlink here."
-		);
-
-		tokio::fs::remove_file(&old_path).await?;
-	}
-
 	Ok(())
 }
 

src/service/media/data.rs

@@ -4,7 +4,7 @@ use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result};
 use database::{Database, Map};
 use ruma::api::client::error::ErrorKind;
 
-use super::preview::UrlPreviewData;
+use crate::media::UrlPreviewData;
 
 pub(crate) struct Data {
 	mediaid_file: Arc<Map>,

src/service/media/mod.rs

@@ -1,5 +1,4 @@
 mod data;
-mod preview;
 mod tests;
 mod thumbnail;
 
@@ -7,15 +6,16 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime};
 
 use async_trait::async_trait;
 use base64::{engine::general_purpose, Engine as _};
-use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server};
+use conduit::{debug, debug_error, err, error, utils, utils::MutexMap, Err, Result, Server};
 use data::{Data, Metadata};
 use ruma::{OwnedMxcUri, OwnedUserId};
+use serde::Serialize;
 use tokio::{
 	fs,
 	io::{AsyncReadExt, AsyncWriteExt, BufReader},
 };
 
-use crate::{client, globals, Dep};
+use crate::{globals, Dep};
 
 #[derive(Debug)]
 pub struct FileMeta {
@@ -24,32 +24,43 @@ pub struct FileMeta {
 	pub content_disposition: Option<String>,
 }
 
+#[derive(Serialize, Default)]
+pub struct UrlPreviewData {
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))]
+	pub title: Option<String>,
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))]
+	pub description: Option<String>,
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))]
+	pub image: Option<String>,
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))]
+	pub image_size: Option<usize>,
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))]
+	pub image_width: Option<u32>,
+	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))]
+	pub image_height: Option<u32>,
+}
+
 pub struct Service {
-	url_preview_mutex: MutexMap<String, ()>,
-	pub(crate) db: Data,
 	services: Services,
+	pub(crate) db: Data,
+	pub url_preview_mutex: MutexMap<String, ()>,
 }
 
 struct Services {
 	server: Arc<Server>,
-	client: Dep<client::Service>,
 	globals: Dep<globals::Service>,
 }
 
-/// generated MXC ID (`media-id`) length
-pub const MXC_LENGTH: usize = 32;
-
 #[async_trait]
 impl crate::Service for Service {
 	fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
 		Ok(Arc::new(Self {
-			url_preview_mutex: MutexMap::new(),
-			db: Data::new(args.db),
 			services: Services {
 				server: args.server.clone(),
-				client: args.depend::<client::Service>("client"),
 				globals: args.depend::<globals::Service>("globals"),
 			},
+			db: Data::new(args.db),
+			url_preview_mutex: MutexMap::new(),
 		}))
 	}
 
@@ -88,15 +99,10 @@ impl Service {
 	pub async fn delete(&self, mxc: &str) -> Result<()> {
 		if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) {
 			for key in keys {
-				trace!(?mxc, ?key, "Deleting from filesystem");
+				self.remove_media_file(&key).await?;
-				if let Err(e) = self.remove_media_file(&key).await {
-					error!(?mxc, ?key, "Failed to remove media file: {e}");
-				}
 
-				trace!(?mxc, ?key, "Deleting from database");
+				debug!("Deleting MXC {mxc} from database");
-				if let Err(e) = self.db.delete_file_mxc(mxc) {
+				self.db.delete_file_mxc(mxc)?;
-					error!(?mxc, ?key, "Failed to remove media from database: {e}");
-				}
 			}
 
 			Ok(())
@@ -218,6 +224,22 @@ impl Service {
 		Ok(deletion_count)
 	}
 
+	pub async fn get_url_preview(&self, url: &str) -> Option<UrlPreviewData> { self.db.get_url_preview(url) }
+
+	/// TODO: use this?
+	#[allow(dead_code)]
+	pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
+		// TODO: also remove the downloaded image
+		self.db.remove_url_preview(url)
+	}
+
+	pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
+		let now = SystemTime::now()
+			.duration_since(SystemTime::UNIX_EPOCH)
+			.expect("valid system time");
+		self.db.set_url_preview(url, data, now)
+	}
+
 	pub async fn create_media_dir(&self) -> Result<()> {
 		let dir = self.get_media_dir();
 		Ok(fs::create_dir_all(dir).await?)

src/service/media/preview.rs

@@ -1,274 +0,0 @@
-use std::{io::Cursor, time::SystemTime};
-
-use conduit::{debug, utils, warn, Err, Result};
-use conduit_core::implement;
-use image::ImageReader as ImgReader;
-use ipaddress::IPAddress;
-use serde::Serialize;
-use url::Url;
-use webpage::HTML;
-
-use super::{Service, MXC_LENGTH};
-
-#[derive(Serialize, Default)]
-pub struct UrlPreviewData {
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))]
-	pub title: Option<String>,
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))]
-	pub description: Option<String>,
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))]
-	pub image: Option<String>,
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))]
-	pub image_size: Option<usize>,
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))]
-	pub image_width: Option<u32>,
-	#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))]
-	pub image_height: Option<u32>,
-}
-
-#[implement(Service)]
-pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
-	// TODO: also remove the downloaded image
-	self.db.remove_url_preview(url)
-}
-
-#[implement(Service)]
-pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
-	let now = SystemTime::now()
-		.duration_since(SystemTime::UNIX_EPOCH)
-		.expect("valid system time");
-	self.db.set_url_preview(url, data, now)
-}
-
-#[implement(Service)]
-pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
-	let client = &self.services.client.url_preview;
-	let image = client.get(url).send().await?.bytes().await?;
-	let mxc = format!(
-		"mxc://{}/{}",
-		self.services.globals.server_name(),
-		utils::random_string(MXC_LENGTH)
-	);
-
-	self.create(None, &mxc, None, None, &image).await?;
-
-	let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() {
-		Err(_) => (None, None),
-		Ok(reader) => match reader.into_dimensions() {
-			Err(_) => (None, None),
-			Ok((width, height)) => (Some(width), Some(height)),
-		},
-	};
-
-	Ok(UrlPreviewData {
-		image: Some(mxc),
-		image_size: Some(image.len()),
-		image_width: width,
-		image_height: height,
-		..Default::default()
-	})
-}
-
-#[implement(Service)]
-pub async fn get_url_preview(&self, url: &str) -> Result<UrlPreviewData> {
-	if let Some(preview) = self.db.get_url_preview(url) {
-		return Ok(preview);
-	}
-
-	// ensure that only one request is made per URL
-	let _request_lock = self.url_preview_mutex.lock(url).await;
-
-	match self.db.get_url_preview(url) {
-		Some(preview) => Ok(preview),
-		None => self.request_url_preview(url).await,
-	}
-}
-
-#[implement(Service)]
-async fn request_url_preview(&self, url: &str) -> Result<UrlPreviewData> {
-	if let Ok(ip) = IPAddress::parse(url) {
-		if !self.services.globals.valid_cidr_range(&ip) {
-			return Err!(BadServerResponse("Requesting from this address is forbidden"));
-		}
-	}
-
-	let client = &self.services.client.url_preview;
-	let response = client.head(url).send().await?;
-
-	if let Some(remote_addr) = response.remote_addr() {
-		if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
-			if !self.services.globals.valid_cidr_range(&ip) {
-				return Err!(BadServerResponse("Requesting from this address is forbidden"));
-			}
-		}
-	}
-
-	let Some(content_type) = response
-		.headers()
-		.get(reqwest::header::CONTENT_TYPE)
-		.and_then(|x| x.to_str().ok())
-	else {
-		return Err!(Request(Unknown("Unknown Content-Type")));
-	};
-	let data = match content_type {
-		html if html.starts_with("text/html") => self.download_html(url).await?,
-		img if img.starts_with("image/") => self.download_image(url).await?,
-		_ => return Err!(Request(Unknown("Unsupported Content-Type"))),
-	};
-
-	self.set_url_preview(url, &data).await?;
-
-	Ok(data)
-}
-
-#[implement(Service)]
-async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
-	let client = &self.services.client.url_preview;
-	let mut response = client.get(url).send().await?;
-
-	let mut bytes: Vec<u8> = Vec::new();
-	while let Some(chunk) = response.chunk().await? {
-		bytes.extend_from_slice(&chunk);
-		if bytes.len() > self.services.globals.url_preview_max_spider_size() {
-			debug!(
-				"Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \
-				 response body and assuming our necessary data is in this range.",
-				url,
-				self.services.globals.url_preview_max_spider_size()
-			);
-			break;
-		}
-	}
-	let body = String::from_utf8_lossy(&bytes);
-	let Ok(html) = HTML::from_string(body.to_string(), Some(url.to_owned())) else {
-		return Err!(Request(Unknown("Failed to parse HTML")));
-	};
-
-	let mut data = match html.opengraph.images.first() {
-		None => UrlPreviewData::default(),
-		Some(obj) => self.download_image(&obj.url).await?,
-	};
-
-	let props = html.opengraph.properties;
-
-	/* use OpenGraph title/description, but fall back to HTML if not available */
-	data.title = props.get("title").cloned().or(html.title);
-	data.description = props.get("description").cloned().or(html.description);
-
-	Ok(data)
-}
-
-#[implement(Service)]
-pub fn url_preview_allowed(&self, url_str: &str) -> bool {
-	let url: Url = match Url::parse(url_str) {
-		Ok(u) => u,
-		Err(e) => {
-			warn!("Failed to parse URL from a str: {}", e);
-			return false;
-		},
-	};
-
-	if ["http", "https"]
-		.iter()
-		.all(|&scheme| scheme != url.scheme().to_lowercase())
-	{
-		debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
-		return false;
-	}
-
-	let host = match url.host_str() {
-		None => {
-			debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
-			return false;
-		},
-		Some(h) => h.to_owned(),
-	};
-
-	let allowlist_domain_contains = self
-		.services
-		.globals
-		.url_preview_domain_contains_allowlist();
-	let allowlist_domain_explicit = self
-		.services
-		.globals
-		.url_preview_domain_explicit_allowlist();
-	let denylist_domain_explicit = self.services.globals.url_preview_domain_explicit_denylist();
-	let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist();
-
-	if allowlist_domain_contains.contains(&"*".to_owned())
-		|| allowlist_domain_explicit.contains(&"*".to_owned())
-		|| allowlist_url_contains.contains(&"*".to_owned())
-	{
-		debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url);
-		return true;
-	}
-
-	if !host.is_empty() {
-		if denylist_domain_explicit.contains(&host) {
-			debug!(
-				"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
-				&host
-			);
-			return false;
-		}
-
-		if allowlist_domain_explicit.contains(&host) {
-			debug!("Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", &host);
-			return true;
-		}
-
-		if allowlist_domain_contains
-			.iter()
-			.any(|domain_s| domain_s.contains(&host.clone()))
-		{
-			debug!("Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", &host);
-			return true;
-		}
-
-		if allowlist_url_contains
-			.iter()
-			.any(|url_s| url.to_string().contains(&url_s.to_string()))
-		{
-			debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host);
-			return true;
-		}
-
-		// check root domain if available and if user has root domain checks
-		if self.services.globals.url_preview_check_root_domain() {
-			debug!("Checking root domain");
-			match host.split_once('.') {
-				None => return false,
-				Some((_, root_domain)) => {
-					if denylist_domain_explicit.contains(&root_domain.to_owned()) {
-						debug!(
-							"Root domain {} is not allowed by url_preview_domain_explicit_denylist (check 1/3)",
-							&root_domain
-						);
-						return true;
-					}
-
-					if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
-						debug!(
-							"Root domain {} is allowed by url_preview_domain_explicit_allowlist (check 2/3)",
-							&root_domain
-						);
-						return true;
-					}
-
-					if allowlist_domain_contains
-						.iter()
-						.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
-					{
-						debug!(
-							"Root domain {} is allowed by url_preview_domain_contains_allowlist (check 3/3)",
-							&root_domain
-						);
-						return true;
-					}
-				},
-			}
-		}
-	}
-
-	false
-}

src/service/mod.rs

@@ -18,7 +18,6 @@ pub mod pusher;
 pub mod resolver;
 pub mod rooms;
 pub mod sending;
-pub mod server_keys;
 pub mod transaction_ids;
 pub mod uiaa;
 pub mod updates;

src/service/presence/data.rs

@@ -61,6 +61,22 @@ impl Data {
 			Some(ref presence) => presence.1.content.presence != *presence_state,
 		};
 
+		let status_msg_changed = match last_presence {
+			None => true,
+			Some(ref last_presence) => {
+				let old_msg = last_presence
+					.1
+					.content
+					.status_msg
+					.clone()
+					.unwrap_or_default();
+
+				let new_msg = status_msg.clone().unwrap_or_default();
+
+				new_msg != old_msg
+			},
+		};
+
 		let now = utils::millis_since_unix_epoch();
 		let last_last_active_ts = match last_presence {
 			None => 0,
@@ -72,10 +88,10 @@ impl Data {
 			Some(last_active_ago) => now.saturating_sub(last_active_ago.into()),
 		};
 
-		// tighten for state flicker?
+		// TODO: tighten for state flicker?
-		if !state_changed && last_active_ts <= last_last_active_ts {
+		if !status_msg_changed && !state_changed && last_active_ts < last_last_active_ts {
 			debug_warn!(
-				"presence spam {:?} last_active_ts:{:?} <= {:?}",
+				"presence spam {:?} last_active_ts:{:?} < {:?}",
 				user_id,
 				last_active_ts,
 				last_last_active_ts

src/service/pusher/mod.rs

@@ -3,7 +3,7 @@ mod data;
 use std::{fmt::Debug, mem, sync::Arc};
 
 use bytes::BytesMut;
-use conduit::{debug_error, err, trace, utils::string_from_bytes, warn, Err, PduEvent, Result};
+use conduit::{debug_info, info, trace, utils::string_from_bytes, warn, Error, PduEvent, Result};
 use ipaddress::IPAddress;
 use ruma::{
 	api::{
@@ -84,9 +84,8 @@ impl Service {
 		let http_request = request
 			.try_into_http_request::<BytesMut>(&dest, SendAccessToken::IfRequired(""), &VERSIONS)
 			.map_err(|e| {
-				err!(BadServerResponse(warn!(
+				warn!("Failed to find destination {dest} for push gateway: {e}");
-					"Failed to find destination {dest} for push gateway: {e}"
+				Error::BadServerResponse("Invalid push gateway destination")
-				)))
 			})?
 			.map(BytesMut::freeze);
 
@@ -96,7 +95,7 @@ impl Service {
 			trace!("Checking request URL for IP");
 			if let Ok(ip) = IPAddress::parse(url_host) {
 				if !self.services.globals.valid_cidr_range(&ip) {
-					return Err!(BadServerResponse("Not allowed to send requests to this IP"));
+					return Err(Error::BadServerResponse("Not allowed to send requests to this IP"));
 				}
 			}
 		}
@@ -111,7 +110,7 @@ impl Service {
 				if let Some(remote_addr) = response.remote_addr() {
 					if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
 						if !self.services.globals.valid_cidr_range(&ip) {
-							return Err!(BadServerResponse("Not allowed to send requests to this IP"));
+							return Err(Error::BadServerResponse("Not allowed to send requests to this IP"));
 						}
 					}
 				}
@@ -130,10 +129,10 @@ impl Service {
 				let body = response.bytes().await?; // TODO: handle timeout
 
 				if !status.is_success() {
-					debug_error!("Push gateway response body: {:?}", string_from_bytes(&body));
+					info!("Push gateway {dest} returned unsuccessful HTTP response ({status})");
-					return Err!(BadServerResponse(error!(
+					debug_info!("Push gateway response body: {:?}", string_from_bytes(&body));
-						"Push gateway {dest} returned unsuccessful HTTP response: {status}"
+
-					)));
+					return Err(Error::BadServerResponse("Push gateway returned unsuccessful response"));
 				}
 
 				let response = T::IncomingResponse::try_from_http_response(
@@ -141,11 +140,13 @@ impl Service {
 						.body(body)
 						.expect("reqwest body is valid http body"),
 				);
-				response
+				response.map_err(|e| {
-					.map_err(|e| err!(BadServerResponse(error!("Push gateway {dest} returned invalid response: {e}"))))
+					warn!("Push gateway {dest} returned invalid response bytes: {e}");
+					Error::BadServerResponse("Push gateway returned bad/invalid response")
+				})
 			},
 			Err(e) => {
-				debug_error!("Could not send request to pusher {dest}: {e}");
+				warn!("Could not send request to pusher {dest}: {e}");
 				Err(e.into())
 			},
 		}
@@ -164,7 +165,7 @@ impl Service {
 			.room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")?
 			.map(|ev| {
 				serde_json::from_str(ev.content.get())
-					.map_err(|e| err!(Database("invalid m.room.power_levels event: {e:?}")))
+					.map_err(|_| Error::bad_database("invalid m.room.power_levels event"))
 			})
 			.transpose()?
 			.unwrap_or_default();
@@ -180,8 +181,8 @@ impl Service {
 			};
 
 			if notify.is_some() {
-				return Err!(Database(
+				return Err(Error::bad_database(
-					r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#
+					r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#,
 				));
 			}
 

src/service/resolver/actual.rs

@@ -4,7 +4,7 @@ use std::{
 	sync::Arc,
 };
 
-use conduit::{debug, debug_error, debug_info, debug_warn, err, trace, Err, Result};
+use conduit::{debug, debug_error, debug_info, debug_warn, trace, Err, Error, Result};
 use hickory_resolver::{error::ResolveError, lookup::SrvLookup};
 use ipaddress::IPAddress;
 use ruma::ServerName;
@@ -329,8 +329,10 @@ impl super::Service {
 			dest.is_ip_literal() || !IPAddress::is_valid(dest.host()),
 			"Destination is not an IP literal."
 		);
-		let ip = IPAddress::parse(dest.host())
+		let ip = IPAddress::parse(dest.host()).map_err(|e| {
-			.map_err(|e| err!(BadServerResponse(debug_error!("Failed to parse IP literal from string: {e}"))))?;
+			debug_error!("Failed to parse IP literal from string: {}", e);
+			Error::BadServerResponse("Invalid IP address")
+		})?;
 
 		self.validate_ip(&ip)?;
 
@@ -339,7 +341,7 @@ impl super::Service {
 
 	pub(crate) fn validate_ip(&self, ip: &IPAddress) -> Result<()> {
 		if !self.services.globals.valid_cidr_range(ip) {
-			return Err!(BadServerResponse("Not allowed to send requests to this IP"));
+			return Err(Error::BadServerResponse("Not allowed to send requests to this IP"));
 		}
 
 		Ok(())

src/service/rooms/event_handler/mod.rs

@@ -1,4 +1,5 @@
 mod parse_incoming_pdu;
+mod signing_keys;
 
 use std::{
 	collections::{hash_map, BTreeMap, HashMap, HashSet},
@@ -34,7 +35,7 @@ use ruma::{
 use tokio::sync::RwLock;
 
 use super::state_compressor::CompressedStateEvent;
-use crate::{globals, rooms, sending, server_keys, Dep};
+use crate::{globals, rooms, sending, Dep};
 
 pub struct Service {
 	services: Services,
@@ -49,7 +50,6 @@ struct Services {
 	metadata: Dep<rooms::metadata::Service>,
 	outlier: Dep<rooms::outlier::Service>,
 	pdu_metadata: Dep<rooms::pdu_metadata::Service>,
-	server_keys: Dep<server_keys::Service>,
 	short: Dep<rooms::short::Service>,
 	state: Dep<rooms::state::Service>,
 	state_accessor: Dep<rooms::state_accessor::Service>,
@@ -77,7 +77,6 @@ impl crate::Service for Service {
 				auth_chain: args.depend::<rooms::auth_chain::Service>("rooms::auth_chain"),
 				metadata: args.depend::<rooms::metadata::Service>("rooms::metadata"),
 				outlier: args.depend::<rooms::outlier::Service>("rooms::outlier"),
-				server_keys: args.depend::<server_keys::Service>("server_keys"),
 				pdu_metadata: args.depend::<rooms::pdu_metadata::Service>("rooms::pdu_metadata"),
 				short: args.depend::<rooms::short::Service>("rooms::short"),
 				state: args.depend::<rooms::state::Service>("rooms::state"),
@@ -1188,9 +1187,7 @@ impl Service {
 
 			// We go through all the signatures we see on the PDUs and their unresolved
 			// dependencies and fetch the corresponding signing keys
-			self.services
+			self.fetch_required_signing_keys(
-				.server_keys
-				.fetch_required_signing_keys(
 				events_with_auth_events
 					.iter()
 					.flat_map(|(_id, _local_pdu, events)| events)

src/service/rooms/event_handler/parse_incoming_pdu.rs

@@ -1,18 +1,18 @@
-use conduit::{debug_warn, err, pdu::gen_event_id_canonical_json, Err, Result};
+use conduit::{pdu::gen_event_id_canonical_json, warn, Err, Error, Result};
-use ruma::{CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId};
+use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId};
 use serde_json::value::RawValue as RawJsonValue;
 
 impl super::Service {
 	pub fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> {
 		let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
-			debug_warn!("Error parsing incoming event {pdu:#?}");
+			warn!("Error parsing incoming event {pdu:?}: {e:?}");
-			err!(BadServerResponse("Error parsing incoming event {e:?}"))
+			Error::BadServerResponse("Invalid PDU in server response")
 		})?;
 
 		let room_id: OwnedRoomId = value
 			.get("room_id")
 			.and_then(|id| RoomId::parse(id.as_str()?).ok())
-			.ok_or(err!(Request(InvalidParam("Invalid room id in pdu"))))?;
+			.ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?;
 
 		let Ok(room_version_id) = self.services.state.get_room_version(&room_id) else {
 			return Err!("Server is not in room {room_id}");

src/service/rooms/event_handler/signing_keys.rs

@@ -1,10 +1,9 @@
 use std::{
 	collections::{BTreeMap, HashMap, HashSet},
-	sync::Arc,
 	time::{Duration, SystemTime},
 };
 
-use conduit::{debug, debug_error, debug_warn, err, error, info, trace, warn, Err, Result};
+use conduit::{debug, error, info, trace, warn, Error, Result};
 use futures_util::{stream::FuturesUnordered, StreamExt};
 use ruma::{
 	api::federation::{
@@ -22,31 +21,7 @@ use ruma::{
 use serde_json::value::RawValue as RawJsonValue;
 use tokio::sync::{RwLock, RwLockWriteGuard};
 
-use crate::{globals, sending, Dep};
+impl super::Service {
-
-pub struct Service {
-	services: Services,
-}
-
-struct Services {
-	globals: Dep<globals::Service>,
-	sending: Dep<sending::Service>,
-}
-
-impl crate::Service for Service {
-	fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
-		Ok(Arc::new(Self {
-			services: Services {
-				globals: args.depend::<globals::Service>("globals"),
-				sending: args.depend::<sending::Service>("sending"),
-			},
-		}))
-	}
-
-	fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
-}
-
-impl Service {
 	pub async fn fetch_required_signing_keys<'a, E>(
 		&'a self, events: E, pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
 	) -> Result<()>
@@ -57,13 +32,13 @@ impl Service {
 		for event in events {
 			for (signature_server, signature) in event
 				.get("signatures")
-				.ok_or(err!(BadServerResponse("No signatures in server response pdu.")))?
+				.ok_or(Error::BadServerResponse("No signatures in server response pdu."))?
 				.as_object()
-				.ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))?
+				.ok_or(Error::BadServerResponse("Invalid signatures object in server response pdu."))?
 			{
-				let signature_object = signature.as_object().ok_or(err!(BadServerResponse(
+				let signature_object = signature.as_object().ok_or(Error::BadServerResponse(
 					"Invalid signatures content object in server response pdu.",
-				)))?;
+				))?;
 
 				for signature_id in signature_object.keys() {
 					server_key_ids
@@ -94,12 +69,10 @@ impl Service {
 			.map(|(signature_server, signature_ids)| async {
 				let fetch_res = self
 					.fetch_signing_keys_for_server(
-						signature_server.as_str().try_into().map_err(|e| {
+						signature_server.as_str().try_into().map_err(|_| {
 							(
 								signature_server.clone(),
-								err!(BadServerResponse(
+								Error::BadServerResponse("Invalid servername in signatures of server response pdu."),
-									"Invalid servername in signatures of server response pdu: {e:?}"
-								)),
 							)
 						})?,
 						signature_ids.into_iter().collect(), // HashSet to Vec
@@ -109,9 +82,7 @@ impl Service {
 				match fetch_res {
 					Ok(keys) => Ok((signature_server, keys)),
 					Err(e) => {
-						debug_error!(
+						warn!("Signature verification failed: Could not fetch signing key for {signature_server}: {e}",);
-							"Signature verification failed: Could not fetch signing key for {signature_server}: {e}",
-						);
 						Err((signature_server, e))
 					},
 				}
@@ -127,7 +98,7 @@ impl Service {
 						.insert(signature_server.clone(), keys);
 				},
 				Err((signature_server, e)) => {
-					debug_warn!("Failed to fetch keys for {signature_server}: {e:?}");
+					warn!("Failed to fetch keys for {}: {:?}", signature_server, e);
 				},
 			}
 		}
@@ -145,37 +116,35 @@ impl Service {
 		pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap<String, BTreeMap<String, Base64>>>,
 	) -> Result<()> {
 		let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
-			debug_error!("Invalid PDU in server response: {pdu:#?}");
+			error!("Invalid PDU in server response: {:?}: {:?}", pdu, e);
-			err!(BadServerResponse(error!("Invalid PDU in server response: {e:?}")))
+			Error::BadServerResponse("Invalid PDU in server response")
 		})?;
 
 		let signatures = value
 			.get("signatures")
-			.ok_or(err!(BadServerResponse("No signatures in server response pdu.")))?
+			.ok_or(Error::BadServerResponse("No signatures in server response pdu."))?
 			.as_object()
-			.ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))?;
+			.ok_or(Error::BadServerResponse("Invalid signatures object in server response pdu."))?;
 
 		for (signature_server, signature) in signatures {
-			let signature_object = signature.as_object().ok_or(err!(BadServerResponse(
+			let signature_object = signature.as_object().ok_or(Error::BadServerResponse(
 				"Invalid signatures content object in server response pdu.",
-			)))?;
+			))?;
 
 			let signature_ids = signature_object.keys().cloned().collect::<Vec<_>>();
 
 			let contains_all_ids =
 				|keys: &BTreeMap<String, Base64>| signature_ids.iter().all(|id| keys.contains_key(id));
 
-			let origin = <&ServerName>::try_from(signature_server.as_str()).map_err(|e| {
+			let origin = <&ServerName>::try_from(signature_server.as_str())
-				err!(BadServerResponse(
+				.map_err(|_| Error::BadServerResponse("Invalid servername in signatures of server response pdu."))?;
-					"Invalid servername in signatures of server response pdu: {e:?}"
-				))
-			})?;
 
 			if servers.contains_key(origin) || pub_key_map.contains_key(origin.as_str()) {
 				continue;
 			}
 
-			debug!("Loading signing keys for {origin}");
+			debug!("Loading signing keys for {}", origin);
+
 			let result: BTreeMap<_, _> = self
 				.services
 				.globals
@@ -185,7 +154,7 @@ impl Service {
 				.collect();
 
 			if !contains_all_ids(&result) {
-				debug_warn!("Signing key not loaded for {origin}");
+				debug!("Signing key not loaded for {}", origin);
 				servers.insert(origin.to_owned(), BTreeMap::new());
 			}
 
@@ -202,7 +171,7 @@ impl Service {
 		pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
 	) -> Result<()> {
 		for server in self.services.globals.trusted_servers() {
-			debug!("Asking batch signing keys from trusted server {server}");
+			debug!("Asking batch signing keys from trusted server {}", server);
 			match self
 				.services
 				.sending
@@ -215,16 +184,14 @@ impl Service {
 				.await
 			{
 				Ok(keys) => {
-					debug!("Got signing keys: {keys:?}");
+					debug!("Got signing keys: {:?}", keys);
 					let mut pkm = pub_key_map.write().await;
 					for k in keys.server_keys {
 						let k = match k.deserialize() {
 							Ok(key) => key,
 							Err(e) => {
-								warn!(
+								warn!("Received error {e} while fetching keys from trusted server {server}");
-									"Received error {e} while fetching keys from trusted server {server}: {:#?}",
+								warn!("{}", k.into_json());
-									k.into_json()
-								);
 								continue;
 							},
 						};
@@ -244,10 +211,13 @@ impl Service {
 						pkm.insert(k.server_name.to_string(), result);
 					}
 				},
-				Err(e) => error!(
+				Err(e) => {
+					warn!(
 						"Failed sending batched key request to trusted key server {server} for the remote servers \
-					 {servers:?}: {e}"
+						 {:?}: {e}",
-				),
+						servers
+					);
+				},
 			}
 		}
 
@@ -483,6 +453,7 @@ impl Service {
 			}
 		} else {
 			info!("query_trusted_key_servers_first is set to false, querying {origin} first");
+
 			debug!("Asking {origin} for their signing keys over federation");
 			if let Some(server_key) = self
 				.services
@@ -540,7 +511,7 @@ impl Service {
 							.filter_map(|e| e.deserialize().ok())
 							.collect::<Vec<_>>()
 					}) {
-					debug!("Got signing keys: {server_keys:?}");
+					debug!("Got signing keys: {:?}", server_keys);
 					for k in server_keys {
 						self.services
 							.globals
@@ -565,6 +536,7 @@ impl Service {
 			}
 		}
 
-		Err!(BadServerResponse(warn!("Failed to find public key for server {origin:?}")))
+		warn!("Failed to find public key for server: {origin}");
+		Err(Error::BadServerResponse("Failed to find public key for server"))
 	}
 }

src/service/rooms/timeline/mod.rs

@@ -41,7 +41,7 @@ use tokio::sync::RwLock;
 use self::data::Data;
 use crate::{
 	account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms,
-	rooms::state_compressor::CompressedStateEvent, sending, server_keys, Dep,
+	rooms::state_compressor::CompressedStateEvent, sending, Dep,
 };
 
 // Update Relationships
@@ -86,7 +86,6 @@ struct Services {
 	pdu_metadata: Dep<rooms::pdu_metadata::Service>,
 	read_receipt: Dep<rooms::read_receipt::Service>,
 	sending: Dep<sending::Service>,
-	server_keys: Dep<server_keys::Service>,
 	user: Dep<rooms::user::Service>,
 	pusher: Dep<pusher::Service>,
 	threads: Dep<rooms::threads::Service>,
@@ -115,7 +114,6 @@ impl crate::Service for Service {
 				pdu_metadata: args.depend::<rooms::pdu_metadata::Service>("rooms::pdu_metadata"),
 				read_receipt: args.depend::<rooms::read_receipt::Service>("rooms::read_receipt"),
 				sending: args.depend::<sending::Service>("sending"),
-				server_keys: args.depend::<server_keys::Service>("server_keys"),
 				user: args.depend::<rooms::user::Service>("rooms::user"),
 				pusher: args.depend::<pusher::Service>("pusher"),
 				threads: args.depend::<rooms::threads::Service>("rooms::threads"),
@@ -1183,7 +1181,7 @@ impl Service {
 		}
 
 		self.services
-			.server_keys
+			.event_handler
 			.fetch_required_signing_keys([&value], pub_key_map)
 			.await?;
 

src/service/sending/appservice.rs

@@ -1,7 +1,7 @@
 use std::{fmt::Debug, mem};
 
 use bytes::BytesMut;
-use conduit::{debug_error, err, trace, utils, warn, Err, Result};
+use conduit::{debug_error, trace, utils, warn, Error, Result};
 use reqwest::Client;
 use ruma::api::{appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken};
 
@@ -26,7 +26,10 @@ where
 	let hs_token = registration.hs_token.as_str();
 	let mut http_request = request
 		.try_into_http_request::<BytesMut>(&dest, SendAccessToken::IfRequired(hs_token), &VERSIONS)
-		.map_err(|e| err!(BadServerResponse(warn!("Failed to find destination {dest}: {e}"))))?
+		.map_err(|e| {
+			warn!("Failed to find destination {dest}: {e}");
+			Error::BadServerResponse("Invalid appservice destination")
+		})?
 		.map(BytesMut::freeze);
 
 	let mut parts = http_request.uri().clone().into_parts();
@@ -66,11 +69,13 @@ where
 	let body = response.bytes().await?; // TODO: handle timeout
 
 	if !status.is_success() {
-		debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body));
+		warn!(
-		return Err!(BadServerResponse(error!(
 			"Appservice \"{}\" returned unsuccessful HTTP response {status} at {dest}",
 			registration.id
-		)));
+		);
+		debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body));
+
+		return Err(Error::BadServerResponse("Appservice returned unsuccessful HTTP response"));
 	}
 
 	let response = T::IncomingResponse::try_from_http_response(
@@ -80,9 +85,7 @@ where
 	);
 
 	response.map(Some).map_err(|e| {
-		err!(BadServerResponse(error!(
+		warn!("Appservice \"{}\" returned invalid response bytes {dest}: {e}", registration.id);
-			"Appservice \"{}\" returned invalid response bytes {dest}: {e}",
+		Error::BadServerResponse("Appservice returned bad/invalid response")
-			registration.id
-		)))
 	})
 }

src/service/sending/resolve.rs

@@ -0,0 +1,343 @@
+use std::{
+	fmt::Debug,
+	net::{IpAddr, SocketAddr},
+};
+
+use conduit::{debug, debug_error, debug_info, debug_warn, trace, Err, Error, Result};
+use hickory_resolver::{error::ResolveError, lookup::SrvLookup};
+use ipaddress::IPAddress;
+use ruma::ServerName;
+
+use crate::{
+	resolver::{add_port_to_hostname, get_ip_with_port, CachedDest, CachedOverride, FedDest},
+	services,
+};
+
+#[derive(Clone, Debug)]
+pub(crate) struct ActualDest {
+	pub(crate) dest: FedDest,
+	pub(crate) host: String,
+	pub(crate) string: String,
+	pub(crate) cached: bool,
+}
+
+#[tracing::instrument(skip_all, name = "resolve")]
+pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result<ActualDest> {
+	let cached;
+	let cached_result = services().resolver.get_cached_destination(server_name);
+
+	let CachedDest {
+		dest,
+		host,
+		..
+	} = if let Some(result) = cached_result {
+		cached = true;
+		result
+	} else {
+		cached = false;
+		validate_dest(server_name)?;
+		resolve_actual_dest(server_name, true).await?
+	};
+
+	let string = dest.clone().into_https_string();
+	Ok(ActualDest {
+		dest,
+		host,
+		string,
+		cached,
+	})
+}
+
+/// Returns: `actual_destination`, host header
+/// Implemented according to the specification at <https://matrix.org/docs/spec/server_server/r0.1.4#resolving-server-names>
+/// Numbers in comments below refer to bullet points in linked section of
+/// specification
+#[tracing::instrument(skip_all, name = "actual")]
+pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result<CachedDest> {
+	trace!("Finding actual destination for {dest}");
+	let mut host = dest.as_str().to_owned();
+	let actual_dest = match get_ip_with_port(dest.as_str()) {
+		Some(host_port) => actual_dest_1(host_port)?,
+		None => {
+			if let Some(pos) = dest.as_str().find(':') {
+				actual_dest_2(dest, cache, pos).await?
+			} else if let Some(delegated) = request_well_known(dest.as_str()).await? {
+				actual_dest_3(&mut host, cache, delegated).await?
+			} else if let Some(overrider) = query_srv_record(dest.as_str()).await? {
+				actual_dest_4(&host, cache, overrider).await?
+			} else {
+				actual_dest_5(dest, cache).await?
+			}
+		},
+	};
+
+	// Can't use get_ip_with_port here because we don't want to add a port
+	// to an IP address if it wasn't specified
+	let host = if let Ok(addr) = host.parse::<SocketAddr>() {
+		FedDest::Literal(addr)
+	} else if let Ok(addr) = host.parse::<IpAddr>() {
+		FedDest::Named(addr.to_string(), ":8448".to_owned())
+	} else if let Some(pos) = host.find(':') {
+		let (host, port) = host.split_at(pos);
+		FedDest::Named(host.to_owned(), port.to_owned())
+	} else {
+		FedDest::Named(host, ":8448".to_owned())
+	};
+
+	debug!("Actual destination: {actual_dest:?} hostname: {host:?}");
+	Ok(CachedDest {
+		dest: actual_dest,
+		host: host.into_uri_string(),
+		expire: CachedDest::default_expire(),
+	})
+}
+
+fn actual_dest_1(host_port: FedDest) -> Result<FedDest> {
+	debug!("1: IP literal with provided or default port");
+	Ok(host_port)
+}
+
+async fn actual_dest_2(dest: &ServerName, cache: bool, pos: usize) -> Result<FedDest> {
+	debug!("2: Hostname with included port");
+	let (host, port) = dest.as_str().split_at(pos);
+	conditional_query_and_cache_override(host, host, port.parse::<u16>().unwrap_or(8448), cache).await?;
+	Ok(FedDest::Named(host.to_owned(), port.to_owned()))
+}
+
+async fn actual_dest_3(host: &mut String, cache: bool, delegated: String) -> Result<FedDest> {
+	debug!("3: A .well-known file is available");
+	*host = add_port_to_hostname(&delegated).into_uri_string();
+	match get_ip_with_port(&delegated) {
+		Some(host_and_port) => actual_dest_3_1(host_and_port),
+		None => {
+			if let Some(pos) = delegated.find(':') {
+				actual_dest_3_2(cache, delegated, pos).await
+			} else {
+				trace!("Delegated hostname has no port in this branch");
+				if let Some(overrider) = query_srv_record(&delegated).await? {
+					actual_dest_3_3(cache, delegated, overrider).await
+				} else {
+					actual_dest_3_4(cache, delegated).await
+				}
+			}
+		},
+	}
+}
+
+fn actual_dest_3_1(host_and_port: FedDest) -> Result<FedDest> {
+	debug!("3.1: IP literal in .well-known file");
+	Ok(host_and_port)
+}
+
+async fn actual_dest_3_2(cache: bool, delegated: String, pos: usize) -> Result<FedDest> {
+	debug!("3.2: Hostname with port in .well-known file");
+	let (host, port) = delegated.split_at(pos);
+	conditional_query_and_cache_override(host, host, port.parse::<u16>().unwrap_or(8448), cache).await?;
+	Ok(FedDest::Named(host.to_owned(), port.to_owned()))
+}
+
+async fn actual_dest_3_3(cache: bool, delegated: String, overrider: FedDest) -> Result<FedDest> {
+	debug!("3.3: SRV lookup successful");
+	let force_port = overrider.port();
+	conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?;
+	if let Some(port) = force_port {
+		Ok(FedDest::Named(delegated, format!(":{port}")))
+	} else {
+		Ok(add_port_to_hostname(&delegated))
+	}
+}
+
+async fn actual_dest_3_4(cache: bool, delegated: String) -> Result<FedDest> {
+	debug!("3.4: No SRV records, just use the hostname from .well-known");
+	conditional_query_and_cache_override(&delegated, &delegated, 8448, cache).await?;
+	Ok(add_port_to_hostname(&delegated))
+}
+
+async fn actual_dest_4(host: &str, cache: bool, overrider: FedDest) -> Result<FedDest> {
+	debug!("4: No .well-known; SRV record found");
+	let force_port = overrider.port();
+	conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?;
+	if let Some(port) = force_port {
+		Ok(FedDest::Named(host.to_owned(), format!(":{port}")))
+	} else {
+		Ok(add_port_to_hostname(host))
+	}
+}
+
+async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result<FedDest> {
+	debug!("5: No SRV record found");
+	conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache).await?;
+	Ok(add_port_to_hostname(dest.as_str()))
+}
+
+#[tracing::instrument(skip_all, name = "well-known")]
+async fn request_well_known(dest: &str) -> Result<Option<String>> {
+	trace!("Requesting well known for {dest}");
+	if !services().resolver.has_cached_override(dest) {
+		query_and_cache_override(dest, dest, 8448).await?;
+	}
+
+	let response = services()
+		.client
+		.well_known
+		.get(&format!("https://{dest}/.well-known/matrix/server"))
+		.send()
+		.await;
+
+	trace!("response: {:?}", response);
+	if let Err(e) = &response {
+		debug!("error: {e:?}");
+		return Ok(None);
+	}
+
+	let response = response?;
+	if !response.status().is_success() {
+		debug!("response not 2XX");
+		return Ok(None);
+	}
+
+	let text = response.text().await?;
+	trace!("response text: {:?}", text);
+	if text.len() >= 12288 {
+		debug_warn!("response contains junk");
+		return Ok(None);
+	}
+
+	let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default();
+
+	let m_server = body
+		.get("m.server")
+		.unwrap_or(&serde_json::Value::Null)
+		.as_str()
+		.unwrap_or_default();
+
+	if ruma::identifiers_validation::server_name::validate(m_server).is_err() {
+		debug_error!("response content missing or invalid");
+		return Ok(None);
+	}
+
+	debug_info!("{:?} found at {:?}", dest, m_server);
+	Ok(Some(m_server.to_owned()))
+}
+
+#[inline]
+async fn conditional_query_and_cache_override(overname: &str, hostname: &str, port: u16, cache: bool) -> Result<()> {
+	if cache {
+		query_and_cache_override(overname, hostname, port).await
+	} else {
+		Ok(())
+	}
+}
+
+#[tracing::instrument(skip_all, name = "ip")]
+async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> {
+	match services()
+		.resolver
+		.resolver
+		.lookup_ip(hostname.to_owned())
+		.await
+	{
+		Err(e) => handle_resolve_error(&e),
+		Ok(override_ip) => {
+			if hostname != overname {
+				debug_info!("{overname:?} overriden by {hostname:?}");
+			}
+
+			services().resolver.set_cached_override(
+				overname.to_owned(),
+				CachedOverride {
+					ips: override_ip.iter().collect(),
+					port,
+					expire: CachedOverride::default_expire(),
+				},
+			);
+
+			Ok(())
+		},
+	}
+}
+
+#[tracing::instrument(skip_all, name = "srv")]
+async fn query_srv_record(hostname: &'_ str) -> Result<Option<FedDest>> {
+	fn handle_successful_srv(srv: &SrvLookup) -> Option<FedDest> {
+		srv.iter().next().map(|result| {
+			FedDest::Named(
+				result.target().to_string().trim_end_matches('.').to_owned(),
+				format!(":{}", result.port()),
+			)
+		})
+	}
+
+	async fn lookup_srv(hostname: &str) -> Result<SrvLookup, ResolveError> {
+		debug!("querying SRV for {:?}", hostname);
+		let hostname = hostname.trim_end_matches('.');
+		services()
+			.resolver
+			.resolver
+			.srv_lookup(hostname.to_owned())
+			.await
+	}
+
+	let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")];
+
+	for hostname in hostnames {
+		match lookup_srv(&hostname).await {
+			Ok(result) => return Ok(handle_successful_srv(&result)),
+			Err(e) => handle_resolve_error(&e)?,
+		}
+	}
+
+	Ok(None)
+}
+
+#[allow(clippy::single_match_else)]
+fn handle_resolve_error(e: &ResolveError) -> Result<()> {
+	use hickory_resolver::error::ResolveErrorKind;
+
+	match *e.kind() {
+		ResolveErrorKind::NoRecordsFound {
+			..
+		} => {
+			// Raise to debug_warn if we can find out the result wasn't from cache
+			debug!("{e}");
+			Ok(())
+		},
+		_ => Err!(error!("DNS {e}")),
+	}
+}
+
+fn validate_dest(dest: &ServerName) -> Result<()> {
+	if dest == services().globals.server_name() {
+		return Err!("Won't send federation request to ourselves");
+	}
+
+	if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) {
+		validate_dest_ip_literal(dest)?;
+	}
+
+	Ok(())
+}
+
+fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> {
+	trace!("Destination is an IP literal, checking against IP range denylist.",);
+	debug_assert!(
+		dest.is_ip_literal() || !IPAddress::is_valid(dest.host()),
+		"Destination is not an IP literal."
+	);
+	let ip = IPAddress::parse(dest.host()).map_err(|e| {
+		debug_error!("Failed to parse IP literal from string: {}", e);
+		Error::BadServerResponse("Invalid IP address")
+	})?;
+
+	validate_ip(&ip)?;
+
+	Ok(())
+}
+
+pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> {
+	if !services().globals.valid_cidr_range(ip) {
+		return Err(Error::BadServerResponse("Not allowed to send requests to this IP"));
+	}
+
+	Ok(())
+}

src/service/sending/send.rs

@@ -1,7 +1,8 @@
 use std::{fmt::Debug, mem};
 
 use conduit::{
-	debug, debug_error, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result,
+	debug, debug_error, debug_info, debug_warn, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error,
+	Result,
 };
 use http::{header::AUTHORIZATION, HeaderValue};
 use ipaddress::IPAddress;
@@ -31,6 +32,16 @@ impl super::Service {
 			return Err!(Config("allow_federation", "Federation is disabled."));
 		}
 
+		if self
+			.server
+			.config
+			.forbidden_remote_server_names
+			.contains(&dest.to_owned())
+		{
+			debug_info!("Refusing to send outbound federation request to {dest}");
+			return Err!(Request(Forbidden("Federation with this homeserver is not allowed.")));
+		}
+
 		let actual = self.services.resolver.get_actual_dest(dest).await?;
 		let request = self.prepare::<T>(dest, &actual, req).await?;
 		self.execute::<T>(dest, &actual, request, client).await
@@ -62,7 +73,7 @@ impl super::Service {
 		trace!("Preparing request");
 		let mut http_request = req
 			.try_into_http_request::<Vec<u8>>(&actual.string, SATIR, &VERSIONS)
-			.map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?;
+			.map_err(|_| Error::BadServerResponse("Invalid destination"))?;
 
 		sign_request::<T>(&self.services.globals, dest, &mut http_request);
 
@@ -139,7 +150,10 @@ where
 		);
 	}
 
-	response.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
+	match response {
+		Err(_) => Err(Error::BadServerResponse("Server returned bad 200 response.")),
+		Ok(response) => Ok(response),
+	}
 }
 
 fn handle_error<T>(

src/service/services.rs

@@ -12,7 +12,7 @@ use tokio::sync::Mutex;
 use crate::{
 	account_data, admin, appservice, client, emergency, globals, key_backups,
 	manager::Manager,
-	media, presence, pusher, resolver, rooms, sending, server_keys, service,
+	media, presence, pusher, resolver, rooms, sending, service,
 	service::{Args, Map, Service},
 	transaction_ids, uiaa, updates, users,
 };
@@ -31,7 +31,6 @@ pub struct Services {
 	pub resolver: Arc<resolver::Service>,
 	pub rooms: rooms::Service,
 	pub sending: Arc<sending::Service>,
-	pub server_keys: Arc<server_keys::Service>,
 	pub transaction_ids: Arc<transaction_ids::Service>,
 	pub uiaa: Arc<uiaa::Service>,
 	pub updates: Arc<updates::Service>,
@@ -95,7 +94,6 @@ impl Services {
 				user: build!(rooms::user::Service),
 			},
 			sending: build!(sending::Service),
-			server_keys: build!(server_keys::Service),
 			transaction_ids: build!(transaction_ids::Service),
 			uiaa: build!(uiaa::Service),
 			updates: build!(updates::Service),

src/service/users/data.rs

@@ -246,6 +246,7 @@ impl Data {
 	/// Adds a new device to a user.
 	pub(super) fn create_device(
 		&self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option<String>,
+		client_ip: Option<String>,
 	) -> Result<()> {
 		// This method should never be called for nonexistent users. We shouldn't assert
 		// though...
@@ -266,7 +267,7 @@ impl Data {
 			&serde_json::to_vec(&Device {
 				device_id: device_id.into(),
 				display_name: initial_device_display_name,
-				last_seen_ip: None, // TODO
+				last_seen_ip: client_ip,
 				last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()),
 			})
 			.expect("Device::to_string never fails."),

src/service/users/mod.rs

@@ -328,9 +328,10 @@ impl Service {
 	/// Adds a new device to a user.
 	pub fn create_device(
 		&self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option<String>,
+		client_ip: Option<String>,
 	) -> Result<()> {
 		self.db
-			.create_device(user_id, device_id, token, initial_device_display_name)
+			.create_device(user_id, device_id, token, initial_device_display_name, client_ip)
 	}
 
 	/// Removes a device from a user.