Revert Object trait id() ref

* Accept Public aliases in URL deserializer

Update deserialize_one_or_many to deserialize recipient URL fields while
accepting `Public` and `as:Public` as aliases for the canonical
ActivityStreams public URL.

Add focused tests for single and array inputs, and verify that unrelated
string fields such as `content` are left unchanged.

https://github.com/LemmyNet/lemmy/issues/6465

* Deduplicate deserialized recipients

Drop repeated recipient URLs after deserialization so equivalent public
aliases such as `Public`, `as:Public`, and the canonical public URL do
not produce duplicate entries.

Update the helper documentation and tests to match the deduplicated
result.

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @Nutomic @dessalines

.gitignore

@@ -1,5 +1,12 @@
 /target
 /.idea
-/Cargo.lock
 perf.data*
-flamegraph.svg
+flamegraph.svg
+
+# direnv
+/.direnv
+/.envrc
+
+# nix flake
+/flake.nix
+/flake.lock

.rustfmt.toml

@@ -1,4 +1,4 @@
-edition="2021"
-imports_layout="HorizontalVertical"
-imports_granularity="Crate"
-reorder_imports=true
+edition = "2021"
+imports_layout = "HorizontalVertical"
+imports_granularity = "Crate"
+reorder_imports = true

.woodpecker.yml

@@ -1,54 +1,56 @@
-pipeline:
+variables:
+  - &rust_image "rust:1.91-bullseye"
+
+steps:
   cargo_fmt:
     image: rustdocker/rust:nightly
     commands:
       - /root/.cargo/bin/cargo fmt -- --check
-
-  cargo_check:
-    image: rust:1.70-bullseye
-    environment:
-      CARGO_HOME: .cargo
-    commands:
-      - cargo check --all-features --all-targets
+    when:
+      - event: pull_request
 
   cargo_clippy:
-    image: rust:1.70-bullseye
+    image: *rust_image
     environment:
       CARGO_HOME: .cargo
     commands:
       - rustup component add clippy
-      - cargo clippy --all-targets --all-features --
-        -D warnings -D deprecated -D clippy::perf -D clippy::complexity
-        -D clippy::dbg_macro -D clippy::inefficient_to_string
-        -D clippy::items-after-statements -D clippy::implicit_clone
-        -D clippy::wildcard_imports -D clippy::cast_lossless
-        -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls
-      - cargo clippy --all-features -- -D clippy::unwrap_used
+      - cargo clippy --all-targets --all-features
+    when:
+      - event: pull_request
 
   cargo_test:
-    image: rust:1.70-bullseye
+    image: *rust_image
     environment:
       CARGO_HOME: .cargo
     commands:
       - cargo test --all-features --no-fail-fast
+    when:
+      - event: pull_request
 
   cargo_doc:
-    image: rust:1.70-bullseye
+    image: *rust_image
     environment:
       CARGO_HOME: .cargo
     commands:
       - cargo doc --all-features
+    when:
+      - event: pull_request
 
   cargo_run_actix_example:
-    image: rust:1.70-bullseye
+    image: *rust_image
     environment:
       CARGO_HOME: .cargo
     commands:
       - cargo run --example local_federation actix-web
+    when:
+      - event: pull_request
 
   cargo_run_axum_example:
-    image: rust:1.70-bullseye
+    image: *rust_image
     environment:
       CARGO_HOME: .cargo
     commands:
       - cargo run --example local_federation axum
+    when:
+      - event: pull_request

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "activitypub_federation"
-version = "0.5.0-beta.3"
+version = "0.7.0-beta.11"
 edition = "2021"
 description = "High-level Activitypub framework"
 keywords = ["activitypub", "activitystreams", "federation", "fediverse"]
@@ -8,73 +8,91 @@ license = "AGPL-3.0"
 repository = "https://github.com/LemmyNet/activitypub-federation-rust"
 documentation = "https://docs.rs/activitypub_federation/"
 
+[features]
+default = ["actix-web", "axum"]
+actix-web = ["dep:actix-web", "dep:http02"]
+axum = ["dep:axum", "dep:tower"]
+axum-original-uri = ["dep:axum", "axum/original-uri"]
+
+[lints.clippy]
+perf = { level = "deny", priority = -1 }
+complexity = { level = "deny", priority = -1 }
+dbg_macro = "deny"
+inefficient_to_string = "deny"
+items-after-statements = "deny"
+implicit_clone = "deny"
+wildcard_imports = "deny"
+cast_lossless = "deny"
+manual_string_new = "deny"
+redundant_closure_for_method_calls = "deny"
+unwrap_used = "deny"
+
 [dependencies]
-chrono = { version = "0.4.26", features = ["clock"], default-features = false }
-serde = { version = "1.0.164", features = ["derive"] }
-async-trait = "0.1.68"
-url = { version = "2.4.0", features = ["serde"] }
-serde_json = { version = "1.0.96", features = ["preserve_order"] }
-anyhow = "1.0.71"
-reqwest = { version = "0.11.18", features = ["json", "stream"] }
-reqwest-middleware = "0.2.2"
-tracing = "0.1.37"
-base64 = "0.21.2"
-openssl = "0.10.54"
-once_cell = "1.18.0"
-http = "0.2.9"
-sha2 = "0.10.6"
-thiserror = "1.0.40"
-derive_builder = "0.12.0"
-itertools = "0.10.5"
-dyn-clone = "1.0.11"
+chrono = { version = "0.4.42", features = ["clock"], default-features = false }
+serde = { version = "1.0.228", features = ["derive"] }
+async-trait = "0.1.89"
+url = { version = "2.5.8", features = ["serde"] }
+serde_json = { version = "1.0.149", features = ["preserve_order"] }
+reqwest = { version = "0.13.1", default-features = false, features = [
+  "json",
+  "stream",
+] }
+reqwest-middleware = "0.5.0"
+tracing = "0.1.44"
+base64 = "0.22.1"
+rand = "0.8.5"
+rsa = "0.9.10"
+http = "1.4.0"
+sha2 = { version = "0.10.9", features = ["oid"] }
+thiserror = "2.0.17"
+derive_builder = "0.20.2"
+itertools = "0.14.0"
+dyn-clone = "1.0.20"
 enum_delegate = "0.2.0"
-httpdate = "1.0.2"
-http-signature-normalization-reqwest = { version = "0.8.0", default-features = false, features = [
+httpdate = "1.0.3"
+http-signature-normalization-reqwest = { version = "0.14.0", default-features = false, features = [
   "sha-2",
   "middleware",
+  "default-spawner",
 ] }
 http-signature-normalization = "0.7.0"
-bytes = "1.4.0"
-futures-core = { version = "0.3.28", default-features = false }
-pin-project-lite = "0.2.9"
+bytes = "1.11.0"
+futures-core = { version = "0.3.31", default-features = false }
+pin-project-lite = "0.2.16"
 activitystreams-kinds = "0.3.0"
-regex = { version = "1.8.4", default-features = false, features = ["std", "unicode-case"] }
-tokio = { version = "1.21.2", features = [
+regex = { version = "1.12.2", default-features = false, features = [
+  "std",
+  "unicode",
+] }
+tokio = { version = "1.49.0", features = [
   "sync",
   "rt",
   "rt-multi-thread",
   "time",
 ] }
+futures = "0.3.31"
+moka = { version = "0.12.12", features = ["future"] }
+either = "1.15.0"
 
 # Actix-web
-actix-web = { version = "4.3.1", default-features = false, optional = true }
+actix-web = { version = "4.12.1", default-features = false, optional = true }
+http02 = { package = "http", version = "0.2.12", optional = true }
 
 # Axum
-axum = { version = "0.6.18", features = [
+axum = { version = "0.8.8", features = [
   "json",
-  "headers",
 ], default-features = false, optional = true }
-tower = { version = "0.4.13", optional = true }
-hyper = { version = "0.14", optional = true }
-futures = "0.3.28"
-moka = { version = "0.11.2", features = ["future"] }
-
-[features]
-default = ["actix-web", "axum"]
-actix-web = ["dep:actix-web"]
-axum = ["dep:axum", "dep:tower", "dep:hyper"]
+tower = { version = "0.5.2", optional = true }
 
 [dev-dependencies]
-rand = "0.8.5"
-env_logger = "0.10.0"
-tower-http = { version = "0.4.0", features = ["map-request-body", "util"] }
-axum = { version = "0.6.18", features = [
-  "http1",
-  "tokio",
-  "query",
-], default-features = false }
-axum-macros = "0.3.7"
-tokio = { version = "1.21.2", features = ["full"] }
+anyhow = "1.0.100"
+axum = { version = "0.8.8", features = ["macros"] }
+axum-extra = { version = "0.12.5", features = ["typed-header"] }
+env_logger = "0.11.8"
+tokio = { version = "1.49.0", features = ["full"] }
+reqwest = { version = "0.13.1",features = [
+  "rustls"
+] }
 
 [profile.dev]
 strip = "symbols"
@@ -87,3 +105,8 @@ path = "examples/local_federation/main.rs"
 [[example]]
 name = "live_federation"
 path = "examples/live_federation/main.rs"
+
+# Speedup RSA key generation
+# https://github.com/RustCrypto/RSA/blob/master/README.md#example
+[profile.dev.package.num-bigint-dig]
+opt-level = 3

README.md

@@ -10,7 +10,7 @@ A high-level framework for [ActivityPub](https://www.w3.org/TR/activitypub/) fed
 
 The ActivityPub protocol is a decentralized social networking protocol. It allows web servers to exchange data using JSON over HTTP. Data can be fetched on demand, and also delivered directly to inboxes for live updates.
 
-While Activitypub is not in widespread use yet, is has the potential to form the basis of the next generation of social media. This is because it has a number of major advantages compared to existing platforms and alternative technologies:
+Activitypub has the potential to form the basis of the next generation of social media. This is because it has a number of major advantages compared to existing platforms and alternative technologies:
 
 - **Interoperability**: Imagine being able to comment under a Youtube video directly from twitter.com, and having the comment shown under the video on youtube.com. Or following a Subreddit from Facebook. Such functionality is already available on the equivalent Fediverse platforms, thanks to common usage of Activitypub.
 - **Ease of use**: From a user perspective, decentralized social media works almost identically to existing websites: a website with email and password based login. Unlike pure peer-to-peer networks, it is not necessary to handle private keys or install any local software.

docs/06_http_endpoints_axum.md

@@ -15,9 +15,9 @@ The next step is to allow other servers to fetch our actors and objects. For thi
 # use activitypub_federation::config::FederationMiddleware;
 # use axum::routing::get;
 # use crate::activitypub_federation::traits::Object;
-# use axum::headers::ContentType;
+# use axum_extra::headers::ContentType;
 # use activitypub_federation::FEDERATION_CONTENT_TYPE;
-# use axum::TypedHeader;
+# use axum_extra::TypedHeader;
 # use axum::response::IntoResponse;
 # use http::HeaderMap;
 # async fn generate_user_html(_: String, _: Data<DbConnection>) -> axum::response::Response { todo!() }
@@ -34,10 +34,9 @@ async fn main() -> Result<(), Error> {
         .layer(FederationMiddleware::new(data));
 
     let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
+    let listener = tokio::net::TcpListener::bind(addr).await?;
     tracing::debug!("listening on {}", addr);
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await?;
+    axum::serve(listener, app.into_make_service()).await?;
     Ok(())
 }
 
@@ -48,7 +47,7 @@ async fn http_get_user(
 ) -> impl IntoResponse {
     let accept = header_map.get("accept").map(|v| v.to_str().unwrap());
     if accept == Some(FEDERATION_CONTENT_TYPE) {
-        let db_user = data.read_local_user(name).await.unwrap();
+        let db_user = data.read_local_user(&name).await.unwrap();
         let json_user = db_user.into_json(&data).await.unwrap();
         FederationJson(WithContext::new_default(json_user)).into_response()
     }

docs/07_fetching_data.md

@@ -39,4 +39,4 @@ let user: DbUser = webfinger_resolve_actor("ruud@lemmy.world", &data).await?;
 # }).unwrap();
 ```
 
-Note that webfinger queries don't contain a leading `@`. It is possible tha there are multiple Activitypub IDs returned for a single webfinger query in case of multiple actors with the same name (for example Lemmy permits group and person with the same name). In this case `webfinger_resolve_actor` automatically loops and returns the first item which can be dereferenced successfully to the given type.
+Note that webfinger queries don't contain a leading `@`. It is possible that there are multiple Activitypub IDs returned for a single webfinger query in case of multiple actors with the same name (for example Lemmy permits group and person with the same name). In this case `webfinger_resolve_actor` automatically loops and returns the first item which can be dereferenced successfully to the given type.

docs/08_receiving_activities.md

@@ -1,6 +1,6 @@
 ## Sending and receiving activities
 
-Activitypub propagates actions across servers using `Activities`. For this each actor has an inbox and a public/private key pair. We already defined a `Person` actor with keypair. Whats left is to define an activity. This is similar to the way we defined `Person` and `Note` structs before. In this case we need to implement the [ActivityHandler](trait@crate::traits::ActivityHandler) trait.
+Activitypub propagates actions across servers using `Activities`. For this each actor has an inbox and a public/private key pair. We already defined a `Person` actor with keypair. Whats left is to define an activity. This is similar to the way we defined `Person` and `Note` structs before. In this case we need to implement the [Activity](trait@crate::traits::Activity) trait.
 
 ```
 # use serde::{Deserialize, Serialize};
@@ -10,7 +10,7 @@ Activitypub propagates actions across servers using `Activities`. For this each
 # use activitypub_federation::fetch::object_id::ObjectId;
 # use activitypub_federation::traits::tests::{DbConnection, DbUser};
 # use activitystreams_kinds::activity::FollowType;
-# use activitypub_federation::traits::ActivityHandler;
+# use activitypub_federation::traits::Activity;
 # use activitypub_federation::config::Data;
 # async fn send_accept() -> Result<(), Error> { Ok(()) }
 
@@ -25,7 +25,7 @@ pub struct Follow {
 }
 
 #[async_trait]
-impl ActivityHandler for Follow {
+impl Activity for Follow {
     type DataType = DbConnection;
     type Error = Error;
 
@@ -59,14 +59,14 @@ Next its time to setup the actual HTTP handler for the inbox. For this we first
 # use activitypub_federation::axum::inbox::{ActivityData, receive_activity};
 # use activitypub_federation::config::Data;
 # use activitypub_federation::protocol::context::WithContext;
-# use activitypub_federation::traits::ActivityHandler;
+# use activitypub_federation::traits::Activity;
 # use activitypub_federation::traits::tests::{DbConnection, DbUser, Follow};
 # use serde::{Deserialize, Serialize};
 # use url::Url;
 
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(untagged)]
-#[enum_delegate::implement(ActivityHandler)]
+#[enum_delegate::implement(Activity)]
 pub enum PersonAcceptedActivities {
     Follow(Follow),
 }

docs/09_sending_activities.md

@@ -4,6 +4,46 @@ To send an activity we need to initialize our previously defined struct, and pic
 
 ```
 # use activitypub_federation::config::FederationConfig;
+# use activitypub_federation::activity_queue::queue_activity;
+# use activitypub_federation::http_signatures::generate_actor_keypair;
+# use activitypub_federation::traits::Actor;
+# use activitypub_federation::fetch::object_id::ObjectId;
+# use activitypub_federation::traits::tests::{DB_USER, DbConnection, Follow};
+# tokio::runtime::Runtime::new().unwrap().block_on(async {
+# let db_connection = DbConnection;
+# let config = FederationConfig::builder()
+#     .domain("example.com")
+#     .app_data(db_connection)
+#     .build().await?;
+# let data = config.to_request_data();
+# let sender = DB_USER.clone();
+# let recipient = DB_USER.clone();
+let activity = Follow {
+    actor: ObjectId::parse("https://lemmy.ml/u/nutomic")?,
+    object: recipient.federation_id.clone().into(),
+    kind: Default::default(),
+    id: "https://lemmy.ml/activities/321".try_into()?
+};
+let inboxes = vec![recipient.shared_inbox_or_inbox()];
+
+queue_activity(&activity, &sender, inboxes, &data).await?;
+# Ok::<(), anyhow::Error>(())
+# }).unwrap()
+```
+
+The list of inboxes gets deduplicated (important for shared inbox). All inboxes on the local domain and those which fail the [crate::config::UrlVerifier] check are excluded from delivery. For each remaining inbox a background tasks is created. It signs the HTTP header with the given private key. Finally the activity is delivered to the inbox.
+
+It is possible that delivery fails because the target instance is temporarily unreachable. In this case the task is scheduled for retry after a certain waiting time. For each task delivery is retried up to 3 times after the initial attempt. The retry intervals are as follows:
+
+- one minute, in case of service restart
+- one hour, in case of instance maintenance
+- 2.5 days, in case of major incident with rebuild from backup
+
+In case [crate::config::FederationConfigBuilder::debug] is enabled, no background thread is used but activities are sent directly on the foreground. This makes it easier to catch delivery errors and avoids complicated steps to await delivery in tests.
+
+In some cases you may want to bypass the builtin activity queue, and implement your own. For example to specify different retry intervals, or to persist retries across application restarts. You can do it with the following code:
+```rust
+# use activitypub_federation::config::FederationConfig;
 # use activitypub_federation::activity_sending::SendActivityTask;
 # use activitypub_federation::http_signatures::generate_actor_keypair;
 # use activitypub_federation::traits::Actor;
@@ -28,23 +68,8 @@ let inboxes = vec![recipient.shared_inbox_or_inbox()];
 
 let sends = SendActivityTask::prepare(&activity, &sender, inboxes, &data).await?;
 for send in sends {
-    send.sign_and_send(&data).await?;
+send.sign_and_send(&data).await?;
 }
 # Ok::<(), anyhow::Error>(())
 # }).unwrap()
-```
-
-The list of inboxes gets deduplicated (important for shared inbox). All inboxes on the local
-domain and those which fail the [crate::config::UrlVerifier] check are excluded from delivery.
-For each remaining inbox a background tasks is created. It signs the HTTP header with the given
-private key. Finally the activity is delivered to the inbox.
-
-It is possible that delivery fails because the target instance is temporarily unreachable. In
-this case the task is scheduled for retry after a certain waiting time. For each task delivery
-is retried up to 3 times after the initial attempt. The retry intervals are as follows:
-
-- one minute, in case of service restart
-- one hour, in case of instance maintenance
-- 2.5 days, in case of major incident with rebuild from backup
-
-In case [crate::config::FederationConfigBuilder::debug] is enabled, no background thread is used but activities are sent directly on the foreground. This makes it easier to catch delivery errors and avoids complicated steps to await delivery in tests.
+```

docs/10_fetching_objects_with_unknown_type.md

@@ -32,6 +32,13 @@ impl Object for SearchableDbObjects {
     type Kind = SearchableObjects;
     type Error = anyhow::Error;
 
+    fn id(&self) -> Url {
+        match self {
+            SearchableDbObjects::User(p) => p.federation_id.clone(),
+            SearchableDbObjects::Post(n) => n.federation_id.clone(),
+        }
+    }
+
     async fn read_from_id(
         object_id: Url,
         data: &Data<Self::DataType>,

examples/live_federation/activities/create_post.rs

@@ -11,7 +11,7 @@ use activitypub_federation::{
     fetch::object_id::ObjectId,
     kinds::activity::CreateType,
     protocol::{context::WithContext, helpers::deserialize_one_or_many},
-    traits::{ActivityHandler, Object},
+    traits::{Activity, Object},
 };
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -50,7 +50,7 @@ impl CreatePost {
 }
 
 #[async_trait::async_trait]
-impl ActivityHandler for CreatePost {
+impl Activity for CreatePost {
     type DataType = DatabaseHandle;
     type Error = crate::error::Error;
 

examples/live_federation/http.rs

@@ -14,11 +14,11 @@ use activitypub_federation::{
     traits::Object,
 };
 use axum::{
+    debug_handler,
     extract::{Path, Query},
     response::{IntoResponse, Response},
     Json,
 };
-use axum_macros::debug_handler;
 use http::StatusCode;
 use serde::Deserialize;
 
@@ -61,7 +61,7 @@ pub async fn webfinger(
     data: Data<DatabaseHandle>,
 ) -> Result<Json<Webfinger>, Error> {
     let name = extract_webfinger_name(&query.resource, &data)?;
-    let db_user = data.read_user(&name)?;
+    let db_user = data.read_user(name)?;
     Ok(Json(build_webfinger_response(
         query.resource,
         db_user.ap_id.into_inner(),

examples/live_federation/main.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::unwrap_used)]
+
 use crate::{
     database::Database,
     http::{http_get_user, http_post_user_inbox, webfinger},
@@ -53,8 +55,8 @@ async fn main() -> Result<(), Error> {
     info!("Listen with HTTP server on {BIND_ADDRESS}");
     let config = config.clone();
     let app = Router::new()
-        .route("/:user", get(http_get_user))
-        .route("/:user/inbox", post(http_post_user_inbox))
+        .route("/{user}", get(http_get_user))
+        .route("/{user}/inbox", post(http_post_user_inbox))
         .route("/.well-known/webfinger", get(webfinger))
         .layer(FederationMiddleware::new(config));
 
@@ -62,9 +64,8 @@ async fn main() -> Result<(), Error> {
         .to_socket_addrs()?
         .next()
         .expect("Failed to lookup domain name");
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await?;
+    let listener = tokio::net::TcpListener::bind(addr).await?;
+    axum::serve(listener, app.into_make_service()).await?;
 
     Ok(())
 }

examples/live_federation/objects/person.rs

@@ -5,7 +5,7 @@ use activitypub_federation::{
     http_signatures::generate_actor_keypair,
     kinds::actor::PersonType,
     protocol::{public_key::PublicKey, verification::verify_domains_match},
-    traits::{ActivityHandler, Actor, Object},
+    traits::{Activity, Actor, Object},
 };
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
@@ -29,7 +29,7 @@ pub struct DbUser {
 /// List of all activities which this actor can receive.
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(untagged)]
-#[enum_delegate::implement(ActivityHandler)]
+#[enum_delegate::implement(Activity)]
 pub enum PersonAcceptedActivities {
     CreateNote(CreatePost),
 }
@@ -69,6 +69,10 @@ impl Object for DbUser {
     type Kind = Person;
     type Error = Error;
 
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
     fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
         Some(self.last_refreshed_at)
     }
@@ -122,10 +126,6 @@ impl Object for DbUser {
 }
 
 impl Actor for DbUser {
-    fn id(&self) -> Url {
-        self.ap_id.inner().clone()
-    }
-
     fn public_key_pem(&self) -> &str {
         &self.public_key
     }

examples/live_federation/objects/post.rs

@@ -21,7 +21,6 @@ pub struct DbPost {
     pub text: String,
     pub ap_id: ObjectId<DbPost>,
     pub creator: ObjectId<DbUser>,
-    pub local: bool,
 }
 
 #[derive(Deserialize, Serialize, Debug)]
@@ -51,6 +50,10 @@ impl Object for DbPost {
     type Kind = Note;
     type Error = Error;
 
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
     async fn read_from_id(
         _object_id: Url,
         _data: &Data<Self::DataType>,
@@ -59,7 +62,15 @@ impl Object for DbPost {
     }
 
     async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
-        unimplemented!()
+        Ok(Note {
+            kind: NoteType::Note,
+            id: self.ap_id,
+            content: self.text,
+            attributed_to: self.creator,
+            to: vec![public()],
+            tag: vec![],
+            in_reply_to: None,
+        })
     }
 
     async fn verify(
@@ -81,7 +92,6 @@ impl Object for DbPost {
             text: json.content,
             ap_id: json.id.clone(),
             creator: json.attributed_to.clone(),
-            local: false,
         };
 
         let mention = Mention {

examples/local_federation/activities/accept.rs

@@ -3,7 +3,7 @@ use activitypub_federation::{
     config::Data,
     fetch::object_id::ObjectId,
     kinds::activity::AcceptType,
-    traits::ActivityHandler,
+    traits::Activity,
 };
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -30,7 +30,7 @@ impl Accept {
 }
 
 #[async_trait::async_trait]
-impl ActivityHandler for Accept {
+impl Activity for Accept {
     type DataType = DatabaseHandle;
     type Error = crate::error::Error;
 

examples/local_federation/activities/create_post.rs

@@ -8,7 +8,7 @@ use activitypub_federation::{
     fetch::object_id::ObjectId,
     kinds::activity::CreateType,
     protocol::helpers::deserialize_one_or_many,
-    traits::{ActivityHandler, Object},
+    traits::{Activity, Object},
 };
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -38,7 +38,7 @@ impl CreatePost {
 }
 
 #[async_trait::async_trait]
-impl ActivityHandler for CreatePost {
+impl Activity for CreatePost {
     type DataType = DatabaseHandle;
     type Error = crate::error::Error;
 

examples/local_federation/activities/follow.rs

@@ -8,7 +8,7 @@ use activitypub_federation::{
     config::Data,
     fetch::object_id::ObjectId,
     kinds::activity::FollowType,
-    traits::{ActivityHandler, Actor},
+    traits::{Activity, Actor},
 };
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -35,7 +35,7 @@ impl Follow {
 }
 
 #[async_trait::async_trait]
-impl ActivityHandler for Follow {
+impl Activity for Follow {
     type DataType = DatabaseHandle;
     type Error = crate::error::Error;
 
@@ -67,7 +67,7 @@ impl ActivityHandler for Follow {
         let id = generate_object_id(data.domain())?;
         let accept = Accept::new(local_user.ap_id.clone(), self, id.clone());
         local_user
-            .send(accept, vec![follower.shared_inbox_or_inbox()], data)
+            .send(accept, vec![follower.shared_inbox_or_inbox()], false, data)
             .await?;
         Ok(())
     }

examples/local_federation/actix_web/http.rs

@@ -8,7 +8,7 @@ use activitypub_federation::{
     config::{Data, FederationConfig, FederationMiddleware},
     fetch::webfinger::{build_webfinger_response, extract_webfinger_name},
     protocol::context::WithContext,
-    traits::{Actor, Object},
+    traits::Object,
     FEDERATION_CONTENT_TYPE,
 };
 use actix_web::{web, web::Bytes, App, HttpRequest, HttpResponse, HttpServer};
@@ -89,7 +89,7 @@ pub async fn webfinger(
     data: Data<DatabaseHandle>,
 ) -> Result<HttpResponse, Error> {
     let name = extract_webfinger_name(&query.resource, &data)?;
-    let db_user = data.read_user(&name)?;
+    let db_user = data.read_user(name)?;
     Ok(HttpResponse::Ok().json(build_webfinger_response(
         query.resource.clone(),
         db_user.ap_id.into_inner(),

examples/local_federation/axum/http.rs

@@ -14,13 +14,13 @@ use activitypub_federation::{
     traits::Object,
 };
 use axum::{
+    debug_handler,
     extract::{Path, Query},
     response::IntoResponse,
     routing::{get, post},
     Json,
     Router,
 };
-use axum_macros::debug_handler;
 use serde::Deserialize;
 use std::net::ToSocketAddrs;
 use tracing::info;
@@ -29,9 +29,10 @@ pub fn listen(config: &FederationConfig<DatabaseHandle>) -> Result<(), Error> {
     let hostname = config.domain();
     info!("Listening with axum on {hostname}");
     let config = config.clone();
+
     let app = Router::new()
-        .route("/:user/inbox", post(http_post_user_inbox))
-        .route("/:user", get(http_get_user))
+        .route("/{user}/inbox", post(http_post_user_inbox))
+        .route("/{user}", get(http_get_user))
         .route("/.well-known/webfinger", get(webfinger))
         .layer(FederationMiddleware::new(config));
 
@@ -39,9 +40,14 @@ pub fn listen(config: &FederationConfig<DatabaseHandle>) -> Result<(), Error> {
         .to_socket_addrs()?
         .next()
         .expect("Failed to lookup domain name");
-    let server = axum::Server::bind(&addr).serve(app.into_make_service());
+    let fut = async move {
+        let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+        axum::serve(listener, app.into_make_service())
+            .await
+            .unwrap();
+    };
 
-    tokio::spawn(server);
+    tokio::spawn(fut);
     Ok(())
 }
 
@@ -78,7 +84,7 @@ async fn webfinger(
     data: Data<DatabaseHandle>,
 ) -> Result<Json<Webfinger>, Error> {
     let name = extract_webfinger_name(&query.resource, &data)?;
-    let db_user = data.read_user(&name)?;
+    let db_user = data.read_user(name)?;
     Ok(Json(build_webfinger_response(
         query.resource,
         db_user.ap_id.into_inner(),

examples/local_federation/instance.rs

@@ -28,6 +28,7 @@ pub async fn new_instance(
         .domain(hostname)
         .signed_fetch_actor(&system_user)
         .app_data(database)
+        .url_verifier(Box::new(MyUrlVerifier()))
         .debug(true)
         .build()
         .await?;
@@ -49,9 +50,11 @@ struct MyUrlVerifier();
 
 #[async_trait]
 impl UrlVerifier for MyUrlVerifier {
-    async fn verify(&self, url: &Url) -> Result<(), anyhow::Error> {
+    async fn verify(&self, url: &Url) -> Result<(), activitypub_federation::error::Error> {
         if url.domain() == Some("malicious.com") {
-            Err(anyhow!("malicious domain"))
+            Err(activitypub_federation::error::Error::Other(
+                "malicious domain".into(),
+            ))
         } else {
             Ok(())
         }

examples/local_federation/main.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::unwrap_used)]
+
 use crate::{
     instance::{listen, new_instance, Webserver},
     objects::post::DbPost,
@@ -5,6 +7,7 @@ use crate::{
 };
 use error::Error;
 use std::{env::args, str::FromStr};
+use tokio::try_join;
 use tracing::log::{info, LevelFilter};
 
 mod activities;
@@ -32,8 +35,10 @@ async fn main() -> Result<(), Error> {
         .map(|arg| Webserver::from_str(&arg).unwrap())
         .unwrap_or(Webserver::Axum);
 
-    let alpha = new_instance("localhost:8001", "alpha".to_string()).await?;
-    let beta = new_instance("localhost:8002", "beta".to_string()).await?;
+    let (alpha, beta) = try_join!(
+        new_instance("localhost:8001", "alpha".to_string()),
+        new_instance("localhost:8002", "beta".to_string())
+    )?;
     listen(&alpha, &webserver)?;
     listen(&beta, &webserver)?;
     info!("Local instances started");

examples/local_federation/objects/person.rs

@@ -6,13 +6,14 @@ use crate::{
     utils::generate_object_id,
 };
 use activitypub_federation::{
+    activity_queue::queue_activity,
     activity_sending::SendActivityTask,
     config::Data,
     fetch::{object_id::ObjectId, webfinger::webfinger_resolve_actor},
     http_signatures::generate_actor_keypair,
     kinds::actor::PersonType,
     protocol::{context::WithContext, public_key::PublicKey, verification::verify_domains_match},
-    traits::{ActivityHandler, Actor, Object},
+    traits::{Activity, Actor, Object},
 };
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
@@ -36,7 +37,7 @@ pub struct DbUser {
 /// List of all activities which this actor can receive.
 #[derive(Deserialize, Serialize, Debug)]
 #[serde(untagged)]
-#[enum_delegate::implement(ActivityHandler)]
+#[enum_delegate::implement(Activity)]
 pub enum PersonAcceptedActivities {
     Follow(Follow),
     Accept(Accept),
@@ -85,7 +86,7 @@ impl DbUser {
         let other: DbUser = webfinger_resolve_actor(other, data).await?;
         let id = generate_object_id(data.domain())?;
         let follow = Follow::new(self.ap_id.clone(), other.ap_id.clone(), id.clone());
-        self.send(follow, vec![other.shared_inbox_or_inbox()], data)
+        self.send(follow, vec![other.shared_inbox_or_inbox()], false, data)
             .await?;
         Ok(())
     }
@@ -98,24 +99,30 @@ impl DbUser {
             let user: DbUser = ObjectId::from(f).dereference(data).await?;
             inboxes.push(user.shared_inbox_or_inbox());
         }
-        self.send(create, inboxes, data).await?;
+        self.send(create, inboxes, true, data).await?;
         Ok(())
     }
 
-    pub(crate) async fn send<Activity>(
+    pub(crate) async fn send<A>(
         &self,
-        activity: Activity,
+        activity: A,
         recipients: Vec<Url>,
+        use_queue: bool,
         data: &Data<DatabaseHandle>,
-    ) -> Result<(), <Activity as ActivityHandler>::Error>
+    ) -> Result<(), Error>
     where
-        Activity: ActivityHandler + Serialize + Debug + Send + Sync,
-        <Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
+        A: Activity + Serialize + Debug + Send + Sync,
+        <A as Activity>::Error: From<anyhow::Error> + From<serde_json::Error>,
     {
         let activity = WithContext::new_default(activity);
-        let sends = SendActivityTask::prepare(&activity, self, recipients, data).await?;
-        for send in sends {
-            send.sign_and_send(data).await?;
+        // Send through queue in some cases and bypass it in others to test both code paths
+        if use_queue {
+            queue_activity(&activity, self, recipients, data).await?;
+        } else {
+            let sends = SendActivityTask::prepare(&activity, self, recipients, data).await?;
+            for send in sends {
+                send.sign_and_send(data).await?;
+            }
         }
         Ok(())
     }
@@ -127,6 +134,10 @@ impl Object for DbUser {
     type Kind = Person;
     type Error = Error;
 
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
     fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
         Some(self.last_refreshed_at)
     }
@@ -180,10 +191,6 @@ impl Object for DbUser {
 }
 
 impl Actor for DbUser {
-    fn id(&self) -> Url {
-        self.ap_id.inner().clone()
-    }
-
     fn public_key_pem(&self) -> &str {
         &self.public_key
     }

examples/local_federation/objects/post.rs

@@ -47,6 +47,10 @@ impl Object for DbPost {
     type Kind = Note;
     type Error = Error;
 
+    fn id(&self) -> Url {
+        self.ap_id.inner().clone()
+    }
+
     async fn read_from_id(
         object_id: Url,
         data: &Data<Self::DataType>,

src/activity_queue.rs

@@ -0,0 +1,521 @@
+//! Queue for signing and sending outgoing activities with retry
+//!
+#![doc = include_str!("../docs/09_sending_activities.md")]
+
+use crate::{
+    activity_sending::{build_tasks, SendActivityTask},
+    config::Data,
+    error::Error,
+    traits::{Activity, Actor},
+};
+
+use futures_core::Future;
+
+use reqwest_middleware::ClientWithMiddleware;
+use serde::Serialize;
+use std::{
+    fmt::{Debug, Display},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+use tokio::{
+    sync::mpsc::{unbounded_channel, UnboundedSender},
+    task::{JoinHandle, JoinSet},
+};
+use tracing::{info, warn};
+use url::Url;
+
+/// Send a new activity to the given inboxes with automatic retry on failure. Alternatively you
+/// can implement your own queue and then send activities using [[crate::activity_sending::SendActivityTask]].
+///
+/// - `activity`: The activity to be sent, gets converted to json
+/// - `private_key`: Private key belonging to the actor who sends the activity, for signing HTTP
+///   signature. Generated with [crate::http_signatures::generate_actor_keypair].
+/// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
+///   inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
+///   for each target actor.
+pub async fn queue_activity<A, Datatype, ActorType>(
+    activity: &A,
+    actor: &ActorType,
+    inboxes: Vec<Url>,
+    data: &Data<Datatype>,
+) -> Result<(), Error>
+where
+    A: Activity + Serialize + Debug,
+    Datatype: Clone,
+    ActorType: Actor,
+{
+    let config = &data.config;
+    let tasks = build_tasks(activity, actor, inboxes, data).await?;
+
+    for task in tasks {
+        // Don't use the activity queue if this is in debug mode, send and wait directly
+        if config.debug {
+            if let Err(err) = sign_and_send(
+                &task,
+                &config.client,
+                config.request_timeout,
+                Default::default(),
+            )
+            .await
+            {
+                warn!("{err}");
+            }
+        } else {
+            // This field is only optional to make builder work, its always present at this point
+            let activity_queue = config
+                .activity_queue
+                .as_ref()
+                .expect("Config has activity queue");
+            activity_queue.queue(task).await?;
+            let stats = activity_queue.get_stats();
+            let running = stats.running.load(Ordering::Relaxed);
+            if running == config.queue_worker_count && config.queue_worker_count != 0 {
+                warn!("Reached max number of send activity workers ({}). Consider increasing worker count to avoid federation delays", config.queue_worker_count);
+                warn!("{:?}", stats);
+            } else {
+                info!("{:?}", stats);
+            }
+        }
+    }
+    Ok(())
+}
+
+async fn sign_and_send(
+    task: &SendActivityTask,
+    client: &ClientWithMiddleware,
+    timeout: Duration,
+    retry_strategy: RetryStrategy,
+) -> Result<(), Error> {
+    retry(
+        || task.sign_and_send_internal(client, timeout),
+        retry_strategy,
+    )
+    .await
+}
+
+/// A simple activity queue which spawns tokio workers to send out requests
+/// When creating a queue, it will spawn a task per worker thread
+/// Uses an unbounded mpsc queue for communication (i.e, all messages are in memory)
+pub(crate) struct ActivityQueue {
+    // Stats shared between the queue and workers
+    stats: Arc<Stats>,
+    sender: UnboundedSender<SendActivityTask>,
+    sender_task: JoinHandle<()>,
+    retry_sender_task: JoinHandle<()>,
+}
+
+/// Simple stat counter to show where we're up to with sending messages
+/// This is a lock-free way to share things between tasks
+/// When reading these values it's possible (but extremely unlikely) to get stale data if a worker task is in the middle of transitioning
+#[derive(Default)]
+pub(crate) struct Stats {
+    pending: AtomicUsize,
+    running: AtomicUsize,
+    retries: AtomicUsize,
+    dead_last_hour: AtomicUsize,
+    completed_last_hour: AtomicUsize,
+}
+
+impl Debug for Stats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Activity queue stats: pending: {}, running: {}, retries: {}, dead: {}, complete: {}",
+            self.pending.load(Ordering::Relaxed),
+            self.running.load(Ordering::Relaxed),
+            self.retries.load(Ordering::Relaxed),
+            self.dead_last_hour.load(Ordering::Relaxed),
+            self.completed_last_hour.load(Ordering::Relaxed)
+        )
+    }
+}
+
+#[derive(Clone, Copy, Default)]
+struct RetryStrategy {
+    /// Amount of time in seconds to back off
+    backoff: usize,
+    /// Amount of times to retry
+    retries: usize,
+    /// If this particular request has already been retried, you can add an offset here to increment the count to start
+    offset: usize,
+    /// Number of seconds to sleep before trying
+    initial_sleep: usize,
+}
+
+/// A tokio spawned worker which is responsible for submitting requests to federated servers
+/// This will retry up to one time with the same signature, and if it fails, will move it to the retry queue.
+/// We need to retry activity sending in case the target instances is temporarily unreachable.
+/// In this case, the task is stored and resent when the instance is hopefully back up. This
+/// list shows the retry intervals, and which events of the target instance can be covered:
+/// - 60s (one minute, service restart) -- happens in the worker w/ same signature
+/// - 60min (one hour, instance maintenance) --- happens in the retry worker
+/// - 60h (2.5 days, major incident with rebuild from backup) --- happens in the retry worker
+async fn worker(
+    client: ClientWithMiddleware,
+    timeout: Duration,
+    message: SendActivityTask,
+    retry_queue: UnboundedSender<SendActivityTask>,
+    stats: Arc<Stats>,
+    strategy: RetryStrategy,
+) {
+    stats.pending.fetch_sub(1, Ordering::Relaxed);
+    stats.running.fetch_add(1, Ordering::Relaxed);
+
+    let outcome = sign_and_send(&message, &client, timeout, strategy).await;
+
+    // "Running" has finished, check the outcome
+    stats.running.fetch_sub(1, Ordering::Relaxed);
+
+    match outcome {
+        Ok(_) => {
+            stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+        Err(_err) => {
+            stats.retries.fetch_add(1, Ordering::Relaxed);
+            warn!(
+                "Sending activity {} to {} to the retry queue to be tried again later",
+                message.activity_id, message.inbox
+            );
+            // Send to the retry queue.  Ignoring whether it succeeds or not
+            retry_queue.send(message).ok();
+        }
+    }
+}
+
+async fn retry_worker(
+    client: ClientWithMiddleware,
+    timeout: Duration,
+    message: SendActivityTask,
+    stats: Arc<Stats>,
+    strategy: RetryStrategy,
+) {
+    // Because the times are pretty extravagant between retries, we have to re-sign each time
+    let outcome = retry(
+        || {
+            sign_and_send(
+                &message,
+                &client,
+                timeout,
+                RetryStrategy {
+                    backoff: 0,
+                    retries: 0,
+                    offset: 0,
+                    initial_sleep: 0,
+                },
+            )
+        },
+        strategy,
+    )
+    .await;
+
+    stats.retries.fetch_sub(1, Ordering::Relaxed);
+
+    match outcome {
+        Ok(_) => {
+            stats.completed_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+        Err(_err) => {
+            stats.dead_last_hour.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+}
+
+impl ActivityQueue {
+    fn new(
+        client: ClientWithMiddleware,
+        worker_count: usize,
+        retry_count: usize,
+        timeout: Duration,
+        backoff: usize, // This should be 60 seconds by default or 1 second in tests
+    ) -> Self {
+        let stats: Arc<Stats> = Default::default();
+
+        // This task clears the dead/completed stats every hour
+        let hour_stats = stats.clone();
+        tokio::spawn(async move {
+            let duration = Duration::from_secs(3600);
+            loop {
+                tokio::time::sleep(duration).await;
+                hour_stats.completed_last_hour.store(0, Ordering::Relaxed);
+                hour_stats.dead_last_hour.store(0, Ordering::Relaxed);
+            }
+        });
+
+        let (retry_sender, mut retry_receiver) = unbounded_channel();
+        let retry_stats = stats.clone();
+        let retry_client = client.clone();
+
+        // The "fast path" retry
+        // The backoff should be < 5 mins for this to work otherwise signatures may expire
+        // This strategy is the one that is used with the *same* signature
+        let strategy = RetryStrategy {
+            backoff,
+            retries: 1,
+            offset: 0,
+            initial_sleep: 0,
+        };
+
+        // The "retry path" strategy
+        // After the fast path fails, a task will sleep up to backoff ^ 2 and then retry again
+        let retry_strategy = RetryStrategy {
+            backoff,
+            retries: 3,
+            offset: 2,
+            initial_sleep: backoff.pow(2), // wait 60 mins before even trying
+        };
+
+        let retry_sender_task = tokio::spawn(async move {
+            let mut join_set = JoinSet::new();
+
+            while let Some(message) = retry_receiver.recv().await {
+                let retry_task = retry_worker(
+                    retry_client.clone(),
+                    timeout,
+                    message,
+                    retry_stats.clone(),
+                    retry_strategy,
+                );
+
+                if retry_count > 0 {
+                    // If we're over the limit of retries, wait for them to finish before spawning
+                    while join_set.len() >= retry_count {
+                        join_set.join_next().await;
+                    }
+
+                    join_set.spawn(retry_task);
+                } else {
+                    // If the retry worker count is `0` then just spawn and don't use the join_set
+                    tokio::spawn(retry_task);
+                }
+            }
+
+            while !join_set.is_empty() {
+                join_set.join_next().await;
+            }
+        });
+
+        let (sender, mut receiver) = unbounded_channel();
+
+        let sender_stats = stats.clone();
+
+        let sender_task = tokio::spawn(async move {
+            let mut join_set = JoinSet::new();
+
+            while let Some(message) = receiver.recv().await {
+                let task = worker(
+                    client.clone(),
+                    timeout,
+                    message,
+                    retry_sender.clone(),
+                    sender_stats.clone(),
+                    strategy,
+                );
+
+                if worker_count > 0 {
+                    // If we're over the limit of workers, wait for them to finish before spawning
+                    while join_set.len() >= worker_count {
+                        join_set.join_next().await;
+                    }
+
+                    join_set.spawn(task);
+                } else {
+                    // If the worker count is `0` then just spawn and don't use the join_set
+                    tokio::spawn(task);
+                }
+            }
+
+            drop(retry_sender);
+
+            while !join_set.is_empty() {
+                join_set.join_next().await;
+            }
+        });
+
+        Self {
+            stats,
+            sender,
+            sender_task,
+            retry_sender_task,
+        }
+    }
+
+    async fn queue(&self, message: SendActivityTask) -> Result<(), Error> {
+        self.stats.pending.fetch_add(1, Ordering::Relaxed);
+        self.sender
+            .send(message)
+            .map_err(|e| Error::ActivityQueueError(e.0.activity_id))?;
+
+        Ok(())
+    }
+
+    fn get_stats(&self) -> &Stats {
+        &self.stats
+    }
+
+    #[allow(unused)]
+    // Drops all the senders and shuts down the workers
+    pub(crate) async fn shutdown(self, wait_for_retries: bool) -> Result<Arc<Stats>, Error> {
+        drop(self.sender);
+
+        self.sender_task.await?;
+
+        if wait_for_retries {
+            self.retry_sender_task.await?;
+        }
+
+        Ok(self.stats)
+    }
+}
+
+/// Creates an activity queue using tokio spawned tasks
+/// Note: requires a tokio runtime
+pub(crate) fn create_activity_queue(
+    client: ClientWithMiddleware,
+    worker_count: usize,
+    retry_count: usize,
+    request_timeout: Duration,
+) -> ActivityQueue {
+    ActivityQueue::new(client, worker_count, retry_count, request_timeout, 60)
+}
+
+/// Retries a future action factory function up to `amount` times with an exponential backoff timer between tries
+async fn retry<T, E: Display + Debug, F: Future<Output = Result<T, E>>, A: FnMut() -> F>(
+    mut action: A,
+    strategy: RetryStrategy,
+) -> Result<T, E> {
+    let mut count = strategy.offset;
+
+    // Do an initial sleep if it's called for
+    if strategy.initial_sleep > 0 {
+        let sleep_dur = Duration::from_secs(strategy.initial_sleep as u64);
+        tokio::time::sleep(sleep_dur).await;
+    }
+
+    loop {
+        match action().await {
+            Ok(val) => return Ok(val),
+            Err(err) => {
+                if count < strategy.retries {
+                    count += 1;
+
+                    let sleep_amt = strategy.backoff.pow(count as u32) as u64;
+                    let sleep_dur = Duration::from_secs(sleep_amt);
+                    warn!("{err:?}.  Sleeping for {sleep_dur:?} and trying again");
+                    tokio::time::sleep(sleep_dur).await;
+                    continue;
+                } else {
+                    return Err(err);
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::http_signatures::generate_actor_keypair;
+    use axum::extract::State;
+    use bytes::Bytes;
+    use http::{HeaderMap, StatusCode};
+    use std::time::Instant;
+    use tracing::debug;
+
+    // This will periodically send back internal errors to test the retry
+    async fn dodgy_handler(
+        State(state): State<Arc<AtomicUsize>>,
+        headers: HeaderMap,
+        body: Bytes,
+    ) -> Result<(), StatusCode> {
+        debug!("Headers:{:?}", headers);
+        debug!("Body len:{}", body.len());
+
+        if state.fetch_add(1, Ordering::Relaxed) % 20 == 0 {
+            return Err(StatusCode::INTERNAL_SERVER_ERROR);
+        }
+        Ok(())
+    }
+
+    async fn test_server() {
+        use axum::{routing::post, Router};
+
+        // We should break every now and then ;)
+        let state = Arc::new(AtomicUsize::new(0));
+
+        let app = Router::new()
+            .route("/", post(dodgy_handler))
+            .with_state(state);
+
+        let listener = tokio::net::TcpListener::bind("0.0.0.0:8002").await.unwrap();
+        axum::serve(listener, app.into_make_service())
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    // Queues 100 messages and then asserts that the worker runs them
+    async fn test_activity_queue_workers() {
+        let num_workers = 64;
+        let num_messages: usize = 100;
+
+        tokio::spawn(test_server());
+
+        /*
+        // uncomment for debug logs & stats
+        use tracing::log::LevelFilter;
+
+        env_logger::builder()
+            .filter_level(LevelFilter::Warn)
+            .filter_module("activitypub_federation", LevelFilter::Info)
+            .format_timestamp(None)
+            .init();
+
+        */
+
+        let activity_queue = ActivityQueue::new(
+            reqwest::Client::default().into(),
+            num_workers,
+            num_workers,
+            Duration::from_secs(10),
+            1,
+        );
+
+        let keypair = generate_actor_keypair().unwrap();
+
+        let message = SendActivityTask {
+            actor_id: "http://localhost:8002".parse().unwrap(),
+            activity_id: "http://localhost:8002/activity".parse().unwrap(),
+            activity: "{}".into(),
+            inbox: "http://localhost:8002".parse().unwrap(),
+            private_key: keypair.private_key().unwrap(),
+            http_signature_compat: true,
+        };
+
+        let start = Instant::now();
+
+        for _ in 0..num_messages {
+            activity_queue.queue(message.clone()).await.unwrap();
+        }
+
+        info!("Queue Sent: {:?}", start.elapsed());
+
+        let stats = activity_queue.shutdown(true).await.unwrap();
+
+        info!(
+            "Queue Finished.  Num msgs: {}, Time {:?}, msg/s: {:0.0}",
+            num_messages,
+            start.elapsed(),
+            num_messages as f64 / start.elapsed().as_secs_f64()
+        );
+
+        assert_eq!(
+            stats.completed_last_hour.load(Ordering::Relaxed),
+            num_messages
+        );
+    }
+}

src/activity_sending.rs

@@ -7,158 +7,182 @@ use crate::{
     error::Error,
     http_signatures::sign_request,
     reqwest_shim::ResponseExt,
-    traits::{ActivityHandler, Actor},
+    traits::{Activity, Actor},
     FEDERATION_CONTENT_TYPE,
 };
-use anyhow::{anyhow, Context};
-
 use bytes::Bytes;
 use futures::StreamExt;
-use http::{header::HeaderName, HeaderMap, HeaderValue};
+use http::StatusCode;
 use httpdate::fmt_http_date;
 use itertools::Itertools;
-use openssl::pkey::{PKey, Private};
-use reqwest::Request;
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Response,
+};
 use reqwest_middleware::ClientWithMiddleware;
+use rsa::{pkcs8::DecodePrivateKey, RsaPrivateKey};
 use serde::Serialize;
 use std::{
-    self,
     fmt::{Debug, Display},
-    time::{Duration, SystemTime},
+    time::{Duration, Instant, SystemTime},
 };
-use tracing::debug;
+use tracing::{debug, warn};
 use url::Url;
 
 #[derive(Clone, Debug)]
-/// all info needed to send one activity to one inbox
-pub struct SendActivityTask<'a> {
-    actor_id: &'a Url,
-    activity_id: &'a Url,
-    activity: Bytes,
-    inbox: Url,
-    private_key: PKey<Private>,
-    http_signature_compat: bool,
+/// All info needed to sign and send one activity to one inbox. You should generally use
+/// [[crate::activity_queue::queue_activity]] unless you want implement your own queue.
+pub struct SendActivityTask {
+    pub(crate) actor_id: Url,
+    pub(crate) activity_id: Url,
+    pub(crate) activity: Bytes,
+    pub(crate) inbox: Url,
+    pub(crate) private_key: RsaPrivateKey,
+    pub(crate) http_signature_compat: bool,
 }
-impl Display for SendActivityTask<'_> {
+
+impl Display for SendActivityTask {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{} to {}", self.activity_id, self.inbox)
     }
 }
 
-impl SendActivityTask<'_> {
-    /// prepare an activity for sending
+impl SendActivityTask {
+    /// Prepare an activity for sending
     ///
     /// - `activity`: The activity to be sent, gets converted to json
     /// - `inboxes`: List of remote actor inboxes that should receive the activity. Ignores local actor
-    ///              inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
-    ///              for each target actor.
-    pub async fn prepare<'a, Activity, Datatype, ActorType>(
-        activity: &'a Activity,
+    ///   inboxes. Should be built by calling [crate::traits::Actor::shared_inbox_or_inbox]
+    ///   for each target actor.
+    pub async fn prepare<A, Datatype, ActorType>(
+        activity: &A,
         actor: &ActorType,
         inboxes: Vec<Url>,
         data: &Data<Datatype>,
-    ) -> Result<Vec<SendActivityTask<'a>>, <Activity as ActivityHandler>::Error>
+    ) -> Result<Vec<SendActivityTask>, Error>
     where
-        Activity: ActivityHandler + Serialize,
-        <Activity as ActivityHandler>::Error: From<anyhow::Error> + From<serde_json::Error>,
+        A: Activity + Serialize + Debug,
         Datatype: Clone,
         ActorType: Actor,
     {
-        let config = &data.config;
-        let actor_id = activity.actor();
-        let activity_id = activity.id();
-        let activity_serialized: Bytes = serde_json::to_vec(&activity)?.into();
-        let private_key = get_pkey_cached(data, actor).await?;
-
-        Ok(futures::stream::iter(
-            inboxes
-                .into_iter()
-                .unique()
-                .filter(|i| !config.is_local_url(i)),
-        )
-        .filter_map(|inbox| async {
-            if let Err(err) = config.verify_url_valid(&inbox).await {
-                debug!("inbox url invalid, skipping: {inbox}: {err}");
-                return None;
-            };
-            Some(SendActivityTask {
-                actor_id,
-                activity_id,
-                inbox,
-                activity: activity_serialized.clone(),
-                private_key: private_key.clone(),
-                http_signature_compat: config.http_signature_compat,
-            })
-        })
-        .collect()
-        .await)
+        build_tasks(activity, actor, inboxes, data).await
     }
 
     /// convert a sendactivitydata to a request, signing and sending it
-    pub async fn sign_and_send<Datatype: Clone>(
-        &self,
-        data: &Data<Datatype>,
-    ) -> Result<(), anyhow::Error> {
-        let req = self
-            .sign(&data.config.client, data.config.request_timeout)
-            .await?;
-        self.send(&data.config.client, req).await
+    pub async fn sign_and_send<Datatype: Clone>(&self, data: &Data<Datatype>) -> Result<(), Error> {
+        self.sign_and_send_internal(&data.config.client, data.config.request_timeout)
+            .await
     }
-    async fn sign(
+
+    pub(crate) async fn sign_and_send_internal(
         &self,
         client: &ClientWithMiddleware,
         timeout: Duration,
-    ) -> Result<Request, anyhow::Error> {
-        let task = self;
+    ) -> Result<(), Error> {
+        debug!("Sending {} to {}", self.activity_id, self.inbox,);
         let request_builder = client
-            .post(task.inbox.to_string())
+            .post(self.inbox.to_string())
             .timeout(timeout)
-            .headers(generate_request_headers(&task.inbox));
+            .headers(generate_request_headers(&self.inbox));
         let request = sign_request(
             request_builder,
-            task.actor_id,
-            task.activity.clone(),
-            task.private_key.clone(),
-            task.http_signature_compat,
+            &self.actor_id,
+            self.activity.clone(),
+            self.private_key.clone(),
+            self.http_signature_compat,
         )
-        .await
-        .context("signing request")?;
-        Ok(request)
+        .await?;
+
+        // Send the activity, and log a warning if its too slow.
+        let now = Instant::now();
+        let response = client.execute(request).await?;
+        let elapsed = now.elapsed().as_secs();
+        if elapsed > 10 {
+            warn!(
+                "Sending activity {} to {} took {}s",
+                self.activity_id, self.inbox, elapsed
+            );
+        }
+        self.handle_response(response).await
     }
 
-    async fn send(
-        &self,
-        client: &ClientWithMiddleware,
-        request: Request,
-    ) -> Result<(), anyhow::Error> {
-        let response = client.execute(request).await;
-
-        match response {
-            Ok(o) if o.status().is_success() => {
+    /// Based on the HTTP status code determines if an activity was delivered successfully. In that case
+    /// Ok is returned. Otherwise it returns Err and the activity send should be retried later.
+    ///
+    /// Equivalent code in mastodon: https://github.com/mastodon/mastodon/blob/v4.2.8/app/helpers/jsonld_helper.rb#L215-L217
+    async fn handle_response(&self, response: Response) -> Result<(), Error> {
+        match response.status() {
+            status if status.is_success() => {
                 debug!("Activity {self} delivered successfully");
                 Ok(())
             }
-            Ok(o) if o.status().is_client_error() => {
-                let text = o.text_limited().await.map_err(Error::other)?;
+            status
+                if status.is_client_error()
+                    && status != StatusCode::REQUEST_TIMEOUT
+                    && status != StatusCode::TOO_MANY_REQUESTS =>
+            {
+                let text = response.text_limited().await?;
                 debug!("Activity {self} was rejected, aborting: {text}");
                 Ok(())
             }
-            Ok(o) => {
-                let status = o.status();
-                let text = o.text_limited().await.map_err(Error::other)?;
-                Err(anyhow!(
+            status => {
+                let text = response.text_limited().await?;
+
+                Err(Error::Other(format!(
                     "Activity {self} failure with status {status}: {text}",
-                ))
+                )))
             }
-            Err(e) => Err(anyhow!("Activity {self} connection failure: {e}")),
         }
     }
 }
 
-async fn get_pkey_cached<ActorType>(
+pub(crate) async fn build_tasks<A, Datatype, ActorType>(
+    activity: &A,
+    actor: &ActorType,
+    inboxes: Vec<Url>,
+    data: &Data<Datatype>,
+) -> Result<Vec<SendActivityTask>, Error>
+where
+    A: Activity + Serialize + Debug,
+    Datatype: Clone,
+    ActorType: Actor,
+{
+    let config = &data.config;
+    let actor_id = activity.actor();
+    let activity_id = activity.id();
+    let activity_serialized: Bytes = serde_json::to_vec(activity)
+        .map_err(|e| Error::SerializeOutgoingActivity(e, format!("{:?}", activity)))?
+        .into();
+    let private_key = get_pkey_cached(data, actor).await?;
+
+    Ok(futures::stream::iter(
+        inboxes
+            .into_iter()
+            .unique()
+            .filter(|i| !config.is_local_url(i)),
+    )
+    .filter_map(|inbox| async {
+        if let Err(err) = config.verify_url_valid(&inbox).await {
+            debug!("inbox url invalid, skipping: {inbox}: {err}");
+            return None;
+        };
+        Some(SendActivityTask {
+            actor_id: actor_id.clone(),
+            activity_id: activity_id.clone(),
+            inbox,
+            activity: activity_serialized.clone(),
+            private_key: private_key.clone(),
+            http_signature_compat: config.http_signature_compat,
+        })
+    })
+    .collect()
+    .await)
+}
+
+pub(crate) async fn get_pkey_cached<ActorType>(
     data: &Data<impl Clone>,
     actor: &ActorType,
-) -> Result<PKey<Private>, anyhow::Error>
+) -> Result<RsaPrivateKey, Error>
 where
     ActorType: Actor,
 {
@@ -168,20 +192,23 @@ where
         .actor_pkey_cache
         .try_get_with_by_ref(&actor_id, async {
             let private_key_pem = actor.private_key_pem().ok_or_else(|| {
-                anyhow!("Actor {actor_id} does not contain a private key for signing")
+                Error::Other(format!(
+                    "Actor {actor_id} does not contain a private key for signing"
+                ))
             })?;
 
             // This is a mostly expensive blocking call, we don't want to tie up other tasks while this is happening
             let pkey = tokio::task::spawn_blocking(move || {
-                PKey::private_key_from_pem(private_key_pem.as_bytes())
-                    .map_err(|err| anyhow!("Could not create private key from PEM data:{err}"))
+                RsaPrivateKey::from_pkcs8_pem(&private_key_pem).map_err(|err| {
+                    Error::Other(format!("Could not create private key from PEM data:{err}"))
+                })
             })
             .await
-            .map_err(|err| anyhow!("Error joining: {err}"))??;
-            std::result::Result::<PKey<Private>, anyhow::Error>::Ok(pkey)
+            .map_err(|err| Error::Other(format!("Error joining: {err}")))??;
+            std::result::Result::<RsaPrivateKey, Error>::Ok(pkey)
         })
         .await
-        .map_err(|e| anyhow!("cloned error: {e}"))
+        .map_err(|e| Error::Other(format!("cloned error: {e}")))
 }
 
 pub(crate) fn generate_request_headers(inbox_url: &Url) -> HeaderMap {
@@ -207,33 +234,20 @@ pub(crate) fn generate_request_headers(inbox_url: &Url) -> HeaderMap {
 }
 
 #[cfg(test)]
+#[allow(clippy::unwrap_used)]
 mod tests {
-    use axum::extract::State;
-    use bytes::Bytes;
-    use http::StatusCode;
+    use super::*;
+    use crate::{config::FederationConfig, http_signatures::generate_actor_keypair};
     use std::{
         sync::{atomic::AtomicUsize, Arc},
         time::Instant,
     };
     use tracing::info;
 
-    use crate::{config::FederationConfig, http_signatures::generate_actor_keypair};
-
-    use super::*;
-
-    #[allow(unused)]
     // This will periodically send back internal errors to test the retry
-    async fn dodgy_handler(
-        State(state): State<Arc<AtomicUsize>>,
-        headers: HeaderMap,
-        body: Bytes,
-    ) -> Result<(), StatusCode> {
+    async fn dodgy_handler(headers: HeaderMap, body: Bytes) -> Result<(), StatusCode> {
         debug!("Headers:{:?}", headers);
         debug!("Body len:{}", body.len());
-
-        /*if state.fetch_add(1, Ordering::Relaxed) % 20 == 0 {
-            return Err(StatusCode::INTERNAL_SERVER_ERROR);
-        }*/
         Ok(())
     }
 
@@ -247,8 +261,8 @@ mod tests {
             .route("/", post(dodgy_handler))
             .with_state(state);
 
-        axum::Server::bind(&"0.0.0.0:8001".parse().unwrap())
-            .serve(app.into_make_service())
+        let listener = tokio::net::TcpListener::bind("0.0.0.0:8001").await.unwrap();
+        axum::serve(listener, app.into_make_service())
             .await
             .unwrap();
     }
@@ -274,8 +288,8 @@ mod tests {
         let keypair = generate_actor_keypair().unwrap();
 
         let message = SendActivityTask {
-            actor_id: &"http://localhost:8001".parse().unwrap(),
-            activity_id: &"http://localhost:8001/activity".parse().unwrap(),
+            actor_id: "http://localhost:8001".parse().unwrap(),
+            activity_id: "http://localhost:8001/activity".parse().unwrap(),
             activity: "{}".into(),
             inbox: "http://localhost:8001".parse().unwrap(),
             private_key: keypair.private_key().unwrap(),
@@ -291,10 +305,54 @@ mod tests {
         let start = Instant::now();
 
         for _ in 0..num_messages {
-            message.sign_and_send(&data).await?;
+            message.clone().sign_and_send(&data).await?;
         }
 
         info!("Queue Sent: {:?}", start.elapsed());
         Ok(())
     }
+
+    #[tokio::test]
+    async fn test_handle_response() {
+        let keypair = generate_actor_keypair().unwrap();
+        let message = SendActivityTask {
+            actor_id: "http://localhost:8001".parse().unwrap(),
+            activity_id: "http://localhost:8001/activity".parse().unwrap(),
+            activity: "{}".into(),
+            inbox: "http://localhost:8001".parse().unwrap(),
+            private_key: keypair.private_key().unwrap(),
+            http_signature_compat: true,
+        };
+
+        let res = |status| {
+            http::Response::builder()
+                .status(status)
+                .body(vec![])
+                .unwrap()
+                .into()
+        };
+
+        assert!(message.handle_response(res(StatusCode::OK)).await.is_ok());
+        assert!(message
+            .handle_response(res(StatusCode::BAD_REQUEST))
+            .await
+            .is_ok());
+
+        assert!(message
+            .handle_response(res(StatusCode::MOVED_PERMANENTLY))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::REQUEST_TIMEOUT))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::TOO_MANY_REQUESTS))
+            .await
+            .is_err());
+        assert!(message
+            .handle_response(res(StatusCode::INTERNAL_SERVER_ERROR))
+            .await
+            .is_err());
+    }
 }

src/actix_web/http_compat.rs

@@ -0,0 +1,30 @@
+//! Remove these conversion helpers after actix-web upgrades to http 1.0
+
+use std::str::FromStr;
+
+pub fn header_value(v: &http02::HeaderValue) -> http::HeaderValue {
+    http::HeaderValue::from_bytes(v.as_bytes()).expect("can convert http types")
+}
+
+pub fn header_map<'a, H>(m: H) -> http::HeaderMap
+where
+    H: IntoIterator<Item = (&'a http02::HeaderName, &'a http02::HeaderValue)>,
+{
+    let mut new_map = http::HeaderMap::new();
+    for (n, v) in m {
+        new_map.insert(
+            http::HeaderName::from_lowercase(n.as_str().as_bytes())
+                .expect("can convert http types"),
+            header_value(v),
+        );
+    }
+    new_map
+}
+
+pub fn method(m: &http02::Method) -> http::Method {
+    http::Method::from_bytes(m.as_str().as_bytes()).expect("can convert http types")
+}
+
+pub fn uri(m: &http02::Uri) -> http::Uri {
+    http::Uri::from_str(&m.to_string()).expect("can convert http types")
+}

src/actix_web/inbox.rs

@@ -1,52 +1,117 @@
 //! Handles incoming activities, verifying HTTP signatures and other checks
 
+use super::http_compat;
 use crate::{
     config::Data,
     error::Error,
-    fetch::object_id::ObjectId,
     http_signatures::{verify_body_hash, verify_signature},
-    traits::{ActivityHandler, Actor, Object},
+    parse_received_activity,
+    traits::{Activity, Actor, Object},
 };
 use actix_web::{web::Bytes, HttpRequest, HttpResponse};
-use anyhow::Context;
 use serde::de::DeserializeOwned;
 use tracing::debug;
 
 /// Handles incoming activities, verifying HTTP signatures and other checks
 ///
-/// After successful validation, activities are passed to respective [trait@ActivityHandler].
-pub async fn receive_activity<Activity, ActorT, Datatype>(
+/// After successful validation, activities are passed to respective [trait@Activity].
+pub async fn receive_activity<A, ActorT, Datatype>(
     request: HttpRequest,
     body: Bytes,
     data: &Data<Datatype>,
-) -> Result<HttpResponse, <Activity as ActivityHandler>::Error>
+) -> Result<HttpResponse, <A as Activity>::Error>
 where
-    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
-    ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Sync + 'static,
     for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
-    <Activity as ActivityHandler>::Error: From<anyhow::Error>
-        + From<Error>
-        + From<<ActorT as Object>::Error>
-        + From<serde_json::Error>,
-    <ActorT as Object>::Error: From<Error> + From<anyhow::Error>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
     Datatype: Clone,
 {
-    verify_body_hash(request.headers().get("Digest"), &body)?;
+    let (activity, _) = do_stuff::<A, ActorT, Datatype>(request, body, data).await?;
 
-    let activity: Activity = serde_json::from_slice(&body)
-        .with_context(|| format!("deserializing body: {}", String::from_utf8_lossy(&body)))?;
-    data.config.verify_url_and_domain(&activity).await?;
-    let actor = ObjectId::<ActorT>::from(activity.actor().clone())
-        .dereference(data)
-        .await?;
+    do_more_stuff(activity, data).await
+}
 
-    verify_signature(
-        request.headers(),
-        request.method(),
-        request.uri(),
-        actor.public_key_pem(),
-    )?;
+/// Workaround required so we can use references for the hook, instead of cloning data.
+pub trait ReceiveActivityHook<A, ActorT, Datatype>
+where
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + Clone + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Clone + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    /// Called when a new activity is recived
+    fn hook(
+        self,
+        activity: &A,
+        actor: &ActorT,
+        data: &Data<Datatype>,
+    ) -> impl std::future::Future<Output = Result<(), <A as Activity>::Error>>;
+}
 
+/// Same as [receive_activity], only that it calls the provided hook function before
+/// calling activity verify and receive functions.
+pub async fn receive_activity_with_hook<A, ActorT, Datatype>(
+    request: HttpRequest,
+    body: Bytes,
+    hook: impl ReceiveActivityHook<A, ActorT, Datatype>,
+    data: &Data<Datatype>,
+) -> Result<HttpResponse, <A as Activity>::Error>
+where
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + Clone + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Sync + Clone + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    let (activity, actor) = do_stuff::<A, ActorT, Datatype>(request, body, data).await?;
+
+    hook.hook(&activity, &actor, data).await?;
+
+    do_more_stuff(activity, data).await
+}
+
+async fn do_stuff<A, ActorT, Datatype>(
+    request: HttpRequest,
+    body: Bytes,
+    data: &Data<Datatype>,
+) -> Result<(A, ActorT), <A as Activity>::Error>
+where
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Sync + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    let digest_header = request
+        .headers()
+        .get("Digest")
+        .map(http_compat::header_value);
+    verify_body_hash(digest_header.as_ref(), &body)?;
+
+    let (activity, actor) = parse_received_activity::<A, ActorT, _>(&body, data).await?;
+
+    let headers = http_compat::header_map(request.headers());
+    let method = http_compat::method(request.method());
+    let uri = http_compat::uri(request.uri());
+    verify_signature(&headers, &method, &uri, actor.public_key_pem())?;
+
+    Ok((activity, actor))
+}
+
+async fn do_more_stuff<A, Datatype>(
+    activity: A,
+    data: &Data<Datatype>,
+) -> Result<HttpResponse, <A as Activity>::Error>
+where
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    Datatype: Clone,
+{
     debug!("Receiving activity {}", activity.id().to_string());
     activity.verify(data).await?;
     activity.receive(data).await?;
@@ -54,29 +119,65 @@ where
 }
 
 #[cfg(test)]
+#[allow(clippy::unwrap_used)]
 mod test {
     use super::*;
     use crate::{
         activity_sending::generate_request_headers,
         config::FederationConfig,
+        fetch::object_id::ObjectId,
         http_signatures::sign_request,
         traits::tests::{DbConnection, DbUser, Follow, DB_USER_KEYPAIR},
     };
     use actix_web::test::TestRequest;
     use reqwest::Client;
     use reqwest_middleware::ClientWithMiddleware;
+    use serde_json::json;
     use url::Url;
 
+    /// Remove this conversion helper after actix-web upgrades to http 1.0
+    fn header_pair(
+        p: (&http::HeaderName, &http::HeaderValue),
+    ) -> (http02::HeaderName, http02::HeaderValue) {
+        (
+            http02::HeaderName::from_lowercase(p.0.as_str().as_bytes()).unwrap(),
+            http02::HeaderValue::from_bytes(p.1.as_bytes()).unwrap(),
+        )
+    }
+
     #[tokio::test]
-    async fn test_receive_activity() {
+    async fn test_receive_activity_hook() {
         let (body, incoming_request, config) = setup_receive_test().await;
-        receive_activity::<Follow, DbUser, DbConnection>(
+        let res = receive_activity_with_hook::<Follow, DbUser, DbConnection>(
             incoming_request.to_http_request(),
             body,
+            Dummy,
             &config.to_request_data(),
         )
-        .await
-        .unwrap();
+        .await;
+        assert_eq!(res.err(), Some(Error::Other("test-error".to_string())));
+    }
+
+    struct Dummy;
+
+    impl<A, ActorT, Datatype> ReceiveActivityHook<A, ActorT, Datatype> for Dummy
+    where
+        A: Activity<DataType = Datatype> + DeserializeOwned + Send + Clone + 'static,
+        ActorT: Object<DataType = Datatype> + Actor + Send + Clone + 'static,
+        for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+        <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+        <ActorT as Object>::Error: From<Error>,
+        Datatype: Clone,
+    {
+        async fn hook(
+            self,
+            _activity: &A,
+            _actor: &ActorT,
+            _data: &Data<Datatype>,
+        ) -> Result<(), <A as Activity>::Error> {
+            // ensure that hook gets called by returning this value
+            Err(Error::Other("test-error".to_string()).into())
+        }
     }
 
     #[tokio::test]
@@ -91,8 +192,7 @@ mod test {
         .err()
         .unwrap();
 
-        let e = err.root_cause().downcast_ref::<Error>().unwrap();
-        assert_eq!(e, &Error::ActivityBodyDigestInvalid)
+        assert_eq!(&err, &Error::ActivityBodyDigestInvalid)
     }
 
     #[tokio::test]
@@ -108,26 +208,52 @@ mod test {
         .err()
         .unwrap();
 
-        let e = err.root_cause().downcast_ref::<Error>().unwrap();
-        assert_eq!(e, &Error::ActivitySignatureInvalid)
+        assert_eq!(&err, &Error::ActivitySignatureInvalid)
     }
 
-    async fn setup_receive_test() -> (Bytes, TestRequest, FederationConfig<DbConnection>) {
+    #[tokio::test]
+    async fn test_receive_unparseable_activity() {
+        let (_, _, config) = setup_receive_test().await;
+
+        let actor = Url::parse("http://ds9.lemmy.ml/u/lemmy_alpha").unwrap();
+        let activity_id = "http://localhost:123/1";
+        let activity = json!({
+          "actor": actor.as_str(),
+          "to": ["https://www.w3.org/ns/activitystreams#Public"],
+          "object": "http://ds9.lemmy.ml/post/1",
+          "cc": ["http://enterprise.lemmy.ml/c/main"],
+          "type": "Delete",
+          "id": activity_id
+        }
+        );
+        let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
+        let incoming_request = construct_request(&body, &actor).await;
+
+        // intentionally cause a parse error by using wrong type for deser
+        let res = receive_activity::<Follow, DbUser, DbConnection>(
+            incoming_request.to_http_request(),
+            body,
+            &config.to_request_data(),
+        )
+        .await;
+
+        match res {
+            Err(Error::ParseReceivedActivity { err: _, id }) => {
+                assert_eq!(activity_id, id.expect("has url").as_str());
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    async fn construct_request(body: &Bytes, actor: &Url) -> TestRequest {
         let inbox = "https://example.com/inbox";
         let headers = generate_request_headers(&Url::parse(inbox).unwrap());
         let request_builder = ClientWithMiddleware::from(Client::default())
             .post(inbox)
             .headers(headers);
-        let activity = Follow {
-            actor: ObjectId::parse("http://localhost:123").unwrap(),
-            object: ObjectId::parse("http://localhost:124").unwrap(),
-            kind: Default::default(),
-            id: "http://localhost:123/1".try_into().unwrap(),
-        };
-        let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
         let outgoing_request = sign_request(
             request_builder,
-            &activity.actor.into_inner(),
+            actor,
             body.clone(),
             DB_USER_KEYPAIR.private_key().unwrap(),
             false,
@@ -136,8 +262,20 @@ mod test {
         .unwrap();
         let mut incoming_request = TestRequest::post().uri(outgoing_request.url().path());
         for h in outgoing_request.headers() {
-            incoming_request = incoming_request.append_header(h);
+            incoming_request = incoming_request.append_header(header_pair(h));
         }
+        incoming_request
+    }
+
+    async fn setup_receive_test() -> (Bytes, TestRequest, FederationConfig<DbConnection>) {
+        let activity = Follow {
+            actor: ObjectId::parse("http://localhost:123").unwrap(),
+            object: ObjectId::parse("http://localhost:124").unwrap(),
+            kind: Default::default(),
+            id: "http://localhost:123/1".try_into().unwrap(),
+        };
+        let body: Bytes = serde_json::to_vec(&activity).unwrap().into();
+        let incoming_request = construct_request(&body, activity.actor.inner()).await;
 
         let config = FederationConfig::builder()
             .domain("localhost:8002")

src/actix_web/mod.rs

@@ -1,8 +1,10 @@
 //! Utilities for using this library with actix-web framework
 
+mod http_compat;
 pub mod inbox;
 #[doc(hidden)]
 pub mod middleware;
+pub mod response;
 
 use crate::{
     config::Data,
@@ -21,11 +23,18 @@ pub async fn signing_actor<A>(
     data: &Data<<A as Object>::DataType>,
 ) -> Result<A, <A as Object>::Error>
 where
-    A: Object + Actor,
-    <A as Object>::Error: From<Error> + From<anyhow::Error>,
+    A: Object + Actor + Send + Sync,
+    <A as Object>::Error: From<Error>,
     for<'de2> <A as Object>::Kind: Deserialize<'de2>,
 {
-    verify_body_hash(request.headers().get("Digest"), &body.unwrap_or_default())?;
+    let digest_header = request
+        .headers()
+        .get("Digest")
+        .map(http_compat::header_value);
+    verify_body_hash(digest_header.as_ref(), &body.unwrap_or_default())?;
 
-    http_signatures::signing_actor(request.headers(), request.method(), request.uri(), data).await
+    let headers = http_compat::header_map(request.headers());
+    let method = http_compat::method(request.method());
+    let uri = http_compat::uri(request.uri());
+    http_signatures::signing_actor(&headers, &method, &uri, data).await
 }

src/actix_web/response.rs

@@ -0,0 +1,50 @@
+//! Generate HTTP responses for Activitypub ojects
+
+use crate::{
+    protocol::{context::WithContext, tombstone::Tombstone},
+    FEDERATION_CONTENT_TYPE,
+};
+use actix_web::HttpResponse;
+use http02::header::VARY;
+use serde::Serialize;
+use serde_json::Value;
+use url::Url;
+
+/// Generates HTTP response to serve the object for fetching from other instances.
+///
+/// If possible use [Object.http_response]
+/// which also handles redirects for remote objects and deletions.
+///
+/// `federation_context` is the value of `@context`.
+pub fn create_http_response<T: Serialize>(
+    data: T,
+    federation_context: &Value,
+) -> Result<HttpResponse, serde_json::Error> {
+    let json = serde_json::to_string_pretty(&WithContext::new(data, federation_context.clone()))?;
+
+    Ok(HttpResponse::Ok()
+        .content_type(FEDERATION_CONTENT_TYPE)
+        .insert_header((VARY, "Accept"))
+        .body(json))
+}
+
+pub(crate) fn create_tombstone_response(
+    id: Url,
+    federation_context: &Value,
+) -> Result<HttpResponse, serde_json::Error> {
+    let tombstone = Tombstone::new(id);
+    let json =
+        serde_json::to_string_pretty(&WithContext::new(tombstone, federation_context.clone()))?;
+
+    Ok(HttpResponse::Gone()
+        .content_type(FEDERATION_CONTENT_TYPE)
+        .status(actix_web::http::StatusCode::GONE)
+        .insert_header((VARY, "Accept"))
+        .body(json))
+}
+
+pub(crate) fn redirect_remote_object(url: &Url) -> HttpResponse {
+    let mut res = HttpResponse::PermanentRedirect();
+    res.insert_header((actix_web::http::header::LOCATION, url.as_str()));
+    res.finish()
+}

src/axum/inbox.rs

@@ -5,13 +5,12 @@
 use crate::{
     config::Data,
     error::Error,
-    fetch::object_id::ObjectId,
-    http_signatures::{verify_body_hash, verify_signature},
-    traits::{ActivityHandler, Actor, Object},
+    http_signatures::verify_signature,
+    parse_received_activity,
+    traits::{Activity, Actor, Object},
 };
 use axum::{
-    async_trait,
-    body::{Bytes, HttpBody},
+    body::Body,
     extract::FromRequest,
     http::{Request, StatusCode},
     response::{IntoResponse, Response},
@@ -21,28 +20,20 @@ use serde::de::DeserializeOwned;
 use tracing::debug;
 
 /// Handles incoming activities, verifying HTTP signatures and other checks
-pub async fn receive_activity<Activity, ActorT, Datatype>(
+pub async fn receive_activity<A, ActorT, Datatype>(
     activity_data: ActivityData,
     data: &Data<Datatype>,
-) -> Result<(), <Activity as ActivityHandler>::Error>
+) -> Result<(), <A as Activity>::Error>
 where
-    Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
-    ActorT: Object<DataType = Datatype> + Actor + Send + 'static,
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Sync + 'static,
     for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
-    <Activity as ActivityHandler>::Error: From<anyhow::Error>
-        + From<Error>
-        + From<<ActorT as Object>::Error>
-        + From<serde_json::Error>,
-    <ActorT as Object>::Error: From<Error> + From<anyhow::Error>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
     Datatype: Clone,
 {
-    verify_body_hash(activity_data.headers.get("Digest"), &activity_data.body)?;
-
-    let activity: Activity = serde_json::from_slice(&activity_data.body)?;
-    data.config.verify_url_and_domain(&activity).await?;
-    let actor = ObjectId::<ActorT>::from(activity.actor().clone())
-        .dereference(data)
-        .await?;
+    let (activity, actor) =
+        parse_received_activity::<A, ActorT, _>(&activity_data.body, data).await?;
 
     verify_signature(
         &activity_data.headers,
@@ -66,29 +57,38 @@ pub struct ActivityData {
     body: Vec<u8>,
 }
 
-#[async_trait]
-impl<S, B> FromRequest<S, B> for ActivityData
+impl<S> FromRequest<S> for ActivityData
 where
-    Bytes: FromRequest<S, B>,
-    B: HttpBody + Send + 'static,
     S: Send + Sync,
-    <B as HttpBody>::Error: std::fmt::Display,
-    <B as HttpBody>::Data: Send,
 {
     type Rejection = Response;
 
-    async fn from_request(req: Request<B>, _state: &S) -> Result<Self, Self::Rejection> {
-        let (parts, body) = req.into_parts();
+    async fn from_request(req: Request<Body>, _state: &S) -> Result<Self, Self::Rejection> {
+        #[allow(unused_mut)]
+        let (mut parts, body) = req.into_parts();
+
+        // take the full URI to handle nested routers
+        // OriginalUri::from_request_parts has an Infallible error type
+        #[cfg(feature = "axum-original-uri")]
+        let uri = {
+            use axum::extract::{FromRequestParts, OriginalUri};
+            OriginalUri::from_request_parts(&mut parts, _state)
+                .await
+                .expect("infallible")
+                .0
+        };
+        #[cfg(not(feature = "axum-original-uri"))]
+        let uri = parts.uri;
 
         // this wont work if the body is an long running stream
-        let bytes = hyper::body::to_bytes(body)
+        let bytes = axum::body::to_bytes(body, usize::MAX)
             .await
             .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response())?;
 
         Ok(Self {
             headers: parts.headers,
             method: parts.method,
-            uri: parts.uri,
+            uri,
             body: bytes.to_vec(),
         })
     }

src/axum/json.rs

@@ -9,7 +9,7 @@
 //! # use activitypub_federation::traits::Object;
 //! # use activitypub_federation::traits::tests::{DbConnection, DbUser, Person};
 //! async fn http_get_user(Path(name): Path<String>, data: Data<DbConnection>) -> Result<FederationJson<WithContext<Person>>, Error> {
-//!     let user: DbUser = data.read_local_user(name).await?;
+//!     let user: DbUser = data.read_local_user(&name).await?;
 //!     let person = user.into_json(&data).await?;
 //!
 //!     Ok(FederationJson(WithContext::new_default(person)))

src/axum/middleware.rs

@@ -1,5 +1,5 @@
 use crate::config::{Data, FederationConfig, FederationMiddleware};
-use axum::{async_trait, body::Body, extract::FromRequestParts, http::Request, response::Response};
+use axum::{body::Body, extract::FromRequestParts, http::Request, response::Response};
 use http::{request::Parts, StatusCode};
 use std::task::{Context, Poll};
 use tower::{Layer, Service};
@@ -43,7 +43,6 @@ where
     }
 }
 
-#[async_trait]
 impl<S, T: Clone + 'static> FromRequestParts<S> for Data<T>
 where
     S: Send + Sync,

src/config.rs

@@ -15,23 +15,29 @@
 //! ```
 
 use crate::{
+    activity_queue::{create_activity_queue, ActivityQueue},
     error::Error,
+    http_signatures::sign_request,
     protocol::verification::verify_domains_match,
-    traits::{ActivityHandler, Actor},
+    traits::{Activity, Actor},
+    utils::validate_ip,
 };
-use anyhow::anyhow;
 use async_trait::async_trait;
+use bytes::Bytes;
 use derive_builder::Builder;
 use dyn_clone::{clone_trait_object, DynClone};
 use moka::future::Cache;
-use openssl::pkey::{PKey, Private};
-use reqwest_middleware::ClientWithMiddleware;
+use regex::Regex;
+use reqwest::{redirect::Policy, Client, Request};
+use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
+use rsa::{pkcs8::DecodePrivateKey, RsaPrivateKey};
 use serde::de::DeserializeOwned;
 use std::{
     ops::Deref,
     sync::{
         atomic::{AtomicU32, Ordering},
         Arc,
+        OnceLock,
     },
     time::Duration,
 };
@@ -51,9 +57,14 @@ pub struct FederationConfig<T: Clone> {
     /// [crate::fetch::object_id::ObjectId] for more details.
     #[builder(default = "20")]
     pub(crate) http_fetch_limit: u32,
-    #[builder(default = "reqwest::Client::default().into()")]
-    /// HTTP client used for all outgoing requests. Middleware can be used to add functionality
-    /// like log tracing or retry of failed requests.
+    #[builder(default = "default_client()")]
+    /// HTTP client used for all outgoing requests. When passing a custom client here you should
+    /// also disable redirects and set timeouts.
+    ///
+    /// Middleware can be used to add functionality like log tracing or retry of failed requests.
+    /// Redirects are disabled by default, because automatic redirect URLs can't be validated.
+    /// Instead a single redirect is handled manually. The default client sets a timeout of 10s
+    ///  to avoid excessive resource usage when connecting to dead servers.
     pub(crate) client: ClientWithMiddleware,
     /// Run library in debug mode. This allows usage of http and localhost urls. It also sends
     /// outgoing activities synchronously, not in background thread. This helps to make tests
@@ -80,12 +91,31 @@ pub struct FederationConfig<T: Clone> {
     /// This can be used to implement secure mode federation.
     /// <https://docs.joinmastodon.org/spec/activitypub/#secure-mode>
     #[builder(default = "None", setter(custom))]
-    pub(crate) signed_fetch_actor: Option<Arc<(Url, PKey<Private>)>>,
+    pub(crate) signed_fetch_actor: Option<Arc<(Url, RsaPrivateKey)>>,
     #[builder(
         default = "Cache::builder().max_capacity(10000).build()",
         setter(custom)
     )]
-    pub(crate) actor_pkey_cache: Cache<Url, PKey<Private>>,
+    pub(crate) actor_pkey_cache: Cache<Url, RsaPrivateKey>,
+    /// Queue for sending outgoing activities. Only optional to make builder work, its always
+    /// present once constructed.
+    #[builder(setter(skip))]
+    pub(crate) activity_queue: Option<Arc<ActivityQueue>>,
+    /// When sending with activity queue: Number of tasks that can be in-flight concurrently.
+    /// Tasks are retried once after a minute, then put into the retry queue.
+    /// Setting this count to `0` means that there is no limit to concurrency
+    #[builder(default = "0")]
+    pub(crate) queue_worker_count: usize,
+    /// When sending with activity queue: Number of concurrent tasks that are being retried
+    /// in-flight concurrently. Tasks are retried after an hour, then again in 60 hours.
+    /// Setting this count to `0` means that there is no limit to concurrency
+    #[builder(default = "0")]
+    pub(crate) queue_retry_count: usize,
+}
+
+pub(crate) fn domain_regex() -> &'static Regex {
+    static DOMAIN_REGEX: OnceLock<Regex> = OnceLock::new();
+    DOMAIN_REGEX.get_or_init(|| Regex::new(r"^[a-zA-Z0-9.-]*$").expect("compile regex"))
 }
 
 impl<T: Clone> FederationConfig<T> {
@@ -94,19 +124,16 @@ impl<T: Clone> FederationConfig<T> {
         FederationConfigBuilder::default()
     }
 
-    pub(crate) async fn verify_url_and_domain<Activity, Datatype>(
-        &self,
-        activity: &Activity,
-    ) -> Result<(), Error>
+    pub(crate) async fn verify_url_and_domain<A, Datatype>(&self, activity: &A) -> Result<(), Error>
     where
-        Activity: ActivityHandler<DataType = Datatype> + DeserializeOwned + Send + 'static,
+        A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
     {
         verify_domains_match(activity.id(), activity.actor())?;
         self.verify_url_valid(activity.id()).await?;
         if self.is_local_url(activity.id()) {
-            return Err(Error::UrlVerificationError(anyhow!(
-                "Activity was sent from local instance"
-            )));
+            return Err(Error::UrlVerificationError(
+                "Activity was sent from local instance",
+            ));
         }
 
         Ok(())
@@ -129,12 +156,12 @@ impl<T: Clone> FederationConfig<T> {
             "https" => {}
             "http" => {
                 if !self.allow_http_urls {
-                    return Err(Error::UrlVerificationError(anyhow!(
-                        "Http urls are only allowed in debug mode"
-                    )));
+                    return Err(Error::UrlVerificationError(
+                        "Http urls are only allowed in debug mode",
+                    ));
                 }
             }
-            _ => return Err(Error::UrlVerificationError(anyhow!("Invalid url scheme"))),
+            _ => return Err(Error::UrlVerificationError("Invalid url scheme")),
         };
 
         // Urls which use our local domain are not a security risk, no further verification needed
@@ -142,22 +169,35 @@ impl<T: Clone> FederationConfig<T> {
             return Ok(());
         }
 
-        if url.domain().is_none() {
-            return Err(Error::UrlVerificationError(anyhow!(
-                "Url must have a domain"
-            )));
+        let Some(domain) = url.domain() else {
+            return Err(Error::UrlVerificationError("Url must have a domain"));
+        };
+        if !domain_regex().is_match(domain) {
+            return Err(Error::UrlVerificationError("Invalid characters in domain"));
         }
 
-        if url.domain() == Some("localhost") && !self.debug {
-            return Err(Error::UrlVerificationError(anyhow!(
-                "Localhost is only allowed in debug mode"
-            )));
+        // Extra checks only for production mode
+        if !self.debug {
+            if url.port().is_some() {
+                return Err(Error::UrlVerificationError("Explicit port is not allowed"));
+            }
+
+            let allow_local = std::env::var("DANGER_FEDERATION_ALLOW_LOCAL_IP").is_ok();
+            if !allow_local && validate_ip(&url).await.is_err() {
+                return Err(Error::DomainResolveError(domain.to_string()));
+            }
         }
 
-        self.url_verifier
-            .verify(url)
-            .await
-            .map_err(Error::UrlVerificationError)?;
+        // It is valid but uncommon for domains to end with `.` char. Drop this so it cant be used
+        // to bypass domain blocklist. Avoid cloning url in common case.
+        if domain.ends_with('.') {
+            let mut url = url.clone();
+            let domain = &domain[0..domain.len() - 1];
+            url.set_host(Some(domain))?;
+            self.url_verifier.verify(&url).await?;
+        } else {
+            self.url_verifier.verify(url).await?;
+        }
 
         Ok(())
     }
@@ -165,11 +205,17 @@ impl<T: Clone> FederationConfig<T> {
     /// Returns true if the url refers to this instance. Handles hostnames like `localhost:8540` for
     /// local debugging.
     pub(crate) fn is_local_url(&self, url: &Url) -> bool {
-        let mut domain = url.host_str().expect("id has domain").to_string();
-        if let Some(port) = url.port() {
-            domain = format!("{}:{}", domain, port);
+        match url.host_str() {
+            Some(domain) => {
+                let domain = if let Some(port) = url.port() {
+                    format!("{}:{}", domain, port)
+                } else {
+                    domain.to_string()
+                };
+                domain == self.domain
+            }
+            None => false,
         }
-        domain == self.domain
     }
 
     /// Returns the local domain
@@ -185,9 +231,9 @@ impl<T: Clone> FederationConfigBuilder<T> {
             .private_key_pem()
             .expect("actor does not have a private key to sign with");
 
-        let private_key = PKey::private_key_from_pem(private_key_pem.as_bytes())
-            .expect("Could not decode PEM data");
-        self.signed_fetch_actor = Some(Some(Arc::new((actor.id(), private_key))));
+        let private_key =
+            RsaPrivateKey::from_pkcs8_pem(&private_key_pem).expect("Could not decode PEM data");
+        self.signed_fetch_actor = Some(Some(Arc::new((actor.id().clone(), private_key))));
         self
     }
 
@@ -203,7 +249,14 @@ impl<T: Clone> FederationConfigBuilder<T> {
     /// queue for outgoing activities, which is stored internally in the config struct.
     /// Requires a tokio runtime for the background queue.
     pub async fn build(&mut self) -> Result<FederationConfig<T>, FederationConfigBuilderError> {
-        let config = self.partial_build()?;
+        let mut config = self.partial_build()?;
+        let queue = create_activity_queue(
+            config.client.clone(),
+            config.queue_worker_count,
+            config.queue_retry_count,
+            config.request_timeout,
+        );
+        config.activity_queue = Some(Arc::new(queue));
         Ok(config)
     }
 }
@@ -227,7 +280,7 @@ impl<T: Clone> Deref for FederationConfig<T> {
 /// # use async_trait::async_trait;
 /// # use url::Url;
 /// # use activitypub_federation::config::UrlVerifier;
-/// # use anyhow::anyhow;
+/// # use activitypub_federation::error::Error;
 /// # #[derive(Clone)]
 /// # struct DatabaseConnection();
 /// # async fn get_blocklist(_: &DatabaseConnection) -> Vec<String> {
@@ -240,11 +293,11 @@ impl<T: Clone> Deref for FederationConfig<T> {
 ///
 /// #[async_trait]
 /// impl UrlVerifier for Verifier {
-///     async fn verify(&self, url: &Url) -> Result<(), anyhow::Error> {
+///     async fn verify(&self, url: &Url) -> Result<(), Error> {
 ///         let blocklist = get_blocklist(&self.db_connection).await;
 ///         let domain = url.domain().unwrap().to_string();
 ///         if blocklist.contains(&domain) {
-///             Err(anyhow!("Domain is blocked"))
+///             Err(Error::Other("Domain is blocked".into()))
 ///         } else {
 ///             Ok(())
 ///         }
@@ -254,7 +307,7 @@ impl<T: Clone> Deref for FederationConfig<T> {
 #[async_trait]
 pub trait UrlVerifier: DynClone + Send {
     /// Should return Ok iff the given url is valid for processing.
-    async fn verify(&self, url: &Url) -> Result<(), anyhow::Error>;
+    async fn verify(&self, url: &Url) -> Result<(), Error>;
 }
 
 /// Default URL verifier which does nothing.
@@ -263,7 +316,7 @@ struct DefaultUrlVerifier();
 
 #[async_trait]
 impl UrlVerifier for DefaultUrlVerifier {
-    async fn verify(&self, _url: &Url) -> Result<(), anyhow::Error> {
+    async fn verify(&self, _url: &Url) -> Result<(), Error> {
         Ok(())
     }
 }
@@ -278,9 +331,10 @@ clone_trait_object!(UrlVerifier);
 /// prevent denial of service attacks, where an attacker triggers fetching of recursive objects.
 ///
 /// <https://www.w3.org/TR/activitypub/#security-recursive-objects>
+#[derive(Clone)]
 pub struct Data<T: Clone> {
     pub(crate) config: FederationConfig<T>,
-    pub(crate) request_counter: AtomicU32,
+    pub(crate) request_counter: RequestCounter,
 }
 
 impl<T: Clone> Data<T> {
@@ -303,7 +357,35 @@ impl<T: Clone> Data<T> {
     }
     /// Total number of outgoing HTTP requests made with this data.
     pub fn request_count(&self) -> u32 {
-        self.request_counter.load(Ordering::Relaxed)
+        self.request_counter.0.load(Ordering::Relaxed)
+    }
+
+    /// Add HTTP signature to arbitrary request
+    pub async fn sign_request(&self, req: RequestBuilder, body: Bytes) -> Result<Request, Error> {
+        let (actor_id, private_key_pem) =
+            self.config
+                .signed_fetch_actor
+                .as_deref()
+                .ok_or(Error::Other(
+                    "config value signed_fetch_actor is none".to_string(),
+                ))?;
+        sign_request(
+            req,
+            actor_id,
+            body,
+            private_key_pem.clone(),
+            self.config.http_signature_compat,
+        )
+        .await
+    }
+
+    /// Resolve domain of the url and throw error if it points to local/private IP.
+    pub async fn is_valid_ip(&self, url: &Url) -> Result<(), Error> {
+        if self.config.debug {
+            return Ok(());
+        }
+
+        validate_ip(url).await
     }
 }
 
@@ -315,6 +397,16 @@ impl<T: Clone> Deref for Data<T> {
     }
 }
 
+/// Wrapper to implement `Clone`
+#[derive(Default)]
+pub(crate) struct RequestCounter(pub(crate) AtomicU32);
+
+impl Clone for RequestCounter {
+    fn clone(&self) -> Self {
+        RequestCounter(self.0.load(Ordering::Relaxed).into())
+    }
+}
+
 /// Middleware for HTTP handlers which provides access to [Data]
 #[derive(Clone)]
 pub struct FederationMiddleware<T: Clone>(pub(crate) FederationConfig<T>);
@@ -325,3 +417,45 @@ impl<T: Clone> FederationMiddleware<T> {
         FederationMiddleware(config)
     }
 }
+
+fn default_client() -> ClientWithMiddleware {
+    let timeout = Duration::from_secs(10);
+    Client::builder()
+        .redirect(Policy::none())
+        .timeout(timeout)
+        .connect_timeout(timeout)
+        .build()
+        .unwrap_or_else(|_| Client::default())
+        .into()
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod test {
+    use super::*;
+
+    async fn config() -> FederationConfig<i32> {
+        FederationConfig::builder()
+            .domain("example.com")
+            .app_data(1)
+            .build()
+            .await
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_url_is_local() -> Result<(), Error> {
+        let config = config().await;
+        assert!(config.is_local_url(&Url::parse("http://example.com")?));
+        assert!(!config.is_local_url(&Url::parse("http://other.com")?));
+        // ensure that missing domain doesnt cause crash
+        assert!(!config.is_local_url(&Url::parse("http://127.0.0.1")?));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_get_domain() {
+        let config = config().await;
+        assert_eq!("example.com", config.domain());
+    }
+}

src/error.rs

@@ -1,5 +1,15 @@
 //! Error messages returned by this library
 
+use crate::fetch::webfinger::WebFingerError;
+use http_signature_normalization_reqwest::SignError;
+use rsa::{
+    errors::Error as RsaError,
+    pkcs8::{spki::Error as SpkiError, Error as Pkcs8Error},
+};
+use std::string::FromUtf8Error;
+use tokio::task::JoinError;
+use url::Url;
+
 /// Error messages returned by this library
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -13,11 +23,14 @@ pub enum Error {
     #[error("Response body limit was reached during fetch")]
     ResponseBodyLimit,
     /// Object to be fetched was deleted
-    #[error("Object to be fetched was deleted")]
-    ObjectDeleted,
+    #[error("Fetched remote object {0} which was deleted")]
+    ObjectDeleted(Url),
     /// url verification error
     #[error("URL failed verification: {0}")]
-    UrlVerificationError(anyhow::Error),
+    UrlVerificationError(&'static str),
+    /// Resolving domain points to local IP.
+    #[error("Resolving domain {0} points to local IP address. This may indicate an attacker attempting to access internal services. If intentional, you can ignore this error by setting DANGER_FEDERATION_ALLOW_LOCAL_IP=1")]
+    DomainResolveError(String),
     /// Incoming activity has invalid digest for body
     #[error("Incoming activity has invalid digest for body")]
     ActivityBodyDigestInvalid,
@@ -26,18 +39,76 @@ pub enum Error {
     ActivitySignatureInvalid,
     /// Failed to resolve actor via webfinger
     #[error("Failed to resolve actor via webfinger")]
-    WebfingerResolveFailed,
-    /// other error
+    WebfingerResolveFailed(#[from] WebFingerError),
+    /// Failed to serialize outgoing activity
+    #[error("Failed to serialize outgoing activity {1}: {0}")]
+    SerializeOutgoingActivity(serde_json::Error, String),
+    /// Failed to parse an object fetched from url
+    #[error("Failed to parse object {1} with content {2}: {0}")]
+    ParseFetchedObject(serde_json::Error, Url, String),
+    /// Failed to parse an activity received from another instance
+    #[error("Failed to parse incoming activity {}: {0}", match .id {
+        Some(t) => format!("with id {t}"),
+        None => String::new(),
+    })]
+    ParseReceivedActivity {
+        /// The parse error
+        err: serde_json::Error,
+        /// ID of the Activitypub object which caused this error
+        id: Option<Url>,
+    },
+    /// Reqwest Middleware Error
     #[error(transparent)]
-    Other(#[from] anyhow::Error),
+    ReqwestMiddleware(#[from] reqwest_middleware::Error),
+    /// Reqwest Error
+    #[error(transparent)]
+    Reqwest(#[from] reqwest::Error),
+    /// UTF-8 error
+    #[error(transparent)]
+    Utf8(#[from] FromUtf8Error),
+    /// Url Parse
+    #[error(transparent)]
+    UrlParse(#[from] url::ParseError),
+    /// Signing errors
+    #[error(transparent)]
+    SignError(#[from] SignError),
+    /// Failed to queue activity for sending
+    #[error("Failed to queue activity {0} for sending")]
+    ActivityQueueError(Url),
+    /// Stop activity queue
+    #[error(transparent)]
+    StopActivityQueue(#[from] JoinError),
+    /// Attempted to fetch object which doesn't have valid ActivityPub Content-Type
+    #[error(
+        "Attempted to fetch object from {0} which doesn't have valid ActivityPub Content-Type"
+    )]
+    FetchInvalidContentType(Url),
+    /// Attempted to fetch object but the response's id field doesn't match
+    #[error("Attempted to fetch object from {0} but the response's id field doesn't match")]
+    FetchWrongId(Url),
+    /// I/O error from OS
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
+    /// Other generic errors
+    #[error("{0}")]
+    Other(String),
 }
 
-impl Error {
-    pub(crate) fn other<T>(error: T) -> Self
-    where
-        T: Into<anyhow::Error>,
-    {
-        Error::Other(error.into())
+impl From<RsaError> for Error {
+    fn from(value: RsaError) -> Self {
+        Error::Other(value.to_string())
+    }
+}
+
+impl From<Pkcs8Error> for Error {
+    fn from(value: Pkcs8Error) -> Self {
+        Error::Other(value.to_string())
+    }
+}
+
+impl From<SpkiError> for Error {
+    fn from(value: SpkiError) -> Self {
+        Error::Other(value.to_string())
     }
 }
 

src/fetch/collection_id.rs

@@ -20,12 +20,8 @@ where
     for<'de2> <Kind as Collection>::Kind: Deserialize<'de2>,
 {
     /// Construct a new CollectionId instance
-    pub fn parse<T>(url: T) -> Result<Self, url::ParseError>
-    where
-        T: TryInto<Url>,
-        url::ParseError: From<<T as TryInto<Url>>::Error>,
-    {
-        Ok(Self(Box::new(url.try_into()?), PhantomData::<Kind>))
+    pub fn parse(url: &str) -> Result<Self, url::ParseError> {
+        Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
     }
 
     /// Fetches collection over HTTP
@@ -40,9 +36,10 @@ where
     where
         <Kind as Collection>::Error: From<Error>,
     {
-        let json = fetch_object_http(&self.0, data).await?;
-        Kind::verify(&json, &self.0, data).await?;
-        Kind::from_json(json, owner, data).await
+        let res = fetch_object_http(&self.0, data).await?;
+        let redirect_url = &res.url;
+        Kind::verify(&res.object, redirect_url, data).await?;
+        Kind::from_json(res.object, owner, data).await
     }
 }
 
@@ -95,3 +92,13 @@ where
         CollectionId(Box::new(url), PhantomData::<Kind>)
     }
 }
+
+impl<Kind> PartialEq for CollectionId<Kind>
+where
+    Kind: Collection,
+    for<'de2> <Kind as Collection>::Kind: serde::Deserialize<'de2>,
+{
+    fn eq(&self, other: &Self) -> bool {
+        self.0.eq(&other.0) && self.1 == other.1
+    }
+}

src/fetch/mod.rs

@@ -4,13 +4,14 @@
 
 use crate::{
     config::Data,
-    error::Error,
+    error::{Error, Error::ParseFetchedObject},
+    extract_id,
     http_signatures::sign_request,
     reqwest_shim::ResponseExt,
     FEDERATION_CONTENT_TYPE,
 };
 use bytes::Bytes;
-use http::StatusCode;
+use http::{header::LOCATION, HeaderValue, StatusCode};
 use serde::de::DeserializeOwned;
 use std::sync::atomic::Ordering;
 use tracing::info;
@@ -23,6 +24,16 @@ pub mod object_id;
 /// Resolves identifiers of the form `name@example.com`
 pub mod webfinger;
 
+/// Response from fetching a remote object
+pub struct FetchObjectResponse<Kind> {
+    /// The resolved object
+    pub object: Kind,
+    /// Contains the final URL (different from request URL in case of redirect)
+    pub url: Url,
+    content_type: Option<HeaderValue>,
+    object_id: Option<Url>,
+}
+
 /// Fetch a remote object over HTTP and convert to `Kind`.
 ///
 /// [crate::fetch::object_id::ObjectId::dereference] wraps this function to add caching and
@@ -34,12 +45,53 @@ pub mod webfinger;
 /// [Error::RequestLimit]. This prevents denial of service attacks where an attack triggers
 /// infinite, recursive fetching of data.
 ///
-/// The `Accept` header will be set to the content of [`FEDERATION_CONTENT_TYPE`].
+/// The `Accept` header will be set to the content of [`FEDERATION_CONTENT_TYPE`]. When parsing the
+/// response it ensures that it has a valid `Content-Type` header as defined by ActivityPub, to
+/// prevent security vulnerabilities like [this one](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36).
+/// Additionally it checks that the `id` field is identical to the fetch URL (after redirects).
 pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
     url: &Url,
     data: &Data<T>,
-) -> Result<Kind, Error> {
-    fetch_object_http_with_accept(url, data, FEDERATION_CONTENT_TYPE).await
+) -> Result<FetchObjectResponse<Kind>, Error> {
+    static FETCH_CONTENT_TYPE: HeaderValue = HeaderValue::from_static(FEDERATION_CONTENT_TYPE);
+    const VALID_RESPONSE_CONTENT_TYPES: [&str; 3] = [
+        FEDERATION_CONTENT_TYPE, // lemmy
+        r#"application/ld+json; profile="https://www.w3.org/ns/activitystreams""#, // activitypub standard
+        r#"application/activity+json; charset=utf-8"#,                             // mastodon
+    ];
+    let res = fetch_object_http_with_accept(url, data, &FETCH_CONTENT_TYPE, false).await?;
+
+    // Ensure correct content-type to prevent vulnerabilities, with case insensitive comparison.
+    let content_type = res
+        .content_type
+        .as_ref()
+        .and_then(|c| Some(c.to_str().ok()?.to_lowercase()))
+        .ok_or(Error::FetchInvalidContentType(res.url.clone()))?;
+    if !VALID_RESPONSE_CONTENT_TYPES.contains(&content_type.as_str()) {
+        return Err(Error::FetchInvalidContentType(res.url));
+    }
+
+    // Ensure id field matches final url after redirect
+    if res.object_id.as_ref() != Some(&res.url) {
+        if let Some(res_object_id) = res.object_id {
+            data.config.verify_url_valid(&res_object_id).await?;
+            // If id is different but still on the same domain, attempt to request object
+            // again from url in id field.
+            if res_object_id.domain() == res.url.domain() {
+                return Box::pin(fetch_object_http(&res_object_id, data)).await;
+            }
+        }
+        // Failed to fetch the object from its specified id
+        return Err(Error::FetchWrongId(res.url));
+    }
+
+    // Dont allow fetching local object. Only check this after the request as a local url
+    // may redirect to a remote object.
+    if data.config.is_local_url(&res.url) {
+        return Err(Error::NotFound);
+    }
+
+    Ok(res)
 }
 
 /// Fetch a remote object over HTTP and convert to `Kind`. This function works exactly as
@@ -47,15 +99,16 @@ pub async fn fetch_object_http<T: Clone, Kind: DeserializeOwned>(
 async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
     url: &Url,
     data: &Data<T>,
-    content_type: &str,
-) -> Result<Kind, Error> {
+    content_type: &HeaderValue,
+    recursive: bool,
+) -> Result<FetchObjectResponse<Kind>, Error> {
     let config = &data.config;
-    // dont fetch local objects this way
-    debug_assert!(url.domain() != Some(&config.domain));
     config.verify_url_valid(url).await?;
     info!("Fetching remote object {}", url.to_string());
 
-    let counter = data.request_counter.fetch_add(1, Ordering::SeqCst);
+    let mut counter = data.request_counter.0.fetch_add(1, Ordering::SeqCst);
+    // fetch_add returns old value so we need to increment manually here
+    counter += 1;
     if counter > config.http_fetch_limit {
         return Err(Error::RequestLimit);
     }
@@ -75,14 +128,75 @@ async fn fetch_object_http_with_accept<T: Clone, Kind: DeserializeOwned>(
             data.config.http_signature_compat,
         )
         .await?;
-        config.client.execute(req).await.map_err(Error::other)?
+        config.client.execute(req).await?
     } else {
-        req.send().await.map_err(Error::other)?
+        req.send().await?
     };
 
-    if res.status() == StatusCode::GONE {
-        return Err(Error::ObjectDeleted);
+    // Allow a single redirect using recursion. Further redirects are ignored.
+    let location = res.headers().get(LOCATION).and_then(|l| l.to_str().ok());
+    if let (Some(location), false) = (location, recursive) {
+        let location = location.parse()?;
+        return Box::pin(fetch_object_http_with_accept(
+            &location,
+            data,
+            content_type,
+            true,
+        ))
+        .await;
     }
 
-    res.json_limited().await
+    if res.status() == StatusCode::GONE {
+        return Err(Error::ObjectDeleted(url.clone()));
+    }
+
+    let url = res.url().clone();
+    let content_type = res.headers().get("Content-Type").cloned();
+    let text = res.bytes_limited().await?;
+    let object_id = extract_id(&text).ok();
+
+    match serde_json::from_slice(&text) {
+        Ok(object) => Ok(FetchObjectResponse {
+            object,
+            url,
+            content_type,
+            object_id,
+        }),
+        Err(e) => Err(ParseFetchedObject(
+            e,
+            url,
+            String::from_utf8(Vec::from(text))?,
+        )),
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use crate::{
+        config::FederationConfig,
+        traits::tests::{DbConnection, Person},
+    };
+
+    #[tokio::test]
+    async fn test_request_limit() -> Result<(), Error> {
+        let config = FederationConfig::builder()
+            .domain("example.com")
+            .app_data(DbConnection)
+            .http_fetch_limit(0)
+            .build()
+            .await
+            .unwrap();
+        let data = config.to_request_data();
+
+        let fetch_url = "https://example.net/".to_string();
+
+        let res: Result<FetchObjectResponse<Person>, Error> =
+            fetch_object_http(&Url::parse(&fetch_url).map_err(Error::UrlParse)?, &data).await;
+
+        assert_eq!(res.err(), Some(Error::RequestLimit));
+
+        Ok(())
+    }
 }

src/fetch/object_id.rs

@@ -1,5 +1,4 @@
 use crate::{config::Data, error::Error, fetch::fetch_object_http, traits::Object};
-use anyhow::anyhow;
 use chrono::{DateTime, Duration as ChronoDuration, Utc};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -11,7 +10,7 @@ use url::Url;
 
 impl<T> FromStr for ObjectId<T>
 where
-    T: Object + Send + Debug + 'static,
+    T: Object + Send + Sync + Debug + 'static,
     for<'de2> <T as Object>::Kind: Deserialize<'de2>,
 {
     type Err = url::ParseError;
@@ -58,20 +57,16 @@ where
 pub struct ObjectId<Kind>(Box<Url>, PhantomData<Kind>)
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>;
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>;
 
 impl<Kind> ObjectId<Kind>
 where
-    Kind: Object + Send + Debug + 'static,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    Kind: Object + Send + Sync + Debug + 'static,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     /// Construct a new objectid instance
-    pub fn parse<T>(url: T) -> Result<Self, url::ParseError>
-    where
-        T: TryInto<Url>,
-        url::ParseError: From<<T as TryInto<Url>>::Error>,
-    {
-        Ok(ObjectId(Box::new(url.try_into()?), PhantomData::<Kind>))
+    pub fn parse(url: &str) -> Result<Self, url::ParseError> {
+        Ok(Self(Box::new(Url::parse(url)?), PhantomData::<Kind>))
     }
 
     /// Returns a reference to the wrapped URL value
@@ -90,22 +85,16 @@ where
         data: &Data<<Kind as Object>::DataType>,
     ) -> Result<Kind, <Kind as Object>::Error>
     where
-        <Kind as Object>::Error: From<Error> + From<anyhow::Error>,
+        <Kind as Object>::Error: From<Error>,
     {
         let db_object = self.dereference_from_db(data).await?;
-        // if its a local object, only fetch it from the database and not over http
-        if data.config.is_local_url(&self.0) {
-            return match db_object {
-                None => Err(Error::NotFound.into()),
-                Some(o) => Ok(o),
-            };
-        }
 
         // object found in database
         if let Some(object) = db_object {
-            // object is old and should be refetched
             if let Some(last_refreshed_at) = object.last_refreshed_at() {
-                if should_refetch_object(last_refreshed_at) {
+                let is_local = self.is_local(data);
+                if !is_local && should_refetch_object(last_refreshed_at) {
+                    // object is outdated and should be refetched
                     return self.dereference_from_http(data, Some(object)).await;
                 }
             }
@@ -117,6 +106,25 @@ where
         }
     }
 
+    /// If this is a remote object, fetch it from origin instance unconditionally to get the
+    /// latest version, regardless of refresh interval.
+    pub async fn dereference_forced(
+        &self,
+        data: &Data<<Kind as Object>::DataType>,
+    ) -> Result<Kind, <Kind as Object>::Error>
+    where
+        <Kind as Object>::Error: From<Error>,
+    {
+        if data.config.is_local_url(&self.0) {
+            self.dereference_from_db(data)
+                .await
+                .map(|o| o.ok_or(Error::NotFound.into()))?
+        } else {
+            // Don't pass in any db object, otherwise it would be returned in case http fetch fails
+            self.dereference_from_http(data, None).await
+        }
+    }
+
     /// Fetch an object from the local db. Instead of falling back to http, this throws an error if
     /// the object is not found in the database.
     pub async fn dereference_local(
@@ -139,27 +147,50 @@ where
         Object::read_from_id(*id, data).await
     }
 
+    /// Fetch object from origin instance over HTTP, then verify and parse it.
+    ///
+    /// Uses Box::pin to wrap futures to reduce stack size and avoid stack overflow when
+    /// when fetching objects recursively.
     async fn dereference_from_http(
         &self,
         data: &Data<<Kind as Object>::DataType>,
         db_object: Option<Kind>,
     ) -> Result<Kind, <Kind as Object>::Error>
     where
-        <Kind as Object>::Error: From<Error> + From<anyhow::Error>,
+        <Kind as Object>::Error: From<Error>,
     {
-        let res = fetch_object_http(&self.0, data).await;
+        let res = Box::pin(fetch_object_http(&self.0, data)).await;
 
-        if let Err(Error::ObjectDeleted) = &res {
+        if let Err(Error::ObjectDeleted(url)) = res {
             if let Some(db_object) = db_object {
                 db_object.delete(data).await?;
+                return Ok(db_object);
             }
-            return Err(anyhow!("Fetched remote object {} which was deleted", self).into());
+            return Err(Error::ObjectDeleted(url).into());
         }
 
-        let res2 = res?;
+        // If fetch failed, return the existing object from local database
+        if let (Err(_), Some(db_object)) = (&res, db_object) {
+            return Ok(db_object);
+        }
+        let res = res?;
+        let redirect_url = &res.url;
 
-        Kind::verify(&res2, self.inner(), data).await?;
-        Kind::from_json(res2, data).await
+        // Prevent overwriting local object
+        if data.config.is_local_url(redirect_url) {
+            return self
+                .dereference_from_db(data)
+                .await?
+                .ok_or(Error::NotFound.into());
+        }
+
+        Box::pin(Kind::verify(&res.object, redirect_url, data)).await?;
+        Box::pin(Kind::from_json(res.object, data)).await
+    }
+
+    /// Returns true if the object's domain matches the one defined in [[FederationConfig.domain]].
+    pub fn is_local(&self, data: &Data<<Kind as Object>::DataType>) -> bool {
+        data.config.is_local_url(&self.0)
     }
 }
 
@@ -167,7 +198,7 @@ where
 impl<Kind> Clone for ObjectId<Kind>
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn clone(&self) -> Self {
         ObjectId(self.0.clone(), self.1)
@@ -183,9 +214,9 @@ static ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG: i64 = 20;
 fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
     let update_interval = if cfg!(debug_assertions) {
         // avoid infinite loop when fetching community outbox
-        ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG)
+        ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG).expect("valid duration")
     } else {
-        ChronoDuration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS)
+        ChronoDuration::try_seconds(ACTOR_REFETCH_INTERVAL_SECONDS).expect("valid duration")
     };
     let refresh_limit = Utc::now() - update_interval;
     last_refreshed.lt(&refresh_limit)
@@ -194,7 +225,7 @@ fn should_refetch_object(last_refreshed: DateTime<Utc>) -> bool {
 impl<Kind> Display for ObjectId<Kind>
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.0.as_str())
@@ -204,7 +235,7 @@ where
 impl<Kind> Debug for ObjectId<Kind>
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.0.as_str())
@@ -214,7 +245,7 @@ where
 impl<Kind> From<ObjectId<Kind>> for Url
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn from(id: ObjectId<Kind>) -> Self {
         *id.0
@@ -224,7 +255,7 @@ where
 impl<Kind> From<Url> for ObjectId<Kind>
 where
     Kind: Object + Send + 'static,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn from(url: Url) -> Self {
         ObjectId(Box::new(url), PhantomData::<Kind>)
@@ -234,17 +265,19 @@ where
 impl<Kind> PartialEq for ObjectId<Kind>
 where
     Kind: Object,
-    for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
 {
     fn eq(&self, other: &Self) -> bool {
         self.0.eq(&other.0) && self.1 == other.1
     }
 }
 
+/// Internal only
 #[cfg(test)]
+#[allow(clippy::unwrap_used)]
 pub mod tests {
     use super::*;
-    use crate::{fetch::object_id::should_refetch_object, traits::tests::DbUser};
+    use crate::traits::tests::DbUser;
 
     #[test]
     fn test_deserialize() {
@@ -259,10 +292,10 @@ pub mod tests {
 
     #[test]
     fn test_should_refetch_object() {
-        let one_second_ago = Utc::now() - ChronoDuration::seconds(1);
+        let one_second_ago = Utc::now() - ChronoDuration::try_seconds(1).unwrap();
         assert!(!should_refetch_object(one_second_ago));
 
-        let two_days_ago = Utc::now() - ChronoDuration::days(2);
+        let two_days_ago = Utc::now() - ChronoDuration::try_days(2).unwrap();
         assert!(should_refetch_object(two_days_ago));
     }
 }

src/fetch/webfinger.rs

@@ -1,18 +1,41 @@
 use crate::{
-    config::Data,
-    error::{Error, Error::WebfingerResolveFailed},
+    config::{domain_regex, Data},
+    error::Error,
     fetch::{fetch_object_http_with_accept, object_id::ObjectId},
     traits::{Actor, Object},
     FEDERATION_CONTENT_TYPE,
 };
-use anyhow::anyhow;
+use http::HeaderValue;
 use itertools::Itertools;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::{collections::HashMap, fmt::Display, sync::LazyLock};
 use tracing::debug;
 use url::Url;
 
+/// Errors relative to webfinger handling
+#[derive(thiserror::Error, Debug)]
+pub enum WebFingerError {
+    /// The webfinger identifier is invalid
+    #[error("The webfinger identifier is invalid")]
+    WrongFormat,
+    /// The webfinger identifier doesn't match the expected instance domain name
+    #[error("The webfinger identifier doesn't match the expected instance domain name")]
+    WrongDomain,
+    /// The wefinger object did not contain any link to an activitypub item
+    #[error("The webfinger object did not contain any link to an activitypub item")]
+    NoValidLink,
+}
+
+impl WebFingerError {
+    fn into_crate_error(self) -> Error {
+        self.into()
+    }
+}
+
+/// The content-type for webfinger responses.
+pub static WEBFINGER_CONTENT_TYPE: HeaderValue = HeaderValue::from_static("application/jrd+json");
+
 /// Takes an identifier of the form `name@example.com`, and returns an object of `Kind`.
 ///
 /// For this the identifier is first resolved via webfinger protocol to an Activitypub ID. This ID
@@ -22,26 +45,39 @@ pub async fn webfinger_resolve_actor<T: Clone, Kind>(
     data: &Data<T>,
 ) -> Result<Kind, <Kind as Object>::Error>
 where
-    Kind: Object + Actor + Send + 'static + Object<DataType = T>,
+    Kind: Object + Actor + Send + Sync + 'static + Object<DataType = T>,
     for<'de2> <Kind as Object>::Kind: serde::Deserialize<'de2>,
-    <Kind as Object>::Error:
-        From<crate::error::Error> + From<anyhow::Error> + From<url::ParseError> + Send + Sync,
+    <Kind as Object>::Error: From<crate::error::Error> + Send + Sync + Display,
 {
     let (_, domain) = identifier
         .splitn(2, '@')
         .collect_tuple()
-        .ok_or(WebfingerResolveFailed)?;
+        .ok_or(WebFingerError::WrongFormat.into_crate_error())?;
+
+    // For production mode make sure that domain doesnt contain any port or path.
+    if !data.config.debug && !domain_regex().is_match(domain) {
+        return Err(Error::UrlVerificationError("Invalid characters in domain").into());
+    }
+
     let protocol = if data.config.debug { "http" } else { "https" };
     let fetch_url =
         format!("{protocol}://{domain}/.well-known/webfinger?resource=acct:{identifier}");
     debug!("Fetching webfinger url: {}", &fetch_url);
 
-    let res: Webfinger =
-        fetch_object_http_with_accept(&Url::parse(&fetch_url)?, data, "application/jrd+json")
-            .await?;
+    let res = fetch_object_http_with_accept::<_, Webfinger>(
+        &Url::parse(&fetch_url).map_err(Error::UrlParse)?,
+        data,
+        &WEBFINGER_CONTENT_TYPE,
+        false,
+    )
+    .await?;
+    if res.url.as_str() != fetch_url {
+        data.config.verify_url_valid(&res.url).await?;
+    }
 
-    debug_assert_eq!(res.subject, format!("acct:{identifier}"));
+    debug_assert_eq!(res.object.subject, format!("acct:{identifier}"));
     let links: Vec<Url> = res
+        .object
         .links
         .iter()
         .filter(|link| {
@@ -52,14 +88,17 @@ where
             }
         })
         .filter_map(|l| l.href.clone())
+        .rev()
         .collect();
+
     for l in links {
         let object = ObjectId::<Kind>::from(l).dereference(data).await;
-        if object.is_ok() {
-            return object;
+        match object {
+            Ok(obj) => return Ok(obj),
+            Err(error) => debug!(%error, "Failed to dereference link"),
         }
     }
-    Err(WebfingerResolveFailed.into())
+    Err(WebFingerError::NoValidLink.into_crate_error().into())
 }
 
 /// Extracts username from a webfinger resource parameter.
@@ -87,24 +126,24 @@ where
 /// # Ok::<(), anyhow::Error>(())
 /// }).unwrap();
 ///```
-pub fn extract_webfinger_name<T>(query: &str, data: &Data<T>) -> Result<String, Error>
+pub fn extract_webfinger_name<'i, T>(query: &'i str, data: &Data<T>) -> Result<&'i str, Error>
 where
     T: Clone,
 {
-    // TODO: would be nice if we could implement this without regex and remove the dependency
-    // Regex taken from Mastodon -
-    // https://github.com/mastodon/mastodon/blob/2b113764117c9ab98875141bcf1758ba8be58173/app/models/account.rb#L65
-    let regex = Regex::new(&format!(
-        "^acct:((?i)[a-z0-9_]+([a-z0-9_\\.-]+[a-z0-9_]+)?)@{}$",
-        data.domain()
-    ))
-    .map_err(Error::other)?;
-    Ok(regex
+    static WEBFINGER_REGEX: LazyLock<Regex> =
+        LazyLock::new(|| Regex::new(r"^acct:([\p{L}0-9_\.\-]+)@(.*)$").expect("compile regex"));
+    // Regex to extract usernames from webfinger query. Supports different alphabets using `\p{L}`.
+    // TODO: This should use a URL parser
+    let captures = WEBFINGER_REGEX
         .captures(query)
-        .and_then(|c| c.get(1))
-        .ok_or_else(|| Error::other(anyhow!("Webfinger regex failed to match")))?
-        .as_str()
-        .to_string())
+        .ok_or(WebFingerError::WrongFormat)?;
+
+    let account_name = captures.get(1).ok_or(WebFingerError::WrongFormat)?;
+
+    if captures.get(2).map(|m| m.as_str()) != Some(data.domain()) {
+        return Err(WebFingerError::WrongDomain.into());
+    }
+    Ok(account_name.as_str())
 }
 
 /// Builds a basic webfinger response for the actor.
@@ -184,7 +223,7 @@ pub fn build_webfinger_response_with_type(
 }
 
 /// A webfinger response with information about a `Person` or other type of actor.
-#[derive(Serialize, Deserialize, Debug, Default)]
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
 pub struct Webfinger {
     /// The actor which is described here, for example `acct:LemmyDev@mastodon.social`
     pub subject: String,
@@ -199,7 +238,7 @@ pub struct Webfinger {
 }
 
 /// A single link included as part of a [Webfinger] response.
-#[derive(Serialize, Deserialize, Debug, Default)]
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
 pub struct WebfingerLink {
     /// Relationship of the link, such as `self` or `http://webfinger.net/rel/profile-page`
     pub rel: Option<String>,
@@ -216,6 +255,7 @@ pub struct WebfingerLink {
 }
 
 #[cfg(test)]
+#[allow(clippy::unwrap_used)]
 mod tests {
     use super::*;
     use crate::{
@@ -234,8 +274,41 @@ mod tests {
         let data = config.to_request_data();
 
         webfinger_resolve_actor::<DbConnection, DbUser>("LemmyDev@mastodon.social", &data).await?;
-        // poa.st is as of 2023-07-14 the largest Pleroma instance
-        webfinger_resolve_actor::<DbConnection, DbUser>("graf@poa.st", &data).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_webfinger_extract_name() -> Result<(), Error> {
+        use crate::traits::tests::DbConnection;
+        let data = Data {
+            config: FederationConfig::builder()
+                .domain("example.com")
+                .app_data(DbConnection)
+                .build()
+                .await
+                .unwrap(),
+            request_counter: Default::default(),
+        };
+        assert_eq!(
+            Ok("test123"),
+            extract_webfinger_name("acct:test123@example.com", &data)
+        );
+        assert_eq!(
+            Ok("Владимир"),
+            extract_webfinger_name("acct:Владимир@example.com", &data)
+        );
+        assert_eq!(
+            Ok("example.com"),
+            extract_webfinger_name("acct:example.com@example.com", &data)
+        );
+        assert_eq!(
+            Ok("da-sh"),
+            extract_webfinger_name("acct:da-sh@example.com", &data)
+        );
+        assert_eq!(
+            Ok("تجريب"),
+            extract_webfinger_name("acct:تجريب@example.com", &data)
+        );
         Ok(())
     }
 }

src/http_signatures.rs

@@ -1,7 +1,7 @@
 //! Generating keypairs, creating and verifying signatures
 //!
 //! Signature creation and verification is handled internally in the library. See
-//! [send_activity](crate::activity_sending::send_activity) and
+//! [send_activity](crate::activity_sending::SendActivityTask::sign_and_send) and
 //! [receive_activity (actix-web)](crate::actix_web::inbox::receive_activity) /
 //! [receive_activity (axum)](crate::axum::inbox::receive_activity).
 
@@ -12,23 +12,24 @@ use crate::{
     protocol::public_key::main_key_id,
     traits::{Actor, Object},
 };
-use anyhow::Context;
 use base64::{engine::general_purpose::STANDARD as Base64, Engine};
 use bytes::Bytes;
 use http::{header::HeaderName, uri::PathAndQuery, HeaderValue, Method, Uri};
-use http_signature_normalization_reqwest::prelude::{Config, SignExt};
-use once_cell::sync::Lazy;
-use openssl::{
-    hash::MessageDigest,
-    pkey::{PKey, Private},
-    rsa::Rsa,
-    sign::{Signer, Verifier},
+use http_signature_normalization_reqwest::{
+    prelude::{Config, SignExt},
+    DefaultSpawner,
 };
 use reqwest::Request;
 use reqwest_middleware::RequestBuilder;
+use rsa::{
+    pkcs8::{DecodePublicKey, EncodePrivateKey, EncodePublicKey, LineEnding},
+    Pkcs1v15Sign,
+    RsaPrivateKey,
+    RsaPublicKey,
+};
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
-use std::{collections::BTreeMap, fmt::Debug, io::ErrorKind, time::Duration};
+use std::{collections::BTreeMap, fmt::Debug, sync::LazyLock, time::Duration};
 use tracing::debug;
 use url::Url;
 
@@ -44,27 +45,27 @@ pub struct Keypair {
 impl Keypair {
     /// Helper method to turn this into an openssl private key
     #[cfg(test)]
-    pub(crate) fn private_key(&self) -> Result<PKey<Private>, anyhow::Error> {
-        Ok(PKey::private_key_from_pem(self.private_key.as_bytes())?)
+    pub(crate) fn private_key(&self) -> Result<RsaPrivateKey, anyhow::Error> {
+        use rsa::pkcs8::DecodePrivateKey;
+
+        Ok(RsaPrivateKey::from_pkcs8_pem(&self.private_key)?)
     }
 }
 
 /// Generate a random asymmetric keypair for ActivityPub HTTP signatures.
-pub fn generate_actor_keypair() -> Result<Keypair, std::io::Error> {
-    let rsa = Rsa::generate(2048)?;
-    let pkey = PKey::from_rsa(rsa)?;
-    let public_key = pkey.public_key_to_pem()?;
-    let private_key = pkey.private_key_to_pem_pkcs8()?;
-    let key_to_string = |key| match String::from_utf8(key) {
-        Ok(s) => Ok(s),
-        Err(e) => Err(std::io::Error::new(
-            ErrorKind::Other,
-            format!("Failed converting key to string: {}", e),
-        )),
-    };
+///
+/// Note that this method is very slow in debug mode. To make it faster, follow
+/// instructions in the RSA crate's readme.
+/// <https://github.com/RustCrypto/RSA/blob/master/README.md>
+pub fn generate_actor_keypair() -> Result<Keypair, Error> {
+    let mut rng = rand::thread_rng();
+    let rsa = RsaPrivateKey::new(&mut rng, 2048)?;
+    let pkey = RsaPublicKey::from(&rsa);
+    let public_key = pkey.to_public_key_pem(LineEnding::default())?;
+    let private_key = rsa.to_pkcs8_pem(LineEnding::default())?.to_string();
     Ok(Keypair {
-        private_key: key_to_string(private_key)?,
-        public_key: key_to_string(public_key)?,
+        private_key,
+        public_key,
     })
 }
 
@@ -81,11 +82,12 @@ pub(crate) async fn sign_request(
     request_builder: RequestBuilder,
     actor_id: &Url,
     activity: Bytes,
-    private_key: PKey<Private>,
+    private_key: RsaPrivateKey,
     http_signature_compat: bool,
-) -> Result<Request, anyhow::Error> {
-    static CONFIG: Lazy<Config> = Lazy::new(|| Config::new().set_expiration(EXPIRES_AFTER));
-    static CONFIG_COMPAT: Lazy<Config> = Lazy::new(|| {
+) -> Result<Request, Error> {
+    static CONFIG: LazyLock<Config<DefaultSpawner>> =
+        LazyLock::new(|| Config::new().set_expiration(EXPIRES_AFTER));
+    static CONFIG_COMPAT: LazyLock<Config> = LazyLock::new(|| {
         Config::new()
             .mastodon_compat()
             .set_expiration(EXPIRES_AFTER)
@@ -103,14 +105,10 @@ pub(crate) async fn sign_request(
             Sha256::new(),
             activity,
             move |signing_string| {
-                let mut signer = Signer::new(MessageDigest::sha256(), &private_key)
-                    .context("instantiating signer")?;
-                signer
-                    .update(signing_string.as_bytes())
-                    .context("updating signer")?;
-
-                Ok(Base64.encode(signer.sign_to_vec().context("sign to vec")?))
-                    as Result<_, anyhow::Error>
+                Ok(Base64.encode(private_key.sign(
+                    Pkcs1v15Sign::new::<Sha256>(),
+                    &Sha256::digest(signing_string.as_bytes()),
+                )?)) as Result<_, Error>
             },
         )
         .await
@@ -151,8 +149,8 @@ pub(crate) async fn signing_actor<'a, A, H>(
     data: &Data<<A as Object>::DataType>,
 ) -> Result<A, <A as Object>::Error>
 where
-    A: Object + Actor,
-    <A as Object>::Error: From<Error> + From<anyhow::Error>,
+    A: Object + Actor + Send + Sync,
+    <A as Object>::Error: From<Error>,
     for<'de2> <A as Object>::Kind: Deserialize<'de2>,
     H: IntoIterator<Item = (&'a HeaderName, &'a HeaderValue)>,
 {
@@ -190,25 +188,36 @@ fn verify_signature_inner(
     uri: &Uri,
     public_key: &str,
 ) -> Result<(), Error> {
-    static CONFIG: Lazy<http_signature_normalization::Config> =
-        Lazy::new(|| http_signature_normalization::Config::new().set_expiration(EXPIRES_AFTER));
+    static CONFIG: LazyLock<http_signature_normalization::Config> = LazyLock::new(|| {
+        http_signature_normalization::Config::new()
+            .set_expiration(EXPIRES_AFTER)
+            .require_digest()
+    });
 
     let path_and_query = uri.path_and_query().map(PathAndQuery::as_str).unwrap_or("");
 
     let verified = CONFIG
         .begin_verify(method.as_str(), path_and_query, header_map)
-        .map_err(Error::other)?
-        .verify(|signature, signing_string| -> anyhow::Result<bool> {
+        .map_err(|val| Error::Other(val.to_string()))?
+        .verify(|signature, signing_string| -> Result<bool, Error> {
             debug!(
                 "Verifying with key {}, message {}",
                 &public_key, &signing_string
             );
-            let public_key = PKey::public_key_from_pem(public_key.as_bytes())?;
-            let mut verifier = Verifier::new(MessageDigest::sha256(), &public_key)?;
-            verifier.update(signing_string.as_bytes())?;
-            Ok(verifier.verify(&Base64.decode(signature)?)?)
-        })
-        .map_err(Error::other)?;
+            let public_key = RsaPublicKey::from_public_key_pem(public_key)?;
+
+            let base64_decoded = Base64
+                .decode(signature)
+                .map_err(|err| Error::Other(err.to_string()))?;
+
+            Ok(public_key
+                .verify(
+                    Pkcs1v15Sign::new::<Sha256>(),
+                    &Sha256::digest(signing_string.as_bytes()),
+                    &base64_decoded,
+                )
+                .is_ok())
+        })?;
 
     if verified {
         debug!("verified signature for {}", uri);
@@ -271,17 +280,21 @@ pub(crate) fn verify_body_hash(
     Ok(())
 }
 
+/// Internal only
 #[cfg(test)]
+#[allow(clippy::unwrap_used)]
 pub mod test {
     use super::*;
     use crate::activity_sending::generate_request_headers;
     use reqwest::Client;
     use reqwest_middleware::ClientWithMiddleware;
+    use rsa::{pkcs1::DecodeRsaPrivateKey, pkcs8::DecodePrivateKey};
     use std::str::FromStr;
 
-    static ACTOR_ID: Lazy<Url> = Lazy::new(|| Url::parse("https://example.com/u/alice").unwrap());
-    static INBOX_URL: Lazy<Url> =
-        Lazy::new(|| Url::parse("https://example.com/u/alice/inbox").unwrap());
+    static ACTOR_ID: LazyLock<Url> =
+        LazyLock::new(|| Url::parse("https://example.com/u/alice").unwrap());
+    static INBOX_URL: LazyLock<Url> =
+        LazyLock::new(|| Url::parse("https://example.com/u/alice/inbox").unwrap());
 
     #[tokio::test]
     async fn test_sign() {
@@ -299,7 +312,7 @@ pub mod test {
             request_builder,
             &ACTOR_ID,
             "my activity".into(),
-            PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
+            RsaPrivateKey::from_pkcs8_pem(&test_keypair().private_key).unwrap(),
             // set this to prevent created/expires headers to be generated and inserted
             // automatically from current time
             true,
@@ -335,7 +348,7 @@ pub mod test {
             request_builder,
             &ACTOR_ID,
             "my activity".to_string().into(),
-            PKey::private_key_from_pem(test_keypair().private_key.as_bytes()).unwrap(),
+            RsaPrivateKey::from_pkcs8_pem(&test_keypair().private_key).unwrap(),
             false,
         )
         .await
@@ -370,14 +383,15 @@ pub mod test {
         assert_eq!(invalid, Err(Error::ActivityBodyDigestInvalid));
     }
 
+    /// Internal only, return hardcoded keypair for testing
     pub fn test_keypair() -> Keypair {
-        let rsa = Rsa::private_key_from_pem(PRIVATE_KEY.as_bytes()).unwrap();
-        let pkey = PKey::from_rsa(rsa).unwrap();
-        let private_key = pkey.private_key_to_pem_pkcs8().unwrap();
-        let public_key = pkey.public_key_to_pem().unwrap();
+        let rsa = RsaPrivateKey::from_pkcs1_pem(PRIVATE_KEY).unwrap();
+        let pkey = RsaPublicKey::from(&rsa);
+        let public_key = pkey.to_public_key_pem(LineEnding::default()).unwrap();
+        let private_key = rsa.to_pkcs8_pem(LineEnding::default()).unwrap().to_string();
         Keypair {
-            private_key: String::from_utf8(private_key).unwrap(),
-            public_key: String::from_utf8(public_key).unwrap(),
+            private_key,
+            public_key,
         }
     }
 

src/lib.rs

@@ -10,6 +10,7 @@
 #![doc = include_str!("../docs/10_fetching_objects_with_unknown_type.md")]
 #![deny(missing_docs)]
 
+pub mod activity_queue;
 pub mod activity_sending;
 #[cfg(feature = "actix-web")]
 pub mod actix_web;
@@ -22,8 +23,53 @@ pub mod http_signatures;
 pub mod protocol;
 pub(crate) mod reqwest_shim;
 pub mod traits;
+mod utils;
 
+use crate::{
+    config::Data,
+    error::Error,
+    fetch::object_id::ObjectId,
+    traits::{Activity, Actor, Object},
+};
 pub use activitystreams_kinds as kinds;
 
+use serde::{de::DeserializeOwned, Deserialize};
+use url::Url;
+
 /// Mime type for Activitypub data, used for `Accept` and `Content-Type` HTTP headers
-pub static FEDERATION_CONTENT_TYPE: &str = "application/activity+json";
+pub const FEDERATION_CONTENT_TYPE: &str = "application/activity+json";
+
+/// Deserialize incoming inbox activity to the given type, perform basic
+/// validation and extract the actor.
+async fn parse_received_activity<A, ActorT, Datatype>(
+    body: &[u8],
+    data: &Data<Datatype>,
+) -> Result<(A, ActorT), <A as Activity>::Error>
+where
+    A: Activity<DataType = Datatype> + DeserializeOwned + Send + 'static,
+    ActorT: Object<DataType = Datatype> + Actor + Send + Sync + 'static,
+    for<'de2> <ActorT as Object>::Kind: serde::Deserialize<'de2>,
+    <A as Activity>::Error: From<Error> + From<<ActorT as Object>::Error>,
+    <ActorT as Object>::Error: From<Error>,
+    Datatype: Clone,
+{
+    let activity: A = serde_json::from_slice(body).map_err(|err| {
+        // Attempt to include activity id in error message
+        let id = extract_id(body).ok();
+        Error::ParseReceivedActivity { err, id }
+    })?;
+    data.config.verify_url_and_domain(&activity).await?;
+    let actor = ObjectId::<ActorT>::from(activity.actor().clone())
+        .dereference(data)
+        .await?;
+    Ok((activity, actor))
+}
+
+/// Attempt to parse id field from serialized json
+fn extract_id(data: &[u8]) -> serde_json::Result<Url> {
+    #[derive(Deserialize)]
+    struct Id {
+        id: Url,
+    }
+    Ok(serde_json::from_slice::<Id>(data)?.id)
+}

src/protocol/context.rs

@@ -15,11 +15,11 @@
 //! };
 //! let note_with_context = WithContext::new_default(note);
 //! let serialized = serde_json::to_string(&note_with_context)?;
-//! assert_eq!(serialized, r#"{"@context":["https://www.w3.org/ns/activitystreams"],"content":"Hello world"}"#);
+//! assert_eq!(serialized, r#"{"@context":"https://www.w3.org/ns/activitystreams","content":"Hello world"}"#);
 //! Ok::<(), serde_json::error::Error>(())
 //! ```
 
-use crate::{config::Data, protocol::helpers::deserialize_one_or_many, traits::ActivityHandler};
+use crate::{config::Data, traits::Activity};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use url::Url;
@@ -31,8 +31,7 @@ const DEFAULT_CONTEXT: &str = "https://www.w3.org/ns/activitystreams";
 #[derive(Serialize, Deserialize, Debug)]
 pub struct WithContext<T> {
     #[serde(rename = "@context")]
-    #[serde(deserialize_with = "deserialize_one_or_many")]
-    context: Vec<Value>,
+    context: Value,
     #[serde(flatten)]
     inner: T,
 }
@@ -40,12 +39,12 @@ pub struct WithContext<T> {
 impl<T> WithContext<T> {
     /// Create a new wrapper with the default Activitypub context.
     pub fn new_default(inner: T) -> WithContext<T> {
-        let context = vec![Value::String(DEFAULT_CONTEXT.to_string())];
+        let context = Value::String(DEFAULT_CONTEXT.to_string());
         WithContext::new(inner, context)
     }
 
     /// Create new wrapper with custom context. Use this in case you are implementing extensions.
-    pub fn new(inner: T, context: Vec<Value>) -> WithContext<T> {
+    pub fn new(inner: T, context: Value) -> WithContext<T> {
         WithContext { context, inner }
     }
 
@@ -56,12 +55,12 @@ impl<T> WithContext<T> {
 }
 
 #[async_trait::async_trait]
-impl<T> ActivityHandler for WithContext<T>
+impl<T> Activity for WithContext<T>
 where
-    T: ActivityHandler + Send + Sync,
+    T: Activity + Send + Sync,
 {
-    type DataType = <T as ActivityHandler>::DataType;
-    type Error = <T as ActivityHandler>::Error;
+    type DataType = <T as Activity>::DataType;
+    type Error = <T as Activity>::Error;
 
     fn id(&self) -> &Url {
         self.inner.id()

src/protocol/helpers.rs

@@ -1,13 +1,22 @@
 //! Serde deserialization functions which help to receive differently shaped data
 
-use serde::{Deserialize, Deserializer};
+use activitystreams_kinds::public;
+use itertools::Itertools;
+use serde::{de::Error, Deserialize, Deserializer};
+use serde_json::Value;
+use url::Url;
 
-/// Deserialize JSON single value or array into Vec.
+/// Deserialize JSON single value or array into `Vec<Url>`.
 ///
 /// Useful if your application can handle multiple values for a field, but another federated
 /// platform only sends a single one.
 ///
+/// Also accepts common `Public` aliases for recipient fields. Some implementations send `Public`
+/// or `as:Public` instead of the canonical `https://www.w3.org/ns/activitystreams#Public` URL
+/// in fields such as `to` and `cc`.
+///
 /// ```
+/// # use activitypub_federation::kinds::public;
 /// # use activitypub_federation::protocol::helpers::deserialize_one_or_many;
 /// # use url::Url;
 /// #[derive(serde::Deserialize)]
@@ -25,24 +34,39 @@ use serde::{Deserialize, Deserializer};
 ///      "https://lemmy.ml/u/bob"
 /// ]}"#)?;
 /// assert_eq!(multiple.to.len(), 2);
-/// Ok::<(), anyhow::Error>(())
-pub fn deserialize_one_or_many<'de, T, D>(deserializer: D) -> Result<Vec<T>, D::Error>
+///
+/// let note: Note = serde_json::from_str(r#"{"to": ["Public", "as:Public"]}"#)?;
+/// assert_eq!(note.to, vec![public()]);
+/// # Ok::<(), anyhow::Error>(())
+/// ```
+pub fn deserialize_one_or_many<'de, D>(deserializer: D) -> Result<Vec<Url>, D::Error>
 where
-    T: Deserialize<'de>,
     D: Deserializer<'de>,
 {
     #[derive(Deserialize)]
     #[serde(untagged)]
-    enum OneOrMany<T> {
-        One(T),
-        Many(Vec<T>),
+    enum OneOrMany {
+        Many(Vec<Value>),
+        One(Value),
     }
 
-    let result: OneOrMany<T> = Deserialize::deserialize(deserializer)?;
-    Ok(match result {
-        OneOrMany::Many(list) => list,
+    let result: OneOrMany = Deserialize::deserialize(deserializer)?;
+    let values = match result {
         OneOrMany::One(value) => vec![value],
-    })
+        OneOrMany::Many(values) => values,
+    };
+
+    values
+        .into_iter()
+        .map(|value| match value {
+            Value::String(value) if matches!(value.as_str(), "Public" | "as:Public") => {
+                Ok(public())
+            }
+            Value::String(value) => Url::parse(&value).map_err(D::Error::custom),
+            value => Url::deserialize(value).map_err(D::Error::custom),
+        })
+        .collect::<Result<Vec<_>, _>>()
+        .map(|values| values.into_iter().unique().collect())
 }
 
 /// Deserialize JSON single value or single element array into single value.
@@ -56,12 +80,12 @@ where
 /// #[derive(serde::Deserialize)]
 /// struct Note {
 ///     #[serde(deserialize_with = "deserialize_one")]
-///     to: Url
+///     to: [Url; 1]
 /// }
 ///
 /// let note = serde_json::from_str::<Note>(r#"{"to": ["https://example.com/u/alice"] }"#);
 /// assert!(note.is_ok());
-pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<T, D::Error>
+pub fn deserialize_one<'de, T, D>(deserializer: D) -> Result<[T; 1], D::Error>
 where
     T: Deserialize<'de>,
     D: Deserializer<'de>,
@@ -75,8 +99,8 @@ where
 
     let result: MaybeArray<T> = Deserialize::deserialize(deserializer)?;
     Ok(match result {
-        MaybeArray::Simple(value) => value,
-        MaybeArray::Array([value]) => value,
+        MaybeArray::Simple(value) => [value],
+        MaybeArray::Array([value]) => [value],
     })
 }
 
@@ -116,8 +140,35 @@ where
     Ok(inner)
 }
 
+/// Deserialize either single value or last item from an array into an optional field.
+pub fn deserialize_last<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
+where
+    T: Deserialize<'de>,
+    D: Deserializer<'de>,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum MaybeArray<T> {
+        Simple(T),
+        Array(Vec<T>),
+        None,
+    }
+
+    let result = Deserialize::deserialize(deserializer)?;
+    Ok(match result {
+        MaybeArray::Simple(value) => Some(value),
+        MaybeArray::Array(value) => value.into_iter().last(),
+        MaybeArray::None => None,
+    })
+}
+
 #[cfg(test)]
 mod tests {
+    use super::deserialize_one_or_many;
+    use activitystreams_kinds::public;
+    use anyhow::Result;
+    use serde::Deserialize;
+
     #[test]
     fn deserialize_one_multiple_values() {
         use crate::protocol::helpers::deserialize_one;
@@ -125,7 +176,7 @@ mod tests {
         #[derive(serde::Deserialize)]
         struct Note {
             #[serde(deserialize_with = "deserialize_one")]
-            _to: Url,
+            _to: [Url; 1],
         }
 
         let note = serde_json::from_str::<Note>(
@@ -133,4 +184,70 @@ mod tests {
         );
         assert!(note.is_err());
     }
+
+    #[test]
+    fn deserialize_one_or_many_single_public_aliases() -> Result<()> {
+        use url::Url;
+
+        #[derive(Deserialize)]
+        struct Note {
+            #[serde(deserialize_with = "deserialize_one_or_many")]
+            to: Vec<Url>,
+        }
+
+        for alias in ["Public", "as:Public"] {
+            let note = serde_json::from_str::<Note>(&format!(r#"{{"to": "{alias}"}}"#))?;
+            assert_eq!(note.to, vec![public()]);
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn deserialize_one_or_many_array() -> Result<()> {
+        use url::Url;
+
+        #[derive(Deserialize)]
+        struct Note {
+            #[serde(deserialize_with = "deserialize_one_or_many")]
+            to: Vec<Url>,
+        }
+
+        let note = serde_json::from_str::<Note>(
+            r#"{
+                "to": [
+                    "https://example.com/c/main",
+                    "Public",
+                    "as:Public",
+                    "https://www.w3.org/ns/activitystreams#Public"
+                ]
+            }"#,
+        )?;
+
+        assert_eq!(
+            note.to,
+            vec![Url::parse("https://example.com/c/main")?, public(),]
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn deserialize_one_or_many_leaves_other_strings_unchanged() -> Result<()> {
+        use url::Url;
+
+        #[derive(Deserialize)]
+        struct Note {
+            #[serde(deserialize_with = "deserialize_one_or_many")]
+            to: Vec<Url>,
+            content: String,
+        }
+
+        let note = serde_json::from_str::<Note>(r#"{"to": "Public", "content": "Public"}"#)?;
+
+        assert_eq!(note.to, vec![public()]);
+        assert_eq!(note.content, "Public");
+
+        Ok(())
+    }
 }

src/protocol/mod.rs

@@ -3,5 +3,6 @@
 pub mod context;
 pub mod helpers;
 pub mod public_key;
+pub mod tombstone;
 pub mod values;
 pub mod verification;

src/protocol/public_key.rs

@@ -6,7 +6,7 @@ use url::Url;
 /// Public key of actors which is used for HTTP signatures.
 ///
 /// This needs to be federated in the `public_key` field of all actors.
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct PublicKey {
     /// Id of this private key.

src/protocol/tombstone.rs

@@ -0,0 +1,27 @@
+//! Tombstone is used to serve deleted objects
+
+use crate::kinds::object::TombstoneType;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+/// Represents a local object that was deleted
+///
+/// <https://www.w3.org/TR/activitystreams-vocabulary/#dfn-tombstone>
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Tombstone {
+    /// Id of the deleted object
+    pub id: Url,
+    #[serde(rename = "type")]
+    pub(crate) kind: TombstoneType,
+}
+
+impl Tombstone {
+    /// Create a new tombstone for the given object id
+    pub fn new(id: Url) -> Tombstone {
+        Tombstone {
+            id,
+            kind: TombstoneType::Tombstone,
+        }
+    }
+}

src/protocol/values.rs

@@ -35,7 +35,7 @@ use serde::{Deserialize, Serialize};
 /// Media type for markdown text.
 ///
 /// <https://www.iana.org/assignments/media-types/media-types.xhtml>
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
 pub enum MediaTypeMarkdown {
     /// `text/markdown`
     #[serde(rename = "text/markdown")]
@@ -45,7 +45,7 @@ pub enum MediaTypeMarkdown {
 /// Media type for HTML text.
 ///
 /// <https://www.iana.org/assignments/media-types/media-types.xhtml>
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
 pub enum MediaTypeHtml {
     /// `text/html`
     #[serde(rename = "text/html")]

src/protocol/verification.rs

@@ -1,7 +1,7 @@
 //! Verify that received data is valid
 
-use crate::error::Error;
-use anyhow::anyhow;
+use crate::{config::Data, error::Error, fetch::object_id::ObjectId, traits::Object};
+use serde::Deserialize;
 use url::Url;
 
 /// Check that both urls have the same domain. If not, return UrlVerificationError.
@@ -16,7 +16,7 @@ use url::Url;
 /// ```
 pub fn verify_domains_match(a: &Url, b: &Url) -> Result<(), Error> {
     if a.domain() != b.domain() {
-        return Err(Error::UrlVerificationError(anyhow!("Domains do not match")));
+        return Err(Error::UrlVerificationError("Domains do not match"));
     }
     Ok(())
 }
@@ -33,7 +33,42 @@ pub fn verify_domains_match(a: &Url, b: &Url) -> Result<(), Error> {
 /// ```
 pub fn verify_urls_match(a: &Url, b: &Url) -> Result<(), Error> {
     if a != b {
-        return Err(Error::UrlVerificationError(anyhow!("Urls do not match")));
+        return Err(Error::UrlVerificationError("Urls do not match"));
     }
     Ok(())
 }
+
+/// Check that the given ID doesn't match the local domain.
+///
+/// It is important to verify this to avoid local objects from being overwritten. In general
+/// locally created objects should be considered authorative, while incoming federated data
+/// is untrusted. Lack of such a check could allow an attacker to rewrite local posts. It could
+/// also result in an `object.local` field being overwritten with `false` for local objects, resulting in invalid data.
+///
+/// ```
+/// # use activitypub_federation::fetch::object_id::ObjectId;
+/// # use activitypub_federation::config::FederationConfig;
+/// # use activitypub_federation::protocol::verification::verify_is_remote_object;
+/// # use activitypub_federation::traits::tests::{DbConnection, DbUser};
+/// # tokio::runtime::Runtime::new().unwrap().block_on(async {
+/// # let config = FederationConfig::builder().domain("example.com").app_data(DbConnection).build().await?;
+/// # let data = config.to_request_data();
+/// let id = ObjectId::<DbUser>::parse("https://remote.com/u/name")?;
+/// assert!(verify_is_remote_object(&id, &data).is_ok());
+/// # Ok::<(), anyhow::Error>(())
+/// # }).unwrap();
+/// ```
+pub fn verify_is_remote_object<Kind, R: Clone>(
+    id: &ObjectId<Kind>,
+    data: &Data<<Kind as Object>::DataType>,
+) -> Result<(), Error>
+where
+    Kind: Object<DataType = R> + Send + Sync + 'static,
+    for<'de2> <Kind as Object>::Kind: Deserialize<'de2>,
+{
+    if id.is_local(data) {
+        Err(Error::UrlVerificationError("Object is not remote"))
+    } else {
+        Ok(())
+    }
+}

src/reqwest_shim.rs

@@ -3,17 +3,15 @@ use bytes::{BufMut, Bytes, BytesMut};
 use futures_core::{ready, stream::BoxStream, Stream};
 use pin_project_lite::pin_project;
 use reqwest::Response;
-use serde::de::DeserializeOwned;
 use std::{
     future::Future,
-    marker::PhantomData,
     mem,
     pin::Pin,
     task::{Context, Poll},
 };
 
-/// 200KB
-const MAX_BODY_SIZE: usize = 204800;
+/// 1 MB
+const MAX_BODY_SIZE: usize = 1024 * 1024;
 
 pin_project! {
     pub struct BytesFuture {
@@ -30,10 +28,7 @@ impl Future for BytesFuture {
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         loop {
             let this = self.as_mut().project();
-            if let Some(chunk) = ready!(this.stream.poll_next(cx))
-                .transpose()
-                .map_err(Error::other)?
-            {
+            if let Some(chunk) = ready!(this.stream.poll_next(cx)).transpose()? {
                 this.aggregator.put(chunk);
                 if this.aggregator.len() > *this.limit {
                     return Poll::Ready(Err(Error::ResponseBodyLimit));
@@ -49,27 +44,6 @@ impl Future for BytesFuture {
     }
 }
 
-pin_project! {
-    pub struct JsonFuture<T> {
-        _t: PhantomData<T>,
-        #[pin]
-        future: BytesFuture,
-    }
-}
-
-impl<T> Future for JsonFuture<T>
-where
-    T: DeserializeOwned,
-{
-    type Output = Result<T, Error>;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let this = self.project();
-        let bytes = ready!(this.future.poll(cx))?;
-        Poll::Ready(serde_json::from_slice(&bytes).map_err(Error::other))
-    }
-}
-
 pin_project! {
     pub struct TextFuture {
         #[pin]
@@ -83,7 +57,7 @@ impl Future for TextFuture {
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         let this = self.project();
         let bytes = ready!(this.future.poll(cx))?;
-        Poll::Ready(String::from_utf8(bytes.to_vec()).map_err(Error::other))
+        Poll::Ready(String::from_utf8(bytes.to_vec()).map_err(Error::Utf8))
     }
 }
 
@@ -92,25 +66,21 @@ impl Future for TextFuture {
 /// Reqwest doesn't limit the response body size by default nor does it offer an option to configure one.
 /// Since we have to fetch data from untrusted sources, not restricting the maximum size is a DoS hazard for us.
 ///
-/// This shim reimplements the `bytes`, `json`, and `text` functions and restricts the bodies to 100KB.
+/// This shim reimplements the `bytes`, `json`, and `text` functions and restricts the bodies length.
 ///
 /// TODO: Remove this shim as soon as reqwest gets support for size-limited bodies.
 pub trait ResponseExt {
     type BytesFuture;
-    type JsonFuture<T>;
     type TextFuture;
 
     /// Size limited version of `bytes` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
     fn bytes_limited(self) -> Self::BytesFuture;
-    /// Size limited version of `json` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
-    fn json_limited<T>(self) -> Self::JsonFuture<T>;
     /// Size limited version of `text` to work around a reqwest issue. Check [`ResponseExt`] docs for details.
     fn text_limited(self) -> Self::TextFuture;
 }
 
 impl ResponseExt for Response {
     type BytesFuture = BytesFuture;
-    type JsonFuture<T> = JsonFuture<T>;
     type TextFuture = TextFuture;
 
     fn bytes_limited(self) -> Self::BytesFuture {
@@ -121,13 +91,6 @@ impl ResponseExt for Response {
         }
     }
 
-    fn json_limited<T>(self) -> Self::JsonFuture<T> {
-        JsonFuture {
-            _t: PhantomData,
-            future: self.bytes_limited(),
-        }
-    }
-
     fn text_limited(self) -> Self::TextFuture {
         TextFuture {
             future: self.bytes_limited(),

src/traits/either.rs

@@ -0,0 +1,133 @@
+use super::{Actor, Object};
+use crate::{config::Data, error::Error};
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use either::Either;
+use serde::{Deserialize, Serialize};
+use std::fmt::Debug;
+use url::Url;
+
+#[doc(hidden)]
+#[derive(Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum UntaggedEither<L, R> {
+    Left(L),
+    Right(R),
+}
+
+#[async_trait]
+impl<T, R, E, D> Object for Either<T, R>
+where
+    T: Object + Object<Error = E, DataType = D> + Send + Sync,
+    R: Object + Object<Error = E, DataType = D> + Send + Sync,
+    <T as Object>::Kind: Send + Sync,
+    <R as Object>::Kind: Send + Sync,
+    D: Sync + Send + Clone,
+    E: From<Error> + Debug,
+{
+    type DataType = D;
+    type Kind = UntaggedEither<T::Kind, R::Kind>;
+    type Error = E;
+
+    /// `id` field of the object
+    fn id(&self) -> Url {
+        match self {
+            Either::Left(l) => l.id(),
+            Either::Right(r) => r.id(),
+        }
+    }
+
+    fn last_refreshed_at(&self) -> Option<DateTime<Utc>> {
+        match self {
+            Either::Left(l) => l.last_refreshed_at(),
+            Either::Right(r) => r.last_refreshed_at(),
+        }
+    }
+
+    async fn read_from_id(
+        object_id: Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        let l = T::read_from_id(object_id.clone(), data).await?;
+        if let Some(l) = l {
+            return Ok(Some(Either::Left(l)));
+        }
+        let r = R::read_from_id(object_id.clone(), data).await?;
+        if let Some(r) = r {
+            return Ok(Some(Either::Right(r)));
+        }
+        Ok(None)
+    }
+
+    async fn delete(&self, data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        match self {
+            Either::Left(l) => l.delete(data).await,
+            Either::Right(r) => r.delete(data).await,
+        }
+    }
+
+    fn is_deleted(&self) -> bool {
+        match self {
+            Either::Left(l) => l.is_deleted(),
+            Either::Right(r) => r.is_deleted(),
+        }
+    }
+
+    async fn into_json(self, data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        Ok(match self {
+            Either::Left(l) => UntaggedEither::Left(l.into_json(data).await?),
+            Either::Right(r) => UntaggedEither::Right(r.into_json(data).await?),
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        match json {
+            UntaggedEither::Left(l) => T::verify(l, expected_domain, data).await?,
+            UntaggedEither::Right(r) => R::verify(r, expected_domain, data).await?,
+        };
+        Ok(())
+    }
+
+    async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+        Ok(match json {
+            UntaggedEither::Left(l) => Either::Left(T::from_json(l, data).await?),
+            UntaggedEither::Right(r) => Either::Right(R::from_json(r, data).await?),
+        })
+    }
+}
+
+#[async_trait]
+impl<T, R, E, D> Actor for Either<T, R>
+where
+    T: Actor + Object + Object<Error = E, DataType = D> + Send + Sync + 'static,
+    R: Actor + Object + Object<Error = E, DataType = D> + Send + Sync + 'static,
+    <T as Object>::Kind: Send + Sync,
+    <R as Object>::Kind: Send + Sync,
+    D: Sync + Send + Clone,
+    E: From<Error> + Debug,
+{
+    fn public_key_pem(&self) -> &str {
+        match self {
+            Either::Left(l) => l.public_key_pem(),
+            Either::Right(r) => r.public_key_pem(),
+        }
+    }
+
+    fn private_key_pem(&self) -> Option<String> {
+        match self {
+            Either::Left(l) => l.private_key_pem(),
+            Either::Right(r) => r.private_key_pem(),
+        }
+    }
+
+    fn inbox(&self) -> Url {
+        match self {
+            Either::Left(l) => l.inbox(),
+            Either::Right(r) => r.inbox(),
+        }
+    }
+}

src/traits/mod.rs

@@ -7,6 +7,10 @@ use serde::Deserialize;
 use std::{fmt::Debug, ops::Deref};
 use url::Url;
 
+/// `Either` implementations for traits
+pub mod either;
+pub mod tests;
+
 /// Helper for converting between database structs and federated protocol structs.
 ///
 /// ```
@@ -49,6 +53,8 @@ use url::Url;
 ///     type Kind = Note;
 ///     type Error = anyhow::Error;
 ///
+/// fn id(&self) -> Url { self.ap_id.inner().clone() }
+///
 /// async fn read_from_id(object_id: Url, data: &Data<Self::DataType>) -> Result<Option<Self>, Self::Error> {
 ///         // Attempt to read object from local database. Return Ok(None) if not found.
 ///         let post: Option<DbPost> = data.read_post_from_json_id(object_id).await?;
@@ -103,6 +109,9 @@ pub trait Object: Sized + Debug {
     /// Error type returned by handler methods
     type Error;
 
+    /// `id` field of the object
+    fn id(&self) -> Url;
+
     /// Returns the last time this object was updated.
     ///
     /// If this returns `Some` and the value is too long ago, the object is refetched from the
@@ -127,10 +136,15 @@ pub trait Object: Sized + Debug {
     /// Mark remote object as deleted in local database.
     ///
     /// Called when a `Delete` activity is received, or if fetch returns a `Tombstone` object.
-    async fn delete(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+    async fn delete(&self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
         Ok(())
     }
 
+    /// Returns true if the object was deleted
+    fn is_deleted(&self) -> bool {
+        false
+    }
+
     /// Convert database type to Activitypub type.
     ///
     /// Called when a local object gets fetched by another instance over HTTP, or when an object
@@ -156,6 +170,40 @@ pub trait Object: Sized + Debug {
     /// should write the received object to database. Note that there is no distinction between
     /// create and update, so an `upsert` operation should be used.
     async fn from_json(json: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, Self::Error>;
+
+    /// Generates HTTP response to serve the object for fetching from other instances.
+    ///
+    /// - If the object has a remote domain, sends a redirect to the original instance.
+    /// - If [Object.is_deleted] returns true, returns a [crate::protocol::tombstone::Tombstone] instead.
+    /// - Otherwise serves the object JSON using [Object.into_json] and pretty-print
+    ///
+    /// `federation_context` is the value of `@context`.
+    #[cfg(feature = "actix-web")]
+    async fn http_response(
+        self,
+        federation_context: &serde_json::Value,
+        data: &Data<Self::DataType>,
+    ) -> Result<actix_web::HttpResponse, Self::Error>
+    where
+        Self::Error: From<serde_json::Error>,
+        Self::Kind: serde::Serialize + Send,
+    {
+        use crate::actix_web::response::{
+            create_http_response,
+            create_tombstone_response,
+            redirect_remote_object,
+        };
+        let id = self.id();
+        let res = if !data.config.is_local_url(&id) {
+            redirect_remote_object(&id)
+        } else if !self.is_deleted() {
+            let json = self.into_json(data).await?;
+            create_http_response(json, federation_context)?
+        } else {
+            create_tombstone_response(id.clone(), federation_context)?
+        };
+        Ok(res)
+    }
 }
 
 /// Handler for receiving incoming activities.
@@ -165,7 +213,7 @@ pub trait Object: Sized + Debug {
 /// # use url::Url;
 /// # use activitypub_federation::fetch::object_id::ObjectId;
 /// # use activitypub_federation::config::Data;
-/// # use activitypub_federation::traits::ActivityHandler;
+/// # use activitypub_federation::traits::Activity;
 /// # use activitypub_federation::traits::tests::{DbConnection, DbUser};
 /// #[derive(serde::Deserialize)]
 /// struct Follow {
@@ -177,7 +225,7 @@ pub trait Object: Sized + Debug {
 /// }
 ///
 /// #[async_trait::async_trait]
-/// impl ActivityHandler for Follow {
+/// impl Activity for Follow {
 ///     type DataType = DbConnection;
 ///     type Error = anyhow::Error;
 ///
@@ -203,7 +251,7 @@ pub trait Object: Sized + Debug {
 /// ```
 #[async_trait]
 #[enum_delegate::register]
-pub trait ActivityHandler {
+pub trait Activity {
     /// App data type passed to handlers. Must be identical to
     /// [crate::config::FederationConfigBuilder::app_data] type.
     type DataType: Clone + Send + Sync;
@@ -231,9 +279,6 @@ pub trait ActivityHandler {
 
 /// Trait to allow retrieving common Actor data.
 pub trait Actor: Object + Send + 'static {
-    /// `id` field of the actor
-    fn id(&self) -> Url;
-
     /// The actor's public key for verifying signatures of incoming activities.
     ///
     /// Use [generate_actor_keypair](crate::http_signatures::generate_actor_keypair) to create the
@@ -251,7 +296,7 @@ pub trait Actor: Object + Send + 'static {
 
     /// Generates a public key struct for use in the actor json representation
     fn public_key(&self) -> PublicKey {
-        PublicKey::new(self.id(), self.public_key_pem().to_string())
+        PublicKey::new(self.id().clone(), self.public_key_pem().to_string())
     }
 
     /// The actor's shared inbox, if any
@@ -267,9 +312,9 @@ pub trait Actor: Object + Send + 'static {
 
 /// Allow for boxing of enum variants
 #[async_trait]
-impl<T> ActivityHandler for Box<T>
+impl<T> Activity for Box<T>
 where
-    T: ActivityHandler + Send + Sync,
+    T: Activity + Send + Sync,
 {
     type DataType = T::DataType;
     type Error = T::Error;
@@ -331,207 +376,3 @@ pub trait Collection: Sized {
         data: &Data<Self::DataType>,
     ) -> Result<Self, Self::Error>;
 }
-
-/// Some impls of these traits for use in tests. Dont use this from external crates.
-///
-/// TODO: Should be using `cfg[doctest]` but blocked by <https://github.com/rust-lang/rust/issues/67295>
-#[doc(hidden)]
-#[allow(clippy::unwrap_used)]
-pub mod tests {
-    use super::*;
-    use crate::{
-        fetch::object_id::ObjectId,
-        http_signatures::{generate_actor_keypair, Keypair},
-        protocol::{public_key::PublicKey, verification::verify_domains_match},
-    };
-    use activitystreams_kinds::{activity::FollowType, actor::PersonType};
-    use anyhow::Error;
-    use once_cell::sync::Lazy;
-    use serde::{Deserialize, Serialize};
-
-    #[derive(Clone)]
-    pub struct DbConnection;
-
-    impl DbConnection {
-        pub async fn read_post_from_json_id<T>(&self, _: Url) -> Result<Option<T>, Error> {
-            Ok(None)
-        }
-        pub async fn read_local_user(&self, _: String) -> Result<DbUser, Error> {
-            todo!()
-        }
-        pub async fn upsert<T>(&self, _: &T) -> Result<(), Error> {
-            Ok(())
-        }
-        pub async fn add_follower(&self, _: DbUser, _: DbUser) -> Result<(), Error> {
-            Ok(())
-        }
-    }
-
-    #[derive(Clone, Debug, Deserialize, Serialize)]
-    #[serde(rename_all = "camelCase")]
-    pub struct Person {
-        #[serde(rename = "type")]
-        pub kind: PersonType,
-        pub preferred_username: String,
-        pub id: ObjectId<DbUser>,
-        pub inbox: Url,
-        pub public_key: PublicKey,
-    }
-    #[derive(Debug, Clone)]
-    pub struct DbUser {
-        pub name: String,
-        pub federation_id: Url,
-        pub inbox: Url,
-        pub public_key: String,
-        #[allow(dead_code)]
-        private_key: Option<String>,
-        pub followers: Vec<Url>,
-        pub local: bool,
-    }
-
-    pub static DB_USER_KEYPAIR: Lazy<Keypair> = Lazy::new(|| generate_actor_keypair().unwrap());
-
-    pub static DB_USER: Lazy<DbUser> = Lazy::new(|| DbUser {
-        name: String::new(),
-        federation_id: "https://localhost/123".parse().unwrap(),
-        inbox: "https://localhost/123/inbox".parse().unwrap(),
-        public_key: DB_USER_KEYPAIR.public_key.clone(),
-        private_key: Some(DB_USER_KEYPAIR.private_key.clone()),
-        followers: vec![],
-        local: false,
-    });
-
-    #[async_trait]
-    impl Object for DbUser {
-        type DataType = DbConnection;
-        type Kind = Person;
-        type Error = Error;
-
-        async fn read_from_id(
-            _object_id: Url,
-            _data: &Data<Self::DataType>,
-        ) -> Result<Option<Self>, Self::Error> {
-            Ok(Some(DB_USER.clone()))
-        }
-
-        async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
-            Ok(Person {
-                preferred_username: self.name.clone(),
-                kind: Default::default(),
-                id: self.federation_id.clone().into(),
-                inbox: self.inbox.clone(),
-                public_key: self.public_key(),
-            })
-        }
-
-        async fn verify(
-            json: &Self::Kind,
-            expected_domain: &Url,
-            _data: &Data<Self::DataType>,
-        ) -> Result<(), Self::Error> {
-            verify_domains_match(json.id.inner(), expected_domain)?;
-            Ok(())
-        }
-
-        async fn from_json(
-            json: Self::Kind,
-            _data: &Data<Self::DataType>,
-        ) -> Result<Self, Self::Error> {
-            Ok(DbUser {
-                name: json.preferred_username,
-                federation_id: json.id.into(),
-                inbox: json.inbox,
-                public_key: json.public_key.public_key_pem,
-                private_key: None,
-                followers: vec![],
-                local: false,
-            })
-        }
-    }
-
-    impl Actor for DbUser {
-        fn id(&self) -> Url {
-            self.federation_id.clone()
-        }
-
-        fn public_key_pem(&self) -> &str {
-            &self.public_key
-        }
-
-        fn private_key_pem(&self) -> Option<String> {
-            self.private_key.clone()
-        }
-
-        fn inbox(&self) -> Url {
-            self.inbox.clone()
-        }
-    }
-
-    #[derive(Deserialize, Serialize, Clone, Debug)]
-    #[serde(rename_all = "camelCase")]
-    pub struct Follow {
-        pub actor: ObjectId<DbUser>,
-        pub object: ObjectId<DbUser>,
-        #[serde(rename = "type")]
-        pub kind: FollowType,
-        pub id: Url,
-    }
-
-    #[async_trait]
-    impl ActivityHandler for Follow {
-        type DataType = DbConnection;
-        type Error = Error;
-
-        fn id(&self) -> &Url {
-            &self.id
-        }
-
-        fn actor(&self) -> &Url {
-            self.actor.inner()
-        }
-
-        async fn verify(&self, _: &Data<Self::DataType>) -> Result<(), Self::Error> {
-            Ok(())
-        }
-
-        async fn receive(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
-            Ok(())
-        }
-    }
-
-    #[derive(Clone, Debug, Deserialize, Serialize)]
-    #[serde(rename_all = "camelCase")]
-    pub struct Note {}
-    #[derive(Debug, Clone)]
-    pub struct DbPost {}
-
-    #[async_trait]
-    impl Object for DbPost {
-        type DataType = DbConnection;
-        type Kind = Note;
-        type Error = Error;
-
-        async fn read_from_id(
-            _: Url,
-            _: &Data<Self::DataType>,
-        ) -> Result<Option<Self>, Self::Error> {
-            todo!()
-        }
-
-        async fn into_json(self, _: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
-            todo!()
-        }
-
-        async fn verify(
-            _: &Self::Kind,
-            _: &Url,
-            _: &Data<Self::DataType>,
-        ) -> Result<(), Self::Error> {
-            todo!()
-        }
-
-        async fn from_json(_: Self::Kind, _: &Data<Self::DataType>) -> Result<Self, Self::Error> {
-            todo!()
-        }
-    }
-}

src/traits/tests.rs

@@ -0,0 +1,201 @@
+#![doc(hidden)]
+#![allow(clippy::unwrap_used)]
+//! Some impls of these traits for use in tests. Dont use this from external crates.
+//!
+//! TODO: Should be using `cfg[doctest]` but blocked by <https://github.com/rust-lang/rust/issues/67295>
+
+use super::{async_trait, Activity, Actor, Data, Debug, Object, PublicKey, Url};
+use crate::{
+    error::Error,
+    fetch::object_id::ObjectId,
+    http_signatures::{generate_actor_keypair, Keypair},
+    protocol::verification::verify_domains_match,
+};
+use activitystreams_kinds::{activity::FollowType, actor::PersonType};
+use serde::{Deserialize, Serialize};
+use std::sync::LazyLock;
+
+#[derive(Clone)]
+pub struct DbConnection;
+
+impl DbConnection {
+    pub async fn read_post_from_json_id<T>(&self, _: Url) -> Result<Option<T>, Error> {
+        Ok(None)
+    }
+    pub async fn read_local_user(&self, _: &str) -> Result<DbUser, Error> {
+        todo!()
+    }
+    pub async fn upsert<T>(&self, _: &T) -> Result<(), Error> {
+        Ok(())
+    }
+    pub async fn add_follower(&self, _: DbUser, _: DbUser) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Person {
+    #[serde(rename = "type")]
+    pub kind: PersonType,
+    pub preferred_username: String,
+    pub id: ObjectId<DbUser>,
+    pub inbox: Url,
+    pub public_key: PublicKey,
+}
+#[derive(Debug, Clone)]
+pub struct DbUser {
+    pub name: String,
+    pub federation_id: Url,
+    pub inbox: Url,
+    pub public_key: String,
+    #[allow(dead_code)]
+    private_key: Option<String>,
+    pub followers: Vec<Url>,
+    pub local: bool,
+}
+
+pub static DB_USER_KEYPAIR: LazyLock<Keypair> = LazyLock::new(|| generate_actor_keypair().unwrap());
+
+pub static DB_USER: LazyLock<DbUser> = LazyLock::new(|| DbUser {
+    name: String::new(),
+    federation_id: "https://localhost/123".parse().unwrap(),
+    inbox: "https://localhost/123/inbox".parse().unwrap(),
+    public_key: DB_USER_KEYPAIR.public_key.clone(),
+    private_key: Some(DB_USER_KEYPAIR.private_key.clone()),
+    followers: vec![],
+    local: false,
+});
+
+#[async_trait]
+impl Object for DbUser {
+    type DataType = DbConnection;
+    type Kind = Person;
+    type Error = Error;
+
+    fn id(&self) -> Url {
+        self.federation_id.clone()
+    }
+
+    async fn read_from_id(
+        _object_id: Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<Option<Self>, Self::Error> {
+        Ok(Some(DB_USER.clone()))
+    }
+
+    async fn into_json(self, _data: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        Ok(Person {
+            preferred_username: self.name.clone(),
+            kind: Default::default(),
+            id: self.federation_id.clone().into(),
+            inbox: self.inbox.clone(),
+            public_key: self.public_key(),
+        })
+    }
+
+    async fn verify(
+        json: &Self::Kind,
+        expected_domain: &Url,
+        _data: &Data<Self::DataType>,
+    ) -> Result<(), Self::Error> {
+        verify_domains_match(json.id.inner(), expected_domain)?;
+        Ok(())
+    }
+
+    async fn from_json(
+        json: Self::Kind,
+        _data: &Data<Self::DataType>,
+    ) -> Result<Self, Self::Error> {
+        Ok(DbUser {
+            name: json.preferred_username,
+            federation_id: json.id.into(),
+            inbox: json.inbox,
+            public_key: json.public_key.public_key_pem,
+            private_key: None,
+            followers: vec![],
+            local: false,
+        })
+    }
+}
+
+impl Actor for DbUser {
+    fn public_key_pem(&self) -> &str {
+        &self.public_key
+    }
+
+    fn private_key_pem(&self) -> Option<String> {
+        self.private_key.clone()
+    }
+
+    fn inbox(&self) -> Url {
+        self.inbox.clone()
+    }
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Follow {
+    pub actor: ObjectId<DbUser>,
+    pub object: ObjectId<DbUser>,
+    #[serde(rename = "type")]
+    pub kind: FollowType,
+    pub id: Url,
+}
+
+#[async_trait]
+impl Activity for Follow {
+    type DataType = DbConnection;
+    type Error = Error;
+
+    fn id(&self) -> &Url {
+        &self.id
+    }
+
+    fn actor(&self) -> &Url {
+        self.actor.inner()
+    }
+
+    async fn verify(&self, _: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+
+    async fn receive(self, _data: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Note {}
+#[derive(Debug, Clone)]
+pub struct DbPost {
+    pub federation_id: Url,
+}
+
+#[async_trait]
+impl Object for DbPost {
+    type DataType = DbConnection;
+    type Kind = Note;
+    type Error = Error;
+
+    fn id(&self) -> Url {
+        todo!()
+    }
+
+    async fn read_from_id(_: Url, _: &Data<Self::DataType>) -> Result<Option<Self>, Self::Error> {
+        todo!()
+    }
+
+    async fn into_json(self, _: &Data<Self::DataType>) -> Result<Self::Kind, Self::Error> {
+        todo!()
+    }
+
+    async fn verify(_: &Self::Kind, _: &Url, _: &Data<Self::DataType>) -> Result<(), Self::Error> {
+        todo!()
+    }
+
+    async fn from_json(_: Self::Kind, _: &Data<Self::DataType>) -> Result<Self, Self::Error> {
+        todo!()
+    }
+}

src/utils.rs

@@ -0,0 +1,78 @@
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+
+use crate::error::Error;
+use tokio::net::lookup_host;
+use url::{Host, Url};
+
+// TODO: Use is_global() once stabilized
+//       https://doc.rust-lang.org/std/net/enum.IpAddr.html#method.is_global
+pub(crate) async fn validate_ip(url: &Url) -> Result<(), Error> {
+    let mut ip = vec![];
+    let host = url
+        .host()
+        .ok_or(Error::UrlVerificationError("Url must have a domain"))?;
+    match host {
+        Host::Domain(domain) => ip.extend(
+            lookup_host((domain.to_owned(), 80))
+                .await?
+                .map(|s| s.ip().to_canonical()),
+        ),
+        Host::Ipv4(ipv4) => ip.push(ipv4.into()),
+        Host::Ipv6(ipv6) => ip.push(ipv6.into()),
+    };
+
+    let invalid_ip = ip.into_iter().any(|addr| match addr {
+        IpAddr::V4(addr) => v4_is_invalid(addr),
+        IpAddr::V6(addr) => v6_is_invalid(addr),
+    });
+    if invalid_ip {
+        return Err(Error::DomainResolveError(host.to_string()));
+    }
+    Ok(())
+}
+
+fn v4_is_invalid(v4: Ipv4Addr) -> bool {
+    v4.is_private()
+        || v4.is_loopback()
+        || v4.is_link_local()
+        || v4.is_multicast()
+        || v4.is_documentation()
+        || v4.is_unspecified()
+        || v4.is_broadcast()
+}
+
+fn v6_is_invalid(v6: Ipv6Addr) -> bool {
+    v6.is_loopback()
+        || v6.is_multicast()
+        || v6.is_unique_local()
+        || v6.is_unicast_link_local()
+        || v6.is_unspecified()
+        || v6_is_documentation(v6)
+        || v6.to_ipv4_mapped().is_some_and(v4_is_invalid)
+}
+
+fn v6_is_documentation(v6: std::net::Ipv6Addr) -> bool {
+    matches!(
+        v6.segments(),
+        [0x2001, 0xdb8, ..] | [0x3fff, 0..=0x0fff, ..]
+    )
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_is_valid_ip() -> Result<(), Error> {
+        assert!(validate_ip(&Url::parse("http://example.com")?)
+            .await
+            .is_ok());
+        assert!(validate_ip(&Url::parse("http://172.66.147.243")?)
+            .await
+            .is_ok());
+        assert!(validate_ip(&Url::parse("http://localhost")?).await.is_err());
+        assert!(validate_ip(&Url::parse("http://127.0.0.1")?).await.is_err());
+        Ok(())
+    }
+}