Add a system actor to sign outgoing S2S GETs

12 files changed, 293 insertions, 93 deletions

diff --git a/core/ld.py b/core/ld.py
index fff0526..1d79abf 100644
--- a/core/ld.py
+++ b/core/ld.py
@@ -358,6 +358,12 @@ schemas = {
             ]
         },
     },
+    "joinmastodon.org/ns": {
+        "contentType": "application/ld+json",
+        "documentUrl": "http://joinmastodon.org/ns",
+        "contextUrl": None,
+        "document": {},
+    },
 }
 
 DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
diff --git a/core/models/config.py b/core/models/config.py
index ffe7172..d69205c 100644
--- a/core/models/config.py
+++ b/core/models/config.py
@@ -154,6 +154,9 @@ class Config(models.Model):
 
         version: str = __version__
 
+        system_actor_public_key: str = ""
+        system_actor_private_key: str = ""
+
         site_name: str = "Takahē"
         highlight_color: str = "#449c8c"
         site_about: str = "<h2>Welcome!</h2>\n\nThis is a community running Takahē."
diff --git a/core/signatures.py b/core/signatures.py
index d981f87..df3ca61 100644
--- a/core/signatures.py
+++ b/core/signatures.py
@@ -1,10 +1,11 @@
 import base64
 import json
-from typing import Dict, List, Literal, TypedDict
+from typing import Dict, List, Literal, Optional, Tuple, TypedDict
 from urllib.parse import urlparse
 
 import httpx
-from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
 from django.http import HttpRequest
 from django.utils import timezone
 from django.utils.http import http_date, parse_http_date
@@ -30,6 +31,32 @@ class VerificationFormatError(VerificationError):
     pass
 
 
+class RsaKeys:
+    @classmethod
+    def generate_keypair(cls) -> Tuple[str, str]:
+        """
+        Generates a new RSA keypair
+        """
+        private_key = rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=2048,
+        )
+        private_key_serialized = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("ascii")
+        public_key_serialized = (
+            private_key.public_key()
+            .public_bytes(
+                encoding=serialization.Encoding.PEM,
+                format=serialization.PublicFormat.SubjectPublicKeyInfo,
+            )
+            .decode("ascii")
+        )
+        return private_key_serialized, public_key_serialized
+
+
 class HttpSignature:
     """
     Allows for calculation and verification of HTTP signatures
@@ -138,28 +165,37 @@ class HttpSignature:
 
     @classmethod
     async def signed_request(
-        self,
+        cls,
         uri: str,
-        body: Dict,
+        body: Optional[Dict],
         private_key: str,
         key_id: str,
         content_type: str = "application/json",
-        method: Literal["post"] = "post",
+        method: Literal["get", "post"] = "post",
     ):
         """
         Performs an async request to the given path, with a document, signed
         as an identity.
         """
+        # Create the core header field set
         uri_parts = urlparse(uri)
         date_string = http_date()
-        body_bytes = json.dumps(body).encode("utf8")
         headers = {
             "(request-target)": f"{method} {uri_parts.path}",
             "Host": uri_parts.hostname,
             "Date": date_string,
-            "Digest": self.calculate_digest(body_bytes),
-            "Content-Type": content_type,
         }
+        # If we have a body, add a digest and content type
+        if body is not None:
+            body_bytes = json.dumps(body).encode("utf8")
+            headers["Digest"] = cls.calculate_digest(body_bytes)
+            headers["Content-Type"] = content_type
+        else:
+            body_bytes = b""
+        # GET requests get implicit accept headers added
+        if method == "get":
+            headers["Accept"] = "application/activity+json, application/ld+json"
+        # Sign the headers
         signed_string = "\n".join(
             f"{name.lower()}: {value}" for name, value in headers.items()
         )
@@ -172,7 +208,7 @@ class HttpSignature:
             signed_string.encode("ascii"),
             "sha256",
         )
-        headers["Signature"] = self.compile_signature(
+        headers["Signature"] = cls.compile_signature(
             {
                 "keyid": key_id,
                 "headers": list(headers.keys()),
@@ -180,6 +216,7 @@ class HttpSignature:
                 "algorithm": "rsa-sha256",
             }
         )
+        # Send the request with all those headers except the pseudo one
         del headers["(request-target)"]
         async with httpx.AsyncClient() as client:
             response = await client.request(
@@ -187,6 +224,7 @@ class HttpSignature:
                 uri,
                 headers=headers,
                 content=body_bytes,
+                follow_redirects=method == "get",
             )
             if response.status_code >= 400:
                 raise ValueError(
diff --git a/takahe/urls.py b/takahe/urls.py
index abb8b2c..614ec3b 100644
--- a/takahe/urls.py
+++ b/takahe/urls.py
@@ -104,11 +104,12 @@ urlpatterns = [
     path("@<handle>/activate/", identity.ActivateIdentity.as_view()),
     path("identity/select/", identity.SelectIdentity.as_view()),
     path("identity/create/", identity.CreateIdentity.as_view()),
-    # Well-known endpoints
+    # Well-known endpoints and system actor
     path(".well-known/webfinger", activitypub.Webfinger.as_view()),
     path(".well-known/host-meta", activitypub.HostMeta.as_view()),
     path(".well-known/nodeinfo", activitypub.NodeInfo.as_view()),
     path("nodeinfo/2.0/", activitypub.NodeInfo2.as_view()),
+    path("actor/", activitypub.SystemActorView.as_view()),
     # Django admin
     path("djadmin/", djadmin.site.urls),
     # Media files
diff --git a/users/apps.py b/users/apps.py
index 88f7b17..05b5c3f 100644
--- a/users/apps.py
+++ b/users/apps.py
@@ -4,3 +4,9 @@ from django.apps import AppConfig
 class UsersConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "users"
+
+    def ready(self) -> None:
+        # Generate the server actor keypair if needed
+        from users.models import SystemActor
+
+        SystemActor.generate_keys_if_needed()
diff --git a/users/models/__init__.py b/users/models/__init__.py
index fc0d402..1c5f519 100644
--- a/users/models/__init__.py
+++ b/users/models/__init__.py
@@ -5,5 +5,6 @@ from .identity import Identity, IdentityStates  # noqa
 from .inbox_message import InboxMessage, InboxMessageStates  # noqa
 from .invite import Invite  # noqa
 from .password_reset import PasswordReset  # noqa
+from .system_actor import SystemActor  # noqa
 from .user import User  # noqa
 from .user_event import UserEvent  # noqa
diff --git a/users/models/identity.py b/users/models/identity.py
index 2190c9c..98e7df9 100644
--- a/users/models/identity.py
+++ b/users/models/identity.py
@@ -5,8 +5,6 @@ from urllib.parse import urlparse
 import httpx
 import urlman
 from asgiref.sync import async_to_sync, sync_to_async
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import rsa
 from django.db import models
 from django.template.defaultfilters import linebreaks_filter
 from django.templatetags.static import static
@@ -15,9 +13,11 @@ from django.utils import timezone
 from core.exceptions import ActorMismatchError
 from core.html import sanitize_post
 from core.ld import canonicalise, media_type_from_filename
+from core.signatures import RsaKeys
 from core.uploads import upload_namer
 from stator.models import State, StateField, StateGraph, StatorModel
 from users.models.domain import Domain
+from users.models.system_actor import SystemActor
 
 
 class IdentityStates(StateGraph):
@@ -301,15 +301,16 @@ class Identity(StatorModel):
         """
         domain = handle.split("@")[1]
         try:
-            async with httpx.AsyncClient() as client:
-                response = await client.get(
-                    f"https://{domain}/.well-known/webfinger?resource=acct:{handle}",
-                    headers={"Accept": "application/json"},
-                    follow_redirects=True,
-                )
+            response = await SystemActor().signed_request(
+                method="get",
+                uri=f"https://{domain}/.well-known/webfinger?resource=acct:{handle}",
+            )
         except httpx.RequestError:
             return None, None
-        if response.status_code >= 400:
+        if response.status_code == 404:
+            # We don't trust this as much as 410 Gone, but skip for now
+            return None, None
+        if response.status_code >= 500:
             return None, None
         data = response.json()
         if data["subject"].startswith("acct:"):
@@ -329,40 +330,39 @@ class Identity(StatorModel):
         """
         if self.local:
             raise ValueError("Cannot fetch local identities")
-        async with httpx.AsyncClient() as client:
-            try:
-                response = await client.get(
-                    self.actor_uri,
-                    headers={"Accept": "application/json"},
-                    follow_redirects=True,
-                )
-            except httpx.RequestError:
-                return False
-            if response.status_code == 410:
-                # Their account got deleted, so let's do the same.
-                if self.pk:
-                    await Identity.objects.filter(pk=self.pk).adelete()
-                return False
-            if response.status_code >= 400:
-                return False
-            document = canonicalise(response.json(), include_security=True)
-            self.name = document.get("name")
-            self.profile_uri = document.get("url")
-            self.inbox_uri = document.get("inbox")
-            self.outbox_uri = document.get("outbox")
-            self.summary = document.get("summary")
-            self.username = document.get("preferredUsername")
-            if self.username and "@value" in self.username:
-                self.username = self.username["@value"]
-            if self.username:
-                self.username = self.username.lower()
-            self.manually_approves_followers = document.get(
-                "as:manuallyApprovesFollowers"
+        try:
+            response = await SystemActor().signed_request(
+                method="get",
+                uri=self.actor_uri,
             )
-            self.public_key = document.get("publicKey", {}).get("publicKeyPem")
-            self.public_key_id = document.get("publicKey", {}).get("id")
-            self.icon_uri = document.get("icon", {}).get("url")
-            self.image_uri = document.get("image", {}).get("url")
+        except httpx.RequestError:
+            return False
+        if response.status_code == 410:
+            # Their account got deleted, so let's do the same.
+            if self.pk:
+                await Identity.objects.filter(pk=self.pk).adelete()
+            return False
+        if response.status_code == 404:
+            # We don't trust this as much as 410 Gone, but skip for now
+            return False
+        if response.status_code >= 500:
+            return False
+        document = canonicalise(response.json(), include_security=True)
+        self.name = document.get("name")
+        self.profile_uri = document.get("url")
+        self.inbox_uri = document.get("inbox")
+        self.outbox_uri = document.get("outbox")
+        self.summary = document.get("summary")
+        self.username = document.get("preferredUsername")
+        if self.username and "@value" in self.username:
+            self.username = self.username["@value"]
+        if self.username:
+            self.username = self.username.lower()
+        self.manually_approves_followers = document.get("as:manuallyApprovesFollowers")
+        self.public_key = document.get("publicKey", {}).get("publicKeyPem")
+        self.public_key_id = document.get("publicKey", {}).get("id")
+        self.icon_uri = document.get("icon", {}).get("url")
+        self.image_uri = document.get("image", {}).get("url")
         # Now go do webfinger with that info to see if we can get a canonical domain
         actor_url_parts = urlparse(self.actor_uri)
         get_domain = sync_to_async(Domain.get_remote_domain)
@@ -387,22 +387,6 @@ class Identity(StatorModel):
     def generate_keypair(self):
         if not self.local:
             raise ValueError("Cannot generate keypair for remote user")
-        private_key = rsa.generate_private_key(
-            public_exponent=65537,
-            key_size=2048,
-        )
-        self.private_key = private_key.private_bytes(
-            encoding=serialization.Encoding.PEM,
-            format=serialization.PrivateFormat.PKCS8,
-            encryption_algorithm=serialization.NoEncryption(),
-        ).decode("ascii")
-        self.public_key = (
-            private_key.public_key()
-            .public_bytes(
-                encoding=serialization.Encoding.PEM,
-                format=serialization.PublicFormat.SubjectPublicKeyInfo,
-            )
-            .decode("ascii")
-        )
+        self.private_key, self.public_key = RsaKeys.generate_keypair()
         self.public_key_id = self.actor_uri + "#main-key"
         self.save()
diff --git a/users/models/system_actor.py b/users/models/system_actor.py
new file mode 100644
index 0000000..28ef1a8
--- /dev/null
+++ b/users/models/system_actor.py
@@ -0,0 +1,68 @@
+from typing import Dict, Literal, Optional
+
+from django.conf import settings
+
+from core.models import Config
+from core.signatures import HttpSignature, RsaKeys
+
+
+class SystemActor:
+    """
+    Represents the system actor, that we use to sign all HTTP requests
+    that are not on behalf of an Identity.
+
+    Note that this needs Config.system to be set to be initialised.
+    """
+
+    def __init__(self):
+        self.private_key = Config.system.system_actor_private_key
+        self.public_key = Config.system.system_actor_public_key
+        self.actor_uri = f"https://{settings.MAIN_DOMAIN}/actor/"
+        self.public_key_id = self.actor_uri + "#main-key"
+        self.profile_uri = f"https://{settings.MAIN_DOMAIN}/about/"
+        self.username = "__system__"
+
+    def generate_keys(self):
+        self.private_key, self.public_key = RsaKeys.generate_keypair()
+        Config.set_system("system_actor_private_key", self.private_key)
+        Config.set_system("system_actor_public_key", self.public_key)
+
+    @classmethod
+    def generate_keys_if_needed(cls):
+        # Load the system config into the right place
+        Config.system = Config.load_system()
+        instance = cls()
+        if "-----BEGIN" not in instance.private_key:
+            instance.generate_keys()
+
+    def to_ap(self):
+        return {
+            "id": self.actor_uri,
+            "type": "Application",
+            "inbox": self.actor_uri + "/inbox/",
+            "preferredUsername": self.username,
+            "url": self.profile_uri,
+            "as:manuallyApprovesFollowers": True,
+            "publicKey": {
+                "id": self.public_key_id,
+                "owner": self.actor_uri,
+                "publicKeyPem": self.public_key,
+            },
+        }
+
+    async def signed_request(
+        self,
+        method: Literal["get", "post"],
+        uri: str,
+        body: Optional[Dict] = None,
+    ):
+        """
+        Performs a signed request on behalf of the System Actor.
+        """
+        return await HttpSignature.signed_request(
+            method=method,
+            uri=uri,
+            body=body,
+            private_key=self.private_key,
+            key_id=self.public_key_id,
+        )
diff --git a/users/tests/conftest.py b/users/tests/conftest.py
new file mode 100644
index 0000000..e1eeb4b
--- /dev/null
+++ b/users/tests/conftest.py
@@ -0,0 +1,51 @@
+import pytest
+
+from core.models import Config
+
+# Our testing-only keypair
+private_key = """-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCzNJa9JIxQpOtQ
+z8UQKXDPREF9DyBliGu3uPWo6DMnkOm7hoh2+nOryrWDqWOFaVK//n7kltHXUEbm
+U3exh0/0iWfzx2AbNrI04csAvW/hRvHbHBnVTotSxzqTd3ESkpcSW4xVuz9aCcFR
+kW3unSCO3fF0Lh8Jsy9N/CT6oTnwG+ZpeGvHVbh9xfR5Ww6zA7z8A6B17hbzdMd/
+3qUPijyIb5se4cWVtGg/ZJ0X1syn9u9kpwUjhHlyWH/esMRHxPuW49BPZPhhKs1+
+t//4xgZcRX515qFqPS2EtYgZAfh7M3TRv8uCSzL4TT+8ka9IUwKdV6TFaqH27bAG
+KyJQfGaTAgMBAAECggEALZY5qFjlRtiFMfQApdlc5KTw4d7Yt2tqN3zaJUMYTD7d
+boJNMbMJfNCetyT+d6Aw2D1ly0GglNzLhGkEQElzKfpQUt/Lj3CtCa3Mpd4K2Wxi
+NwJhgfUulPqwaHYQchCPVLCsNNziw0VLA7Rymionb6B+/TaEV8PYy0ZSo90ir3UD
+CL5t+IWgIPiy6pk1wGOmeB+tU4+V7/hFel+vPFNahafqVhLE311dfx2aOfweAEfN
+e4JoPeJP1/fB+BVZMyVSAraKz6wheymBBNKKn/vpFsdd6it2AP4UZeFp6ma9wT9t
+nk65IpHg1MBxazQd7621GrPH+ZnhMg62H/FEj6rIDQKBgQC1w1fEbk+zjI54DXU8
+FAe5cJbZS89fMP5CtzlWKzTzfdaavT+5cUYp3XAv37tSGsqYAXxY+4bHGa+qdCQO
+I41cmylWGNX2e29/p2BspDPM6YQ0Z21MxFRBTWvHFrhd0bF1cXKBKPttdkKvzOEP
+6uNy+/QtRNn9xF/ZjaMHcyPPTQKBgQD8ZdOmZ3TMsYJchAjjseN8S+Objw2oZzmK
+6I1ULJBz3DWiyCUfir+pMjSH4fsAf9zrHkiM7xUgMByTukVRt16BrT7TlEBanAxc
+/AKdNB3f0pza829LCz1lMAUn+ngZLTmRR+1rQFXqTjhB+0peJzKiMli+9BBhL9Ry
+jMeTuLHdXwKBgGiz9kL5KIBNX2RYnEfXYfu4l6zktrgnCNB1q1mv2fjJbG4GxkaU
+sc47+Pwa7VUGid22PWMkwSa/7SlLbdmXMT8/QjiOZfJueHQYfrsWe6B2g+mMCrJG
+BiL37jXpKJsiyA7XIxaz/OG5VgDfDGaW8B60dJv/JXPBQ1WW+Wq5MM+hAoGAAUdS
+xykHAnJzwpw4n06rZFnOEV+sJgo/1GBRNvfy02NuMiDpbzt4tRa4BWgzqVD8gYRp
+wa0EYmFcA7OR3lQbenSyOMgre0oHFgGA0eMNs7CRctqA2dR4vyZ7IDS4nwgHnqDK
+pxxwUvuKdWsceVWhgAjZQj5iRtvDK8Fi0XDCFekCgYALTU1v5iMIpaRAe+eyA2B1
+42qm4B/uhXznvOu2YXU6iJFmMgHGYgpa+Dq8uUjKtpn/LIFeX1KN0hH8z/0LW3gB
+e7tN7taW0oLK3RQcEMfkZ7diE9x3LGqo/xMxsZMtxAr88p5eMEU/nxxznOqq+W9b
+qxRbXYzEtHz+cW9+FZkyVw==
+-----END PRIVATE KEY-----"""
+
+public_key = """-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAszSWvSSMUKTrUM/FEClw
+z0RBfQ8gZYhrt7j1qOgzJ5Dpu4aIdvpzq8q1g6ljhWlSv/5+5JbR11BG5lN3sYdP
+9Iln88dgGzayNOHLAL1v4Ubx2xwZ1U6LUsc6k3dxEpKXEluMVbs/WgnBUZFt7p0g
+jt3xdC4fCbMvTfwk+qE58BvmaXhrx1W4fcX0eVsOswO8/AOgde4W83THf96lD4o8
+iG+bHuHFlbRoP2SdF9bMp/bvZKcFI4R5clh/3rDER8T7luPQT2T4YSrNfrf/+MYG
+XEV+deahaj0thLWIGQH4ezN00b/Lgksy+E0/vJGvSFMCnVekxWqh9u2wBisiUHxm
+kwIDAQAB
+-----END PUBLIC KEY-----"""
+
+
+@pytest.fixture
+def config_system(db):
+    Config.system = Config.SystemOptions(
+        system_actor_private_key=private_key, system_actor_public_key=public_key
+    )
+    yield Config.system
diff --git a/users/tests/models/test_identity.py b/users/tests/models/test_identity.py
index 868894a..738abe3 100644
--- a/users/tests/models/test_identity.py
+++ b/users/tests/models/test_identity.py
@@ -96,7 +96,7 @@ def test_identity_max_per_user(client):
 
 
 @pytest.mark.django_db
-def test_fetch_actor(httpx_mock):
+def test_fetch_actor(httpx_mock, config_system):
     """
     Ensures that making identities via actor fetching works
     """
diff --git a/users/views/activitypub.py b/users/views/activitypub.py
index c0fcd98..bb52f8a 100644
--- a/users/views/activitypub.py
+++ b/users/views/activitypub.py
@@ -18,7 +18,7 @@ from core.signatures import (
     VerificationFormatError,
 )
 from takahe import __version__
-from users.models import Identity, InboxMessage
+from users.models import Identity, InboxMessage, SystemActor
 from users.shortcuts import by_handle_or_404
 
 
@@ -96,28 +96,52 @@ class Webfinger(View):
         resource = request.GET.get("resource")
         if not resource.startswith("acct:"):
             raise Http404("Not an account resource")
-        handle = resource[5:].replace("testfedi", "feditest")
-        identity = by_handle_or_404(request, handle)
-        return JsonResponse(
-            {
-                "subject": f"acct:{identity.handle}",
-                "aliases": [
-                    str(identity.urls.view_nice),
-                ],
-                "links": [
-                    {
-                        "rel": "http://webfinger.net/rel/profile-page",
-                        "type": "text/html",
-                        "href": str(identity.urls.view_nice),
-                    },
-                    {
-                        "rel": "self",
-                        "type": "application/activity+json",
-                        "href": identity.actor_uri,
-                    },
-                ],
-            }
-        )
+        handle = resource[5:]
+        if handle.startswith("__system__@"):
+            # They are trying to webfinger the system actor
+            system_actor = SystemActor()
+            return JsonResponse(
+                {
+                    "subject": f"acct:{handle}",
+                    "aliases": [
+                        system_actor.profile_uri,
+                    ],
+                    "links": [
+                        {
+                            "rel": "http://webfinger.net/rel/profile-page",
+                            "type": "text/html",
+                            "href": system_actor.profile_uri,
+                        },
+                        {
+                            "rel": "self",
+                            "type": "application/activity+json",
+                            "href": system_actor.actor_uri,
+                        },
+                    ],
+                }
+            )
+        else:
+            identity = by_handle_or_404(request, handle)
+            return JsonResponse(
+                {
+                    "subject": f"acct:{identity.handle}",
+                    "aliases": [
+                        str(identity.urls.view_nice),
+                    ],
+                    "links": [
+                        {
+                            "rel": "http://webfinger.net/rel/profile-page",
+                            "type": "text/html",
+                            "href": str(identity.urls.view_nice),
+                        },
+                        {
+                            "rel": "self",
+                            "type": "application/activity+json",
+                            "href": identity.actor_uri,
+                        },
+                    ],
+                }
+            )
 
 
 @method_decorator(csrf_exempt, name="dispatch")
@@ -171,3 +195,17 @@ class Inbox(View):
         # Hand off the item to be processed by the queue
         InboxMessage.objects.create(message=document)
         return HttpResponse(status=202)
+
+
+class SystemActorView(View):
+    """
+    Special endpoint for the overall system actor
+    """
+
+    def get(self, request):
+        return JsonResponse(
+            canonicalise(
+                SystemActor().to_ap(),
+                include_security=True,
+            )
+        )
diff --git a/users/views/identity.py b/users/views/identity.py
index 4dae6d5..b96d2eb 100644
--- a/users/views/identity.py
+++ b/users/views/identity.py
@@ -161,6 +161,10 @@ class CreateIdentity(FormView):
                     raise forms.ValidationError(
                         "This username is restricted to administrators only."
                     )
+                if value in ["__system__"]:
+                    raise forms.ValidationError(
+                        "This username is reserved for system use."
+                    )
 
             # Validate it's all ascii characters
             for character in value: