diff options
33 files changed, 670 insertions, 42 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d31e946..42057c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,4 +46,4 @@ repos: rev: v0.991 hooks: - id: mypy - additional_dependencies: [types-pyopenssl, types-bleach, types-mock] + additional_dependencies: [types-pyopenssl, types-bleach, types-mock, types-cachetools] diff --git a/activities/admin.py b/activities/admin.py index edc7365..6b0c8a9 100644 --- a/activities/admin.py +++ b/activities/admin.py @@ -1,8 +1,10 @@ from asgiref.sync import async_to_sync from django.contrib import admin +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ from activities.models import ( + Emoji, FanOut, Hashtag, Post, @@ -50,6 +52,46 @@ class HashtagAdmin(admin.ModelAdmin): instance.transition_perform("outdated") +@admin.register(Emoji) +class EmojiAdmin(admin.ModelAdmin): + list_display = ( + "shortcode", + "preview", + "local", + "domain", + "public", + "state", + "created", + ) + list_filter = ("local", "public", "state") + search_fields = ("shortcode",) + + readonly_fields = ("preview", "created", "updated") + + actions = ["force_execution", "approve_emoji", "reject_emoji"] + + @admin.action(description="Force Execution") + def force_execution(self, request, queryset): + for instance in queryset: + instance.transition_perform("outdated") + + @admin.action(description="Approve Emoji") + def approve_emoji(self, request, queryset): + queryset.update(public=True) + + @admin.action(description="Reject Emoji") + def reject_emoji(self, request, queryset): + queryset.update(public=False) + + @admin.display(description="Emoji Preview") + def preview(self, instance): + if instance.public is False: + return mark_safe(f'<a href="{instance.full_url().relative}">Preview</a>') + return mark_safe( + f'<img src="{instance.full_url().relative}" style="height: 22px">' + ) + + @admin.register(PostAttachment) class PostAttachmentAdmin(admin.ModelAdmin): list_display = ["id", "post", "created"] diff --git a/activities/middleware.py b/activities/middleware.py new file mode 100644 index 0000000..1ed2219 --- /dev/null +++ b/activities/middleware.py @@ -0,0 +1,27 @@ +from time import time + +from activities.models import Emoji + + +class EmojiDefaultsLoadingMiddleware: + """ + Caches the default Emoji + """ + + refresh_interval: float = 30.0 + + def __init__(self, get_response): + self.get_response = get_response + self.loaded_ts: float = 0.0 + + def __call__(self, request): + # Allow test fixtures to force and lock the Emojis + if not getattr(Emoji, "__forced__", False): + if ( + not getattr(Emoji, "locals", None) + or (time() - self.loaded_ts) >= self.refresh_interval + ): + Emoji.locals = Emoji.load_locals() + self.loaded_ts = time() + response = self.get_response(request) + return response diff --git a/activities/migrations/0004_emoji_post_emojis.py b/activities/migrations/0004_emoji_post_emojis.py new file mode 100644 index 0000000..89bc825 --- /dev/null +++ b/activities/migrations/0004_emoji_post_emojis.py @@ -0,0 +1,91 @@ +# Generated by Django 4.1.4 on 2022-12-14 23:49 + +import functools + +import django.db.models.deletion +from django.db import migrations, models + +import activities.models.emoji +import core.uploads +import stator.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0003_identity_followers_etc"), + ("activities", "0003_postattachment_null_thumb"), + ] + + operations = [ + migrations.CreateModel( + name="Emoji", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("state_ready", models.BooleanField(default=True)), + ("state_changed", models.DateTimeField(auto_now_add=True)), + ("state_attempted", models.DateTimeField(blank=True, null=True)), + ("state_locked_until", models.DateTimeField(blank=True, null=True)), + ("shortcode", models.SlugField(max_length=100)), + ("local", models.BooleanField(default=True)), + ("public", models.BooleanField(null=True)), + ( + "object_uri", + models.CharField( + blank=True, max_length=500, null=True, unique=True + ), + ), + ("mimetype", models.CharField(max_length=200)), + ( + "file", + models.ImageField( + blank=True, + null=True, + upload_to=functools.partial( + core.uploads.upload_emoji_namer, *("emoji",), **{} + ), + ), + ), + ("remote_url", models.CharField(blank=True, max_length=500, null=True)), + ("category", models.CharField(blank=True, max_length=100, null=True)), + ( + "state", + stator.models.StateField( + choices=[("outdated", "outdated"), ("updated", "updated")], + default="outdated", + graph=activities.models.emoji.EmojiStates, + max_length=100, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "domain", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="users.domain", + ), + ), + ], + options={ + "unique_together": {("domain", "shortcode")}, + }, + ), + migrations.AddField( + model_name="post", + name="emojis", + field=models.ManyToManyField( + blank=True, related_name="posts_using_emoji", to="activities.emoji" + ), + ), + ] diff --git a/activities/models/__init__.py b/activities/models/__init__.py index aa34c0f..15a08a2 100644 --- a/activities/models/__init__.py +++ b/activities/models/__init__.py @@ -1,3 +1,4 @@ +from .emoji import Emoji, EmojiStates # noqa from .fan_out import FanOut, FanOutStates # noqa from .hashtag import Hashtag, HashtagStates # noqa from .post import Post, PostStates # noqa diff --git a/activities/models/emoji.py b/activities/models/emoji.py new file mode 100644 index 0000000..00f6e67 --- /dev/null +++ b/activities/models/emoji.py @@ -0,0 +1,261 @@ +import re +from functools import partial +from typing import ClassVar, cast + +import urlman +from asgiref.sync import sync_to_async +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.safestring import mark_safe + +from core.files import get_remote_file +from core.html import strip_html +from core.models import Config +from core.uploads import upload_emoji_namer +from core.uris import AutoAbsoluteUrl, RelativeAbsoluteUrl, StaticAbsoluteUrl +from stator.models import State, StateField, StateGraph, StatorModel +from users.models import Domain + + +class EmojiStates(StateGraph): + outdated = State(try_interval=300, force_initial=True) + updated = State() + + outdated.transitions_to(updated) + + @classmethod + async def handle_outdated(cls, instance: "Emoji"): + """ + Fetches remote emoji and uploads to file for local caching + """ + if instance.remote_url and not instance.file: + file, mimetype = await get_remote_file( + instance.remote_url, + timeout=settings.SETUP.REMOTE_TIMEOUT, + max_size=settings.SETUP.EMOJI_MAX_IMAGE_FILESIZE_KB * 1024, + ) + if file: + instance.file = file + instance.mimetype = mimetype + await sync_to_async(instance.save)() + + return cls.updated + + +class EmojiQuerySet(models.QuerySet): + def usable(self, domain: Domain | None = None): + public_q = models.Q(public=True) + if Config.system.emoji_unreviewed_are_public: + public_q |= models.Q(public__isnull=True) + + qs = self.filter(public_q) + if domain: + if domain.local: + qs = qs.filter(local=True) + else: + qs = qs.filter(domain=domain) + return qs + + +class EmojiManager(models.Manager): + def get_queryset(self): + return EmojiQuerySet(self.model, using=self._db) + + def usable(self, domain: Domain | None = None): + return self.get_queryset().usable(domain) + + +class Emoji(StatorModel): + + # Normalized Emoji without the ':' + shortcode = models.SlugField(max_length=100, db_index=True) + + domain = models.ForeignKey( + "users.Domain", null=True, blank=True, on_delete=models.CASCADE + ) + local = models.BooleanField(default=True) + + # Should this be shown in the public UI? + public = models.BooleanField(null=True) + + object_uri = models.CharField(max_length=500, blank=True, null=True, unique=True) + + mimetype = models.CharField(max_length=200) + + # Files may not be populated if it's remote and not cached on our side yet + file = models.ImageField( + upload_to=partial(upload_emoji_namer, "emoji"), + null=True, + blank=True, + ) + + # A link to the custom emoji + remote_url = models.CharField(max_length=500, blank=True, null=True) + + # Used for sorting custom emoji in the picker + category = models.CharField(max_length=100, blank=True, null=True) + + # State of this Emoji + state = StateField(EmojiStates) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + objects = EmojiManager() + + # Cache of the local emojis {shortcode: Emoji} + locals: ClassVar["dict[str, Emoji]"] + + class Meta: + unique_together = ("domain", "shortcode") + + class urls(urlman.Urls): + root = "/admin/emoji/" + create = "{root}/create/" + edit = "{root}{self.Emoji}/" + delete = "{edit}delete/" + + emoji_regex = re.compile(r"\B:([a-zA-Z0-9(_)-]+):\B") + + def clean(self): + super().clean() + if self.local ^ (self.domain is None): + raise ValidationError("Must be local or have a domain") + + def __str__(self): + return f"{self.id}-{self.shortcode}" + + @classmethod + def load_locals(cls) -> dict[str, "Emoji"]: + return {x.shortcode: x for x in Emoji.objects.usable().filter(local=True)} + + @property + def fullcode(self): + return f":{self.shortcode}:" + + @property + def is_usable(self) -> bool: + """ + Return True if this Emoji is usable. + """ + return self.public or ( + self.public is None and Config.system.emoji_unreviewed_are_public + ) + + def full_url(self) -> RelativeAbsoluteUrl: + if self.is_usable: + if self.file: + return AutoAbsoluteUrl(self.file.url) + elif self.remote_url: + return AutoAbsoluteUrl(f"/proxy/emoji/{self.pk}/") + return StaticAbsoluteUrl("img/blank-emoji-128.png") + + def as_html(self): + if self.is_usable: + return mark_safe( + f'<img src="{self.full_url().relative}" class="emoji" alt="Emoji {self.shortcode}">' + ) + return self.fullcode + + @classmethod + def imageify_emojis( + cls, + content: str, + *, + emojis: list["Emoji"] | EmojiQuerySet | None = None, + include_local: bool = True, + ): + """ + Find :emoji: in content and convert to <img>. If include_local is True, + the local emoji will be used as a fallback for any shortcodes not defined + by emojis. + """ + emoji_set = ( + cast(list[Emoji], list(cls.locals.values())) if include_local else [] + ) + + if emojis: + if isinstance(emojis, (EmojiQuerySet, list)): + emoji_set.extend(list(emojis)) + else: + raise TypeError("Unsupported type for emojis") + + possible_matches = { + emoji.shortcode: emoji.as_html() for emoji in emoji_set if emoji.is_usable + } + + def replacer(match): + fullcode = match.group(1).lower() + if fullcode in possible_matches: + return possible_matches[fullcode] + return match.group() + + return mark_safe(Emoji.emoji_regex.sub(replacer, content)) + + @classmethod + def emojis_from_content(cls, content: str, domain: Domain) -> list[str]: + """ + Return a parsed and sanitized of emoji found in content without + the surrounding ':'. + """ + emoji_hits = cls.emoji_regex.findall(strip_html(content)) + emojis = sorted({emoji.lower() for emoji in emoji_hits}) + return list( + cls.objects.filter(local=domain is None) + .usable(domain) + .filter(shortcode__in=emojis) + ) + + def to_ap_tag(self): + """ + Return this Emoji as an ActivityPub Tag + http://joinmastodon.org/ns#Emoji + """ + return { + "id": self.object_uri, + "type": "Emoji", + "name": self.shortcode, + "icon": { + "type": "Image", + "mediaType": self.mimetype, + "url": self.full_url().absolute, + }, + } + + @classmethod + def by_ap_tag(cls, domain: Domain, data: dict, create: bool = False): + """ """ + try: + return cls.objects.get(object_uri=data["id"]) + except cls.DoesNotExist: + if not create: + raise KeyError(f"No emoji with ID {data['id']}", data) + + # create + shortcode = data["name"].lower().strip(":") + icon = data["icon"] + category = (icon.get("category") or "")[:100] + emoji = cls.objects.create( + shortcode=shortcode, + domain=None if domain.local else domain, + local=domain.local, + object_uri=data["id"], + mimetype=icon["mediaType"], + category=category, + remote_url=icon["url"], + ) + return emoji + + ### Mastodon API ### + + def to_mastodon_json(self): + url = self.full_url().absolute + data = { + "shortcode": self.shortcode, + "url": url, + "static_url": self.remote_url or url, + "visible_in_picker": self.public, + "category": self.category or "", + } + return data diff --git a/activities/models/fan_out.py b/activities/models/fan_out.py index 8f4f342..fd52acd 100644 --- a/activities/models/fan_out.py +++ b/activities/models/fan_out.py @@ -184,8 +184,14 @@ class FanOut(StatorModel): """ Returns a version of the object with all relations pre-loaded """ - return await FanOut.objects.select_related( - "identity", - "subject_post", - "subject_post_interaction", - ).aget(pk=self.pk) + return ( + await FanOut.objects.select_related( + "identity", + "subject_post", + "subject_post_interaction", + ) + .prefetch_related( + "subject_post__emojis", + ) + .aget(pk=self.pk) + ) diff --git a/activities/models/post.py b/activities/models/post.py index 8e355bf..2b0a7c2 100644 --- a/activities/models/post.py +++ b/activities/models/post.py @@ -12,8 +12,10 @@ from django.template.defaultfilters import linebreaks_filter from django.utils import timezone from django.utils.safestring import mark_safe +from activities.models.emoji import Emoji from activities.models.fan_out import FanOut from activities.models.hashtag import Hashtag +from activities.templatetags.emoji_tags import imageify_emojis from core.html import sanitize_post, strip_html from core.ld import canonicalise, format_ld_date, get_list, parse_ld_date from stator.models import State, StateField, StateGraph, StatorModel @@ -218,6 +220,12 @@ class Post(StatorModel): # Hashtags in the post hashtags = models.JSONField(blank=True, null=True) + emojis = models.ManyToManyField( + "activities.Emoji", + related_name="posts_using_emoji", + blank=True, + ) + # When the post was originally created (as opposed to when we received it) published = models.DateTimeField(default=timezone.now) @@ -328,8 +336,11 @@ class Post(StatorModel): """ Returns the content formatted for local display """ - return Hashtag.linkify_hashtags( - self.linkify_mentions(sanitize_post(self.content), local=True) + return imageify_emojis( + Hashtag.linkify_hashtags( + self.linkify_mentions(sanitize_post(self.content), local=True) + ), + self.author.domain, ) def safe_content_remote(self): @@ -379,6 +390,8 @@ class Post(StatorModel): visibility = reply_to.Visibilities.local_only # Find hashtags in this post hashtags = Hashtag.hashtags_from_content(content) or None + # Find emoji in this post + emojis = Emoji.emojis_from_content(content, author.domain) # Strip all HTML and apply linebreaks filter content = linebreaks_filter(strip_html(content)) # Make the Post object @@ -395,6 +408,7 @@ class Post(StatorModel): post.object_uri = post.urls.object_uri post.url = post.absolute_object_uri() post.mentions.set(mentions) + post.emojis.set(emojis) if attachments: post.attachments.set(attachments) post.save() @@ -416,6 +430,7 @@ class Post(StatorModel): self.edited = timezone.now() self.hashtags = Hashtag.hashtags_from_content(content) or None self.mentions.set(self.mentions_from_content(content, self.author)) + self.emojis.set(Emoji.emojis_from_content(content, self.author.domain)) self.attachments.set(attachments or []) self.save() @@ -520,14 +535,11 @@ class Post(StatorModel): value["updated"] = format_ld_date(self.edited) # Mentions for mention in self.mentions.all(): - value["tag"].append( - { - "href": mention.actor_uri, - "name": "@" + mention.handle, - "type": "Mention", - } - ) + value["tag"].append(mention.to_ap_tag()) value["cc"].append(mention.actor_uri) + # Emoji + for emoji in self.emojis.all(): + value["tag"].append(emoji.to_ap_tag()) # Attachments for attachment in self.attachments.all(): value["attachment"].append(attachment.to_ap()) @@ -616,7 +628,9 @@ class Post(StatorModel): # Do we have one with the right ID? created = False try: - post = cls.objects.get(object_uri=data["id"]) + post = cls.objects.select_related("author__domain").get( + object_uri=data["id"] + ) except cls.DoesNotExist: if create: # Resolve the author @@ -645,10 +659,10 @@ class Post(StatorModel): mention_identity = Identity.by_actor_uri(tag["href"], create=True) post.mentions.add(mention_identity) elif tag["type"].lower() == "as:hashtag": - post.hashtags.append(tag["name"].lstrip("#")) + post.hashtags.append(tag["name"].lower().lstrip("#")) elif tag["type"].lower() == "http://joinmastodon.org/ns#emoji": - # TODO: Handle incoming emoji - pass + emoji = Emoji.by_ap_tag(post.author.domain, tag, create=True) + post.emojis.add(emoji) else: raise ValueError(f"Unknown tag type {tag['type']}") # Visibility and to @@ -818,7 +832,7 @@ class Post(StatorModel): if self.hashtags else [] ), - "emojis": [], + "emojis": [emoji.to_mastodon_json() for emoji in self.emojis.usable()], "reblogs_count": self.interactions.filter(type="boost").count(), "favourites_count": self.interactions.filter(type="like").count(), "replies_count": 0, diff --git a/activities/templatetags/emoji_tags.py b/activities/templatetags/emoji_tags.py new file mode 100644 index 0000000..ad221db --- /dev/null +++ b/activities/templatetags/emoji_tags.py @@ -0,0 +1,27 @@ +from cachetools import TTLCache, cached +from django import template + +from activities.models import Emoji +from users.models import Domain + +register = template.Library() + + +@cached(cache=TTLCache(maxsize=1000, ttl=60)) +def emoji_from_domain(domain: Domain | None) -> list[Emoji]: + if not domain: + return list(Emoji.locals.values()) + return list(Emoji.objects.usable(domain)) + + +@register.filter +def imageify_emojis(value: str, arg: Domain | None = None): + """ + Convert hashtags in content in to /tags/<hashtag>/ links. + """ + if not value: + return "" + + emojis = emoji_from_domain(arg) + + return Emoji.imageify_emojis(value, emojis=emojis) diff --git a/activities/views/posts.py b/activities/views/posts.py index e285c7e..1b8676d 100644 --- a/activities/views/posts.py +++ b/activities/views/posts.py @@ -67,6 +67,8 @@ class Individual(TemplateView): in_reply_to=self.post_obj.object_uri, ) .distinct() + .select_related("author__domain") + .prefetch_related("emojis") .order_by("published", "created"), } diff --git a/activities/views/timelines.py b/activities/views/timelines.py index 0c1b693..f55e331 100644 --- a/activities/views/timelines.py +++ b/activities/views/timelines.py @@ -98,8 +98,8 @@ class Local(ListView): def get_queryset(self): return ( Post.objects.local_public() - .select_related("author") - .prefetch_related("attachments", "mentions") + .select_related("author", "author__domain") + .prefetch_related("attachments", "mentions", "emojis") .order_by("-created")[:50] ) @@ -126,8 +126,8 @@ class Federated(ListView): Post.objects.filter( visibility=Post.Visibilities.public, in_reply_to__isnull=True ) - .select_related("author") - .prefetch_related("attachments", "mentions") + .select_related("author", "author__domain") + .prefetch_related("attachments", "mentions", "emojis") .order_by("-created")[:50] ) @@ -173,7 +173,13 @@ class Notifications(ListView): return ( TimelineEvent.objects.filter(identity=self.request.identity, type__in=types) .order_by("-created")[:50] - .select_related("subject_post", "subject_post__author", "subject_identity") + .select_related( + "subject_post", + "subject_post__author", + "subject_post__author__domain", + "subject_identity", + ) + .prefetch_related("subject_post__emojis") ) def get_context_data(self, **kwargs): diff --git a/core/files.py b/core/files.py index a04cef9..3ef79aa 100644 --- a/core/files.py +++ b/core/files.py @@ -1,7 +1,10 @@ import io import blurhash +import httpx +from django.conf import settings from django.core.files import File +from django.core.files.base import ContentFile from PIL import Image, ImageOps @@ -37,3 +40,28 @@ def blurhash_image(file) -> str: Returns the blurhash for an image """ return blurhash.encode(file, 4, 4) + + +async def get_remote_file( + url: str, + *, + timeout: float = settings.SETUP.REMOTE_TIMEOUT, + max_size: int | None = None, +) -> tuple[File | None, str | None]: + """ + Download a URL and return the File and content-type. + """ + async with httpx.AsyncClient() as client: + async with client.stream("GET", url, timeout=timeout) as stream: + allow_download = max_size is None + if max_size: + try: + content_length = int(stream.headers["content-length"]) + allow_download = content_length <= max_size + except TypeError: + pass + if allow_download: + file = ContentFile(await stream.aread(), name=url) + return file, stream.headers["content-type"] + + return None, None diff --git a/core/models/config.py b/core/models/config.py index 8f5dc31..9670c11 100644 --- a/core/models/config.py +++ b/core/models/config.py @@ -224,6 +224,8 @@ class Config(models.Model): hashtag_unreviewed_are_public: bool = True hashtag_stats_max_age: int = 60 * 60 + emoji_unreviewed_are_public: bool = False + cache_timeout_page_default: int = 60 cache_timeout_page_timeline: int = 60 * 3 cache_timeout_page_post: int = 60 * 2 diff --git a/core/uploads.py b/core/uploads.py index 41b6e94..f6c0e89 100644 --- a/core/uploads.py +++ b/core/uploads.py @@ -1,10 +1,14 @@ import os import secrets +from typing import TYPE_CHECKING from django.utils import timezone from storages.backends.gcloud import GoogleCloudStorage from storages.backends.s3boto3 import S3Boto3Storage +if TYPE_CHECKING: + from activities.models import Emoji + def upload_namer(prefix, instance, filename): """ @@ -16,6 +20,18 @@ def upload_namer(prefix, instance, filename): return f"{prefix}/{now.year}/{now.month}/{now.day}/{new_filename}{old_extension}" +def upload_emoji_namer(prefix, instance: "Emoji", filename): + """ + Names uploaded emoji per domain + """ + _, old_extension = os.path.splitext(filename) + if instance.domain is None: + domain = "_default" + else: + domain = instance.domain.domain + return f"{prefix}/{domain}/{instance.shortcode}{old_extension}" + + class TakaheS3Storage(S3Boto3Storage): """ Custom override backend that makes webp files store correctly diff --git a/mediaproxy/views.py b/mediaproxy/views.py index 57257f3..4fc09b1 100644 --- a/mediaproxy/views.py +++ b/mediaproxy/views.py @@ -5,7 +5,7 @@ from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.generic import View -from activities.models import PostAttachment +from activities.models import Emoji, PostAttachment from users.models import Identity @@ -57,6 +57,21 @@ class BaseCacheView(View): raise NotImplementedError() +class EmojiCacheView(BaseCacheView): + """ + Caches Emoji + """ + + item_timeout = 86400 * 7 # One week + + def get_remote_url(self): + self.emoji = get_object_or_404(Emoji, pk=self.kwargs["emoji_id"]) + + if not self.emoji.remote_url: + raise Http404() + return self.emoji.remote_url + + class IdentityIconCacheView(BaseCacheView): """ Caches identity icons (avatars) diff --git a/requirements.txt b/requirements.txt index d24b45d..628b19a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ bleach~=5.0.1 blurhash-python~=1.1.3 +cachetools~=5.2.0 cryptography~=38.0 dj_database_url~=1.0.0 django-cache-url~=3.4.2 diff --git a/static/css/style.css b/static/css/style.css index ce01793..9b509b8 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -358,6 +358,10 @@ nav a i { width: auto; } +.icon-menu .option img.emoji { + height: 22px; +} + .icon-menu .option i { display: inline-block; text-align: center; @@ -740,6 +744,10 @@ h1.identity .icon { margin: 0 20px 0 0; } +h1.identity .emoji { + height: 22px; +} + h1.identity small { display: block; font-size: 60%; @@ -752,6 +760,10 @@ h1.identity small { margin: 0 0 20px 0; } +.bio .emoji { + height: 22px; +} + .system-note { background: var(--color-bg-menu); color: var(--color-text-dull); @@ -789,6 +801,10 @@ table.metadata td.name { gap: 1em; } +table.metadata td .emoji { + height: 22px; +} + /* Timelines */ .left-column .timeline-name { @@ -857,6 +873,10 @@ table.metadata td.name { float: left; } +.post .emoji { + height: 22px; +} + .post .handle { display: block; padding: 7px 0 0 64px; @@ -1014,6 +1034,13 @@ table.metadata td.name { padding: 0 0 3px 5px; } +.boost-banner .emoji, +.mention-banner .emoji, +.follow-banner .emoji, +.like-banner .emoji { + height: 22px; +} + .boost-banner a, .mention-banner a, .follow-banner a, diff --git a/static/img/blank-emoji-128.png b/static/img/blank-emoji-128.png Binary files differnew file mode 100644 index 0000000..e1bc56c --- /dev/null +++ b/static/img/blank-emoji-128.png diff --git a/takahe/settings.py b/takahe/settings.py index 91bfe7b..98bc9dd 100644 --- a/takahe/settings.py +++ b/takahe/settings.py @@ -107,6 +107,11 @@ class Settings(BaseSettings): #: is necessary for compatibility with Mastodon’s image proxy. MEDIA_MAX_IMAGE_FILESIZE_MB: int = 10 + #: Maximum filesize for Emoji. Attempting to upload Local Emoji larger than this size will be + #: blocked. Remote Emoji larger than this size will not be fetched and served from media, but + #: served through the image proxy. + EMOJI_MAX_IMAGE_FILESIZE_KB: int = 200 + #: Request timeouts to use when talking to other servers Either #: float or tuple of floats for (connect, read, write, pool) REMOTE_TIMEOUT: float | tuple[float, float, float, float] = 5.0 @@ -194,6 +199,7 @@ MIDDLEWARE = [ "core.middleware.ConfigLoadingMiddleware", "api.middleware.ApiTokenMiddleware", "users.middleware.IdentityMiddleware", + "activities.middleware.EmojiDefaultsLoadingMiddleware", ] ROOT_URLCONF = "takahe.urls" diff --git a/takahe/urls.py b/takahe/urls.py index e8aa359..d6d893f 100644 --- a/takahe/urls.py +++ b/takahe/urls.py @@ -194,6 +194,11 @@ urlpatterns = [ mediaproxy.PostAttachmentCacheView.as_view(), name="proxy_post_attachment", ), + path( + "proxy/emoji/<emoji_id>/", + mediaproxy.EmojiCacheView.as_view(), + name="proxy_emoji", + ), # Well-known endpoints and system actor path(".well-known/webfinger", activitypub.Webfinger.as_view()), path(".well-known/host-meta", activitypub.HostMeta.as_view()), diff --git a/templates/activities/_event.html b/templates/activities/_event.html index 0c94aad..10d1ce4 100644 --- a/templates/activities/_event.html +++ b/templates/activities/_event.html @@ -3,14 +3,14 @@ {% if event.type == "followed" %} <div class="follow-banner"> <a href="{{ event.subject_identity.urls.view }}"> - {{ event.subject_identity.name_or_handle }} + {{ event.subject_identity.html_name_or_handle }} </a> followed you </div> {% include "activities/_identity.html" with identity=event.subject_identity created=event.created %} {% elif event.type == "liked" %} <div class="like-banner"> <a href="{{ event.subject_identity.urls.view }}"> - {{ event.subject_identity.name_or_handle }} + {{ event.subject_identity.html_name_or_handle }} </a> liked your post </div> {% if not event.collapsed %} @@ -19,7 +19,7 @@ {% elif event.type == "mentioned" %} <div class="mention-banner"> <a href="{{ event.subject_identity.urls.view }}"> - {{ event.subject_identity.name_or_handle }} + {{ event.subject_identity.html_name_or_handle }} </a> mentioned you </div> {% if not event.collapsed %} @@ -28,7 +28,7 @@ {% elif event.type == "boosted" %} <div class="boost-banner"> <a href="{{ event.subject_identity.urls.view }}"> - {{ event.subject_identity.name_or_handle }} + {{ event.subject_identity.html_name_or_handle }} </a> boosted your post </div> {% if not event.collapsed %} diff --git a/templates/activities/_identity.html b/templates/activities/_identity.html index feb3178..791f05f 100644 --- a/templates/activities/_identity.html +++ b/templates/activities/_identity.html @@ -12,6 +12,6 @@ {% endif %} <a href="{{ identity.urls.view }}" class="handle"> - {{ identity.name_or_handle }} <small>@{{ identity.handle }}</small> + {{ identity.html_name_or_handle }} <small>@{{ identity.handle }}</small> </a> </div> diff --git a/templates/activities/_mini_post.html b/templates/activities/_mini_post.html index 9f83333..335a8a4 100644 --- a/templates/activities/_mini_post.html +++ b/templates/activities/_mini_post.html @@ -7,7 +7,7 @@ </a> <a href="{{ post.author.urls.view }}" class="handle"> - {{ post.author.name_or_handle }} + {{ post.author.html_name_or_handle }} </a> <div class="content"> diff --git a/templates/activities/_post.html b/templates/activities/_post.html index 5802acd..5d75b78 100644 --- a/templates/activities/_post.html +++ b/templates/activities/_post.html @@ -59,7 +59,7 @@ {% endif %} <a href="{{ post.author.urls.view }}" class="handle"> - {{ post.author.name_or_handle }} <small>@{{ post.author.handle }}</small> + {{ post.author.html_name_or_handle }} <small>@{{ post.author.handle }}</small> </a> {% if post.summary %} diff --git a/templates/activities/follows.html b/templates/activities/follows.html index 27a13d3..18c7811 100644 --- a/templates/activities/follows.html +++ b/templates/activities/follows.html @@ -8,7 +8,7 @@ <a class="option" href="{{ identity.urls.view }}"> <img src="{{ identity.local_icon_url.relative }}"> <span class="handle"> - {{ identity.name_or_handle }} + {{ identity.html_name_or_handle }} <small>@{{ identity.handle }}</small> </span> {% if details.outbound %} diff --git a/templates/activities/home.html b/templates/activities/home.html index d574e2a..546da0d 100644 --- a/templates/activities/home.html +++ b/templates/activities/home.html @@ -10,7 +10,7 @@ {% elif event.type == "boost" %} <div class="boost-banner"> <a href="{{ event.subject_identity.urls.view }}"> - {{ event.subject_identity.name_or_handle }} + {{ event.subject_identity.html_name_or_handle }} </a> boosted <time> {{ event.subject_post_interaction.published | timedeltashort }} ago diff --git a/templates/activities/post.html b/templates/activities/post.html index fd717ad..6205064 100644 --- a/templates/activities/post.html +++ b/templates/activities/post.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Post by {{ post.author.name_or_handle }}{% endblock %} +{% block title %}Post by {{ post.author.html_name_or_handle }}{% endblock %} {% block content %} {% if parent %} diff --git a/templates/identity/select.html b/templates/identity/select.html index b3fc41d..5f73464 100644 --- a/templates/identity/select.html +++ b/templates/identity/select.html @@ -8,7 +8,7 @@ <a class="option" href="{{ identity.urls.activate }}"> <img src="{{ identity.local_icon_url.relative }}"> <span class="handle"> - {{ identity.name_or_handle }} + {{ identity.html_name_or_handle }} <small>@{{ identity.handle }}</small> </span> </a> diff --git a/templates/identity/view.html b/templates/identity/view.html index 7e2c8d4..145a0ef 100644 --- a/templates/identity/view.html +++ b/templates/identity/view.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% load emoji_tags %} {% block title %}{{ identity }}{% endblock %} @@ -40,7 +41,7 @@ {% endif %} {% endif %} - {{ identity.name_or_handle }} + {{ identity.html_name_or_handle }} <small> @{{ identity.handle }} <a title="Copy handle" diff --git a/tests/activities/models/test_post.py b/tests/activities/models/test_post.py index a5f2f79..ff74a9e 100644 --- a/tests/activities/models/test_post.py +++ b/tests/activities/models/test_post.py @@ -100,7 +100,9 @@ def test_linkify_mentions_remote( @pytest.mark.django_db -def test_linkify_mentions_local(identity, identity2, remote_identity): +def test_linkify_mentions_local( + config_system, emoji_locals, identity, identity2, remote_identity +): """ Tests that we can linkify post mentions properly for local use """ diff --git a/tests/conftest.py b/tests/conftest.py index f2b9d64..4466130 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import time import pytest +from activities.models import Emoji from api.models import Application, Token from core.models import Config from stator.runner import StatorModel, StatorRunner @@ -69,6 +70,16 @@ def config_system(keypair): @pytest.fixture @pytest.mark.django_db +def emoji_locals(): + Emoji.locals = Emoji.load_locals() + Emoji.__forced__ = True + yield Emoji.locals + Emoji.__forced__ = False + del Emoji.locals + + +@pytest.fixture +@pytest.mark.django_db def user() -> User: return User.objects.create(email="test@example.com") diff --git a/users/models/identity.py b/users/models/identity.py index 1239ca1..7898b4a 100644 --- a/users/models/identity.py +++ b/users/models/identity.py @@ -169,16 +169,20 @@ class Identity(StatorModel): @property def safe_summary(self): - return sanitize_post(self.summary) + from activities.templatetags.emoji_tags import imageify_emojis + + return imageify_emojis(sanitize_post(self.summary), self.domain) @property def safe_metadata(self): + from activities.templatetags.emoji_tags import imageify_emojis + if not self.metadata: return [] return [ { "name": data["name"], - "value": strip_html(data["value"]), + "value": imageify_emojis(strip_html(data["value"]), self.domain), } for data in self.metadata ] @@ -240,6 +244,15 @@ class Identity(StatorModel): def name_or_handle(self): return self.name or self.handle + @cached_property + def html_name_or_handle(self): + """ + Return the name_or_handle with any HTML substitutions made + """ + from activities.templatetags.emoji_tags import imageify_emojis + + return imageify_emojis(self.name_or_handle, self.domain) + @property def handle(self): if self.username is None: @@ -303,6 +316,17 @@ class Identity(StatorModel): } return response + def to_ap_tag(self): + """ + Return this Identity as an ActivityPub Tag + http://joinmastodon.org/ns#Mention + """ + return { + "href": self.actor_uri, + "name": "@" + self.handle, + "type": "Mention", + } + ### ActivityPub (inbound) ### @classmethod @@ -470,7 +494,15 @@ class Identity(StatorModel): ### Mastodon Client API ### def to_mastodon_json(self): + from activities.models import Emoji + header_image = self.local_image_url() + metadata_value_text = ( + " ".join([m["value"] for m in self.metadata]) if self.metadata else "" + ) + emojis = Emoji.emojis_from_content( + f"{self.name} {self.summary} {metadata_value_text}", self.domain + ) return { "id": self.pk, "username": self.username, @@ -491,7 +523,7 @@ class Identity(StatorModel): if self.metadata else [] ), - "emojis": [], + "emojis": [emoji.to_mastodon_json() for emoji in emojis], "bot": False, "group": False, "discoverable": self.discoverable, diff --git a/users/views/admin/settings.py b/users/views/admin/settings.py index a4e0190..b9e2543 100644 --- a/users/views/admin/settings.py +++ b/users/views/admin/settings.py @@ -85,6 +85,10 @@ class BasicSettings(AdminSettingsPage): "title": "Unreviewed Hashtags Are Public", "help_text": "Public Hashtags may appear in Trending and have a Tags timeline", }, + "emoji_unreviewed_are_public": { + "title": "Unreviewed Emoji Are Public", + "help_text": "Public Emoji may appear as images, instead of shortcodes", + }, } layout = { @@ -100,6 +104,7 @@ class BasicSettings(AdminSettingsPage): "post_length", "content_warning_text", "hashtag_unreviewed_are_public", + "emoji_unreviewed_are_public", ], "Identities": [ "identity_max_per_user", |
