diff options
Diffstat (limited to 'activities')
| -rw-r--r-- | activities/admin.py | 42 | ||||
| -rw-r--r-- | activities/middleware.py | 27 | ||||
| -rw-r--r-- | activities/migrations/0004_emoji_post_emojis.py | 91 | ||||
| -rw-r--r-- | activities/models/__init__.py | 1 | ||||
| -rw-r--r-- | activities/models/emoji.py | 261 | ||||
| -rw-r--r-- | activities/models/fan_out.py | 16 | ||||
| -rw-r--r-- | activities/models/post.py | 42 | ||||
| -rw-r--r-- | activities/templatetags/emoji_tags.py | 27 | ||||
| -rw-r--r-- | activities/views/posts.py | 2 | ||||
| -rw-r--r-- | activities/views/timelines.py | 16 |
10 files changed, 501 insertions, 24 deletions
diff --git a/activities/admin.py b/activities/admin.py index edc7365..6b0c8a9 100644 --- a/activities/admin.py +++ b/activities/admin.py @@ -1,8 +1,10 @@ from asgiref.sync import async_to_sync from django.contrib import admin +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ from activities.models import ( + Emoji, FanOut, Hashtag, Post, @@ -50,6 +52,46 @@ class HashtagAdmin(admin.ModelAdmin): instance.transition_perform("outdated") +@admin.register(Emoji) +class EmojiAdmin(admin.ModelAdmin): + list_display = ( + "shortcode", + "preview", + "local", + "domain", + "public", + "state", + "created", + ) + list_filter = ("local", "public", "state") + search_fields = ("shortcode",) + + readonly_fields = ("preview", "created", "updated") + + actions = ["force_execution", "approve_emoji", "reject_emoji"] + + @admin.action(description="Force Execution") + def force_execution(self, request, queryset): + for instance in queryset: + instance.transition_perform("outdated") + + @admin.action(description="Approve Emoji") + def approve_emoji(self, request, queryset): + queryset.update(public=True) + + @admin.action(description="Reject Emoji") + def reject_emoji(self, request, queryset): + queryset.update(public=False) + + @admin.display(description="Emoji Preview") + def preview(self, instance): + if instance.public is False: + return mark_safe(f'<a href="{instance.full_url().relative}">Preview</a>') + return mark_safe( + f'<img src="{instance.full_url().relative}" style="height: 22px">' + ) + + @admin.register(PostAttachment) class PostAttachmentAdmin(admin.ModelAdmin): list_display = ["id", "post", "created"] diff --git a/activities/middleware.py b/activities/middleware.py new file mode 100644 index 0000000..1ed2219 --- /dev/null +++ b/activities/middleware.py @@ -0,0 +1,27 @@ +from time import time + +from activities.models import Emoji + + +class EmojiDefaultsLoadingMiddleware: + """ + Caches the default Emoji + """ + + refresh_interval: float = 30.0 + + def __init__(self, get_response): + self.get_response = get_response + self.loaded_ts: float = 0.0 + + def __call__(self, request): + # Allow test fixtures to force and lock the Emojis + if not getattr(Emoji, "__forced__", False): + if ( + not getattr(Emoji, "locals", None) + or (time() - self.loaded_ts) >= self.refresh_interval + ): + Emoji.locals = Emoji.load_locals() + self.loaded_ts = time() + response = self.get_response(request) + return response diff --git a/activities/migrations/0004_emoji_post_emojis.py b/activities/migrations/0004_emoji_post_emojis.py new file mode 100644 index 0000000..89bc825 --- /dev/null +++ b/activities/migrations/0004_emoji_post_emojis.py @@ -0,0 +1,91 @@ +# Generated by Django 4.1.4 on 2022-12-14 23:49 + +import functools + +import django.db.models.deletion +from django.db import migrations, models + +import activities.models.emoji +import core.uploads +import stator.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0003_identity_followers_etc"), + ("activities", "0003_postattachment_null_thumb"), + ] + + operations = [ + migrations.CreateModel( + name="Emoji", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("state_ready", models.BooleanField(default=True)), + ("state_changed", models.DateTimeField(auto_now_add=True)), + ("state_attempted", models.DateTimeField(blank=True, null=True)), + ("state_locked_until", models.DateTimeField(blank=True, null=True)), + ("shortcode", models.SlugField(max_length=100)), + ("local", models.BooleanField(default=True)), + ("public", models.BooleanField(null=True)), + ( + "object_uri", + models.CharField( + blank=True, max_length=500, null=True, unique=True + ), + ), + ("mimetype", models.CharField(max_length=200)), + ( + "file", + models.ImageField( + blank=True, + null=True, + upload_to=functools.partial( + core.uploads.upload_emoji_namer, *("emoji",), **{} + ), + ), + ), + ("remote_url", models.CharField(blank=True, max_length=500, null=True)), + ("category", models.CharField(blank=True, max_length=100, null=True)), + ( + "state", + stator.models.StateField( + choices=[("outdated", "outdated"), ("updated", "updated")], + default="outdated", + graph=activities.models.emoji.EmojiStates, + max_length=100, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "domain", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="users.domain", + ), + ), + ], + options={ + "unique_together": {("domain", "shortcode")}, + }, + ), + migrations.AddField( + model_name="post", + name="emojis", + field=models.ManyToManyField( + blank=True, related_name="posts_using_emoji", to="activities.emoji" + ), + ), + ] diff --git a/activities/models/__init__.py b/activities/models/__init__.py index aa34c0f..15a08a2 100644 --- a/activities/models/__init__.py +++ b/activities/models/__init__.py @@ -1,3 +1,4 @@ +from .emoji import Emoji, EmojiStates # noqa from .fan_out import FanOut, FanOutStates # noqa from .hashtag import Hashtag, HashtagStates # noqa from .post import Post, PostStates # noqa diff --git a/activities/models/emoji.py b/activities/models/emoji.py new file mode 100644 index 0000000..00f6e67 --- /dev/null +++ b/activities/models/emoji.py @@ -0,0 +1,261 @@ +import re +from functools import partial +from typing import ClassVar, cast + +import urlman +from asgiref.sync import sync_to_async +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.safestring import mark_safe + +from core.files import get_remote_file +from core.html import strip_html +from core.models import Config +from core.uploads import upload_emoji_namer +from core.uris import AutoAbsoluteUrl, RelativeAbsoluteUrl, StaticAbsoluteUrl +from stator.models import State, StateField, StateGraph, StatorModel +from users.models import Domain + + +class EmojiStates(StateGraph): + outdated = State(try_interval=300, force_initial=True) + updated = State() + + outdated.transitions_to(updated) + + @classmethod + async def handle_outdated(cls, instance: "Emoji"): + """ + Fetches remote emoji and uploads to file for local caching + """ + if instance.remote_url and not instance.file: + file, mimetype = await get_remote_file( + instance.remote_url, + timeout=settings.SETUP.REMOTE_TIMEOUT, + max_size=settings.SETUP.EMOJI_MAX_IMAGE_FILESIZE_KB * 1024, + ) + if file: + instance.file = file + instance.mimetype = mimetype + await sync_to_async(instance.save)() + + return cls.updated + + +class EmojiQuerySet(models.QuerySet): + def usable(self, domain: Domain | None = None): + public_q = models.Q(public=True) + if Config.system.emoji_unreviewed_are_public: + public_q |= models.Q(public__isnull=True) + + qs = self.filter(public_q) + if domain: + if domain.local: + qs = qs.filter(local=True) + else: + qs = qs.filter(domain=domain) + return qs + + +class EmojiManager(models.Manager): + def get_queryset(self): + return EmojiQuerySet(self.model, using=self._db) + + def usable(self, domain: Domain | None = None): + return self.get_queryset().usable(domain) + + +class Emoji(StatorModel): + + # Normalized Emoji without the ':' + shortcode = models.SlugField(max_length=100, db_index=True) + + domain = models.ForeignKey( + "users.Domain", null=True, blank=True, on_delete=models.CASCADE + ) + local = models.BooleanField(default=True) + + # Should this be shown in the public UI? + public = models.BooleanField(null=True) + + object_uri = models.CharField(max_length=500, blank=True, null=True, unique=True) + + mimetype = models.CharField(max_length=200) + + # Files may not be populated if it's remote and not cached on our side yet + file = models.ImageField( + upload_to=partial(upload_emoji_namer, "emoji"), + null=True, + blank=True, + ) + + # A link to the custom emoji + remote_url = models.CharField(max_length=500, blank=True, null=True) + + # Used for sorting custom emoji in the picker + category = models.CharField(max_length=100, blank=True, null=True) + + # State of this Emoji + state = StateField(EmojiStates) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + objects = EmojiManager() + + # Cache of the local emojis {shortcode: Emoji} + locals: ClassVar["dict[str, Emoji]"] + + class Meta: + unique_together = ("domain", "shortcode") + + class urls(urlman.Urls): + root = "/admin/emoji/" + create = "{root}/create/" + edit = "{root}{self.Emoji}/" + delete = "{edit}delete/" + + emoji_regex = re.compile(r"\B:([a-zA-Z0-9(_)-]+):\B") + + def clean(self): + super().clean() + if self.local ^ (self.domain is None): + raise ValidationError("Must be local or have a domain") + + def __str__(self): + return f"{self.id}-{self.shortcode}" + + @classmethod + def load_locals(cls) -> dict[str, "Emoji"]: + return {x.shortcode: x for x in Emoji.objects.usable().filter(local=True)} + + @property + def fullcode(self): + return f":{self.shortcode}:" + + @property + def is_usable(self) -> bool: + """ + Return True if this Emoji is usable. + """ + return self.public or ( + self.public is None and Config.system.emoji_unreviewed_are_public + ) + + def full_url(self) -> RelativeAbsoluteUrl: + if self.is_usable: + if self.file: + return AutoAbsoluteUrl(self.file.url) + elif self.remote_url: + return AutoAbsoluteUrl(f"/proxy/emoji/{self.pk}/") + return StaticAbsoluteUrl("img/blank-emoji-128.png") + + def as_html(self): + if self.is_usable: + return mark_safe( + f'<img src="{self.full_url().relative}" class="emoji" alt="Emoji {self.shortcode}">' + ) + return self.fullcode + + @classmethod + def imageify_emojis( + cls, + content: str, + *, + emojis: list["Emoji"] | EmojiQuerySet | None = None, + include_local: bool = True, + ): + """ + Find :emoji: in content and convert to <img>. If include_local is True, + the local emoji will be used as a fallback for any shortcodes not defined + by emojis. + """ + emoji_set = ( + cast(list[Emoji], list(cls.locals.values())) if include_local else [] + ) + + if emojis: + if isinstance(emojis, (EmojiQuerySet, list)): + emoji_set.extend(list(emojis)) + else: + raise TypeError("Unsupported type for emojis") + + possible_matches = { + emoji.shortcode: emoji.as_html() for emoji in emoji_set if emoji.is_usable + } + + def replacer(match): + fullcode = match.group(1).lower() + if fullcode in possible_matches: + return possible_matches[fullcode] + return match.group() + + return mark_safe(Emoji.emoji_regex.sub(replacer, content)) + + @classmethod + def emojis_from_content(cls, content: str, domain: Domain) -> list[str]: + """ + Return a parsed and sanitized of emoji found in content without + the surrounding ':'. + """ + emoji_hits = cls.emoji_regex.findall(strip_html(content)) + emojis = sorted({emoji.lower() for emoji in emoji_hits}) + return list( + cls.objects.filter(local=domain is None) + .usable(domain) + .filter(shortcode__in=emojis) + ) + + def to_ap_tag(self): + """ + Return this Emoji as an ActivityPub Tag + http://joinmastodon.org/ns#Emoji + """ + return { + "id": self.object_uri, + "type": "Emoji", + "name": self.shortcode, + "icon": { + "type": "Image", + "mediaType": self.mimetype, + "url": self.full_url().absolute, + }, + } + + @classmethod + def by_ap_tag(cls, domain: Domain, data: dict, create: bool = False): + """ """ + try: + return cls.objects.get(object_uri=data["id"]) + except cls.DoesNotExist: + if not create: + raise KeyError(f"No emoji with ID {data['id']}", data) + + # create + shortcode = data["name"].lower().strip(":") + icon = data["icon"] + category = (icon.get("category") or "")[:100] + emoji = cls.objects.create( + shortcode=shortcode, + domain=None if domain.local else domain, + local=domain.local, + object_uri=data["id"], + mimetype=icon["mediaType"], + category=category, + remote_url=icon["url"], + ) + return emoji + + ### Mastodon API ### + + def to_mastodon_json(self): + url = self.full_url().absolute + data = { + "shortcode": self.shortcode, + "url": url, + "static_url": self.remote_url or url, + "visible_in_picker": self.public, + "category": self.category or "", + } + return data diff --git a/activities/models/fan_out.py b/activities/models/fan_out.py index 8f4f342..fd52acd 100644 --- a/activities/models/fan_out.py +++ b/activities/models/fan_out.py @@ -184,8 +184,14 @@ class FanOut(StatorModel): """ Returns a version of the object with all relations pre-loaded """ - return await FanOut.objects.select_related( - "identity", - "subject_post", - "subject_post_interaction", - ).aget(pk=self.pk) + return ( + await FanOut.objects.select_related( + "identity", + "subject_post", + "subject_post_interaction", + ) + .prefetch_related( + "subject_post__emojis", + ) + .aget(pk=self.pk) + ) diff --git a/activities/models/post.py b/activities/models/post.py index 8e355bf..2b0a7c2 100644 --- a/activities/models/post.py +++ b/activities/models/post.py @@ -12,8 +12,10 @@ from django.template.defaultfilters import linebreaks_filter from django.utils import timezone from django.utils.safestring import mark_safe +from activities.models.emoji import Emoji from activities.models.fan_out import FanOut from activities.models.hashtag import Hashtag +from activities.templatetags.emoji_tags import imageify_emojis from core.html import sanitize_post, strip_html from core.ld import canonicalise, format_ld_date, get_list, parse_ld_date from stator.models import State, StateField, StateGraph, StatorModel @@ -218,6 +220,12 @@ class Post(StatorModel): # Hashtags in the post hashtags = models.JSONField(blank=True, null=True) + emojis = models.ManyToManyField( + "activities.Emoji", + related_name="posts_using_emoji", + blank=True, + ) + # When the post was originally created (as opposed to when we received it) published = models.DateTimeField(default=timezone.now) @@ -328,8 +336,11 @@ class Post(StatorModel): """ Returns the content formatted for local display """ - return Hashtag.linkify_hashtags( - self.linkify_mentions(sanitize_post(self.content), local=True) + return imageify_emojis( + Hashtag.linkify_hashtags( + self.linkify_mentions(sanitize_post(self.content), local=True) + ), + self.author.domain, ) def safe_content_remote(self): @@ -379,6 +390,8 @@ class Post(StatorModel): visibility = reply_to.Visibilities.local_only # Find hashtags in this post hashtags = Hashtag.hashtags_from_content(content) or None + # Find emoji in this post + emojis = Emoji.emojis_from_content(content, author.domain) # Strip all HTML and apply linebreaks filter content = linebreaks_filter(strip_html(content)) # Make the Post object @@ -395,6 +408,7 @@ class Post(StatorModel): post.object_uri = post.urls.object_uri post.url = post.absolute_object_uri() post.mentions.set(mentions) + post.emojis.set(emojis) if attachments: post.attachments.set(attachments) post.save() @@ -416,6 +430,7 @@ class Post(StatorModel): self.edited = timezone.now() self.hashtags = Hashtag.hashtags_from_content(content) or None self.mentions.set(self.mentions_from_content(content, self.author)) + self.emojis.set(Emoji.emojis_from_content(content, self.author.domain)) self.attachments.set(attachments or []) self.save() @@ -520,14 +535,11 @@ class Post(StatorModel): value["updated"] = format_ld_date(self.edited) # Mentions for mention in self.mentions.all(): - value["tag"].append( - { - "href": mention.actor_uri, - "name": "@" + mention.handle, - "type": "Mention", - } - ) + value["tag"].append(mention.to_ap_tag()) value["cc"].append(mention.actor_uri) + # Emoji + for emoji in self.emojis.all(): + value["tag"].append(emoji.to_ap_tag()) # Attachments for attachment in self.attachments.all(): value["attachment"].append(attachment.to_ap()) @@ -616,7 +628,9 @@ class Post(StatorModel): # Do we have one with the right ID? created = False try: - post = cls.objects.get(object_uri=data["id"]) + post = cls.objects.select_related("author__domain").get( + object_uri=data["id"] + ) except cls.DoesNotExist: if create: # Resolve the author @@ -645,10 +659,10 @@ class Post(StatorModel): mention_identity = Identity.by_actor_uri(tag["href"], create=True) post.mentions.add(mention_identity) elif tag["type"].lower() == "as:hashtag": - post.hashtags.append(tag["name"].lstrip("#")) + post.hashtags.append(tag["name"].lower().lstrip("#")) elif tag["type"].lower() == "http://joinmastodon.org/ns#emoji": - # TODO: Handle incoming emoji - pass + emoji = Emoji.by_ap_tag(post.author.domain, tag, create=True) + post.emojis.add(emoji) else: raise ValueError(f"Unknown tag type {tag['type']}") # Visibility and to @@ -818,7 +832,7 @@ class Post(StatorModel): if self.hashtags else [] ), - "emojis": [], + "emojis": [emoji.to_mastodon_json() for emoji in self.emojis.usable()], "reblogs_count": self.interactions.filter(type="boost").count(), "favourites_count": self.interactions.filter(type="like").count(), "replies_count": 0, diff --git a/activities/templatetags/emoji_tags.py b/activities/templatetags/emoji_tags.py new file mode 100644 index 0000000..ad221db --- /dev/null +++ b/activities/templatetags/emoji_tags.py @@ -0,0 +1,27 @@ +from cachetools import TTLCache, cached +from django import template + +from activities.models import Emoji +from users.models import Domain + +register = template.Library() + + +@cached(cache=TTLCache(maxsize=1000, ttl=60)) +def emoji_from_domain(domain: Domain | None) -> list[Emoji]: + if not domain: + return list(Emoji.locals.values()) + return list(Emoji.objects.usable(domain)) + + +@register.filter +def imageify_emojis(value: str, arg: Domain | None = None): + """ + Convert hashtags in content in to /tags/<hashtag>/ links. + """ + if not value: + return "" + + emojis = emoji_from_domain(arg) + + return Emoji.imageify_emojis(value, emojis=emojis) diff --git a/activities/views/posts.py b/activities/views/posts.py index e285c7e..1b8676d 100644 --- a/activities/views/posts.py +++ b/activities/views/posts.py @@ -67,6 +67,8 @@ class Individual(TemplateView): in_reply_to=self.post_obj.object_uri, ) .distinct() + .select_related("author__domain") + .prefetch_related("emojis") .order_by("published", "created"), } diff --git a/activities/views/timelines.py b/activities/views/timelines.py index 0c1b693..f55e331 100644 --- a/activities/views/timelines.py +++ b/activities/views/timelines.py @@ -98,8 +98,8 @@ class Local(ListView): def get_queryset(self): return ( Post.objects.local_public() - .select_related("author") - .prefetch_related("attachments", "mentions") + .select_related("author", "author__domain") + .prefetch_related("attachments", "mentions", "emojis") .order_by("-created")[:50] ) @@ -126,8 +126,8 @@ class Federated(ListView): Post.objects.filter( visibility=Post.Visibilities.public, in_reply_to__isnull=True ) - .select_related("author") - .prefetch_related("attachments", "mentions") + .select_related("author", "author__domain") + .prefetch_related("attachments", "mentions", "emojis") .order_by("-created")[:50] ) @@ -173,7 +173,13 @@ class Notifications(ListView): return ( TimelineEvent.objects.filter(identity=self.request.identity, type__in=types) .order_by("-created")[:50] - .select_related("subject_post", "subject_post__author", "subject_identity") + .select_related( + "subject_post", + "subject_post__author", + "subject_post__author__domain", + "subject_identity", + ) + .prefetch_related("subject_post__emojis") ) def get_context_data(self, **kwargs): |