summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.pre-commit-config.yaml2
-rw-r--r--activities/admin.py42
-rw-r--r--activities/middleware.py27
-rw-r--r--activities/migrations/0004_emoji_post_emojis.py91
-rw-r--r--activities/models/__init__.py1
-rw-r--r--activities/models/emoji.py261
-rw-r--r--activities/models/fan_out.py16
-rw-r--r--activities/models/post.py42
-rw-r--r--activities/templatetags/emoji_tags.py27
-rw-r--r--activities/views/posts.py2
-rw-r--r--activities/views/timelines.py16
-rw-r--r--core/files.py28
-rw-r--r--core/models/config.py2
-rw-r--r--core/uploads.py16
-rw-r--r--mediaproxy/views.py17
-rw-r--r--requirements.txt1
-rw-r--r--static/css/style.css27
-rw-r--r--static/img/blank-emoji-128.pngbin0 -> 194 bytes
-rw-r--r--takahe/settings.py6
-rw-r--r--takahe/urls.py5
-rw-r--r--templates/activities/_event.html8
-rw-r--r--templates/activities/_identity.html2
-rw-r--r--templates/activities/_mini_post.html2
-rw-r--r--templates/activities/_post.html2
-rw-r--r--templates/activities/follows.html2
-rw-r--r--templates/activities/home.html2
-rw-r--r--templates/activities/post.html2
-rw-r--r--templates/identity/select.html2
-rw-r--r--templates/identity/view.html3
-rw-r--r--tests/activities/models/test_post.py4
-rw-r--r--tests/conftest.py11
-rw-r--r--users/models/identity.py38
-rw-r--r--users/views/admin/settings.py5
33 files changed, 670 insertions, 42 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index d31e946..42057c1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -46,4 +46,4 @@ repos:
rev: v0.991
hooks:
- id: mypy
- additional_dependencies: [types-pyopenssl, types-bleach, types-mock]
+ additional_dependencies: [types-pyopenssl, types-bleach, types-mock, types-cachetools]
diff --git a/activities/admin.py b/activities/admin.py
index edc7365..6b0c8a9 100644
--- a/activities/admin.py
+++ b/activities/admin.py
@@ -1,8 +1,10 @@
from asgiref.sync import async_to_sync
from django.contrib import admin
+from django.utils.safestring import mark_safe
from django.utils.translation import gettext_lazy as _
from activities.models import (
+ Emoji,
FanOut,
Hashtag,
Post,
@@ -50,6 +52,46 @@ class HashtagAdmin(admin.ModelAdmin):
instance.transition_perform("outdated")
+@admin.register(Emoji)
+class EmojiAdmin(admin.ModelAdmin):
+ list_display = (
+ "shortcode",
+ "preview",
+ "local",
+ "domain",
+ "public",
+ "state",
+ "created",
+ )
+ list_filter = ("local", "public", "state")
+ search_fields = ("shortcode",)
+
+ readonly_fields = ("preview", "created", "updated")
+
+ actions = ["force_execution", "approve_emoji", "reject_emoji"]
+
+ @admin.action(description="Force Execution")
+ def force_execution(self, request, queryset):
+ for instance in queryset:
+ instance.transition_perform("outdated")
+
+ @admin.action(description="Approve Emoji")
+ def approve_emoji(self, request, queryset):
+ queryset.update(public=True)
+
+ @admin.action(description="Reject Emoji")
+ def reject_emoji(self, request, queryset):
+ queryset.update(public=False)
+
+ @admin.display(description="Emoji Preview")
+ def preview(self, instance):
+ if instance.public is False:
+ return mark_safe(f'<a href="{instance.full_url().relative}">Preview</a>')
+ return mark_safe(
+ f'<img src="{instance.full_url().relative}" style="height: 22px">'
+ )
+
+
@admin.register(PostAttachment)
class PostAttachmentAdmin(admin.ModelAdmin):
list_display = ["id", "post", "created"]
diff --git a/activities/middleware.py b/activities/middleware.py
new file mode 100644
index 0000000..1ed2219
--- /dev/null
+++ b/activities/middleware.py
@@ -0,0 +1,27 @@
+from time import time
+
+from activities.models import Emoji
+
+
+class EmojiDefaultsLoadingMiddleware:
+ """
+ Caches the default Emoji
+ """
+
+ refresh_interval: float = 30.0
+
+ def __init__(self, get_response):
+ self.get_response = get_response
+ self.loaded_ts: float = 0.0
+
+ def __call__(self, request):
+ # Allow test fixtures to force and lock the Emojis
+ if not getattr(Emoji, "__forced__", False):
+ if (
+ not getattr(Emoji, "locals", None)
+ or (time() - self.loaded_ts) >= self.refresh_interval
+ ):
+ Emoji.locals = Emoji.load_locals()
+ self.loaded_ts = time()
+ response = self.get_response(request)
+ return response
diff --git a/activities/migrations/0004_emoji_post_emojis.py b/activities/migrations/0004_emoji_post_emojis.py
new file mode 100644
index 0000000..89bc825
--- /dev/null
+++ b/activities/migrations/0004_emoji_post_emojis.py
@@ -0,0 +1,91 @@
+# Generated by Django 4.1.4 on 2022-12-14 23:49
+
+import functools
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import activities.models.emoji
+import core.uploads
+import stator.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("users", "0003_identity_followers_etc"),
+ ("activities", "0003_postattachment_null_thumb"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Emoji",
+ fields=[
+ (
+ "id",
+ models.BigAutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("state_ready", models.BooleanField(default=True)),
+ ("state_changed", models.DateTimeField(auto_now_add=True)),
+ ("state_attempted", models.DateTimeField(blank=True, null=True)),
+ ("state_locked_until", models.DateTimeField(blank=True, null=True)),
+ ("shortcode", models.SlugField(max_length=100)),
+ ("local", models.BooleanField(default=True)),
+ ("public", models.BooleanField(null=True)),
+ (
+ "object_uri",
+ models.CharField(
+ blank=True, max_length=500, null=True, unique=True
+ ),
+ ),
+ ("mimetype", models.CharField(max_length=200)),
+ (
+ "file",
+ models.ImageField(
+ blank=True,
+ null=True,
+ upload_to=functools.partial(
+ core.uploads.upload_emoji_namer, *("emoji",), **{}
+ ),
+ ),
+ ),
+ ("remote_url", models.CharField(blank=True, max_length=500, null=True)),
+ ("category", models.CharField(blank=True, max_length=100, null=True)),
+ (
+ "state",
+ stator.models.StateField(
+ choices=[("outdated", "outdated"), ("updated", "updated")],
+ default="outdated",
+ graph=activities.models.emoji.EmojiStates,
+ max_length=100,
+ ),
+ ),
+ ("created", models.DateTimeField(auto_now_add=True)),
+ ("updated", models.DateTimeField(auto_now=True)),
+ (
+ "domain",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="users.domain",
+ ),
+ ),
+ ],
+ options={
+ "unique_together": {("domain", "shortcode")},
+ },
+ ),
+ migrations.AddField(
+ model_name="post",
+ name="emojis",
+ field=models.ManyToManyField(
+ blank=True, related_name="posts_using_emoji", to="activities.emoji"
+ ),
+ ),
+ ]
diff --git a/activities/models/__init__.py b/activities/models/__init__.py
index aa34c0f..15a08a2 100644
--- a/activities/models/__init__.py
+++ b/activities/models/__init__.py
@@ -1,3 +1,4 @@
+from .emoji import Emoji, EmojiStates # noqa
from .fan_out import FanOut, FanOutStates # noqa
from .hashtag import Hashtag, HashtagStates # noqa
from .post import Post, PostStates # noqa
diff --git a/activities/models/emoji.py b/activities/models/emoji.py
new file mode 100644
index 0000000..00f6e67
--- /dev/null
+++ b/activities/models/emoji.py
@@ -0,0 +1,261 @@
+import re
+from functools import partial
+from typing import ClassVar, cast
+
+import urlman
+from asgiref.sync import sync_to_async
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.safestring import mark_safe
+
+from core.files import get_remote_file
+from core.html import strip_html
+from core.models import Config
+from core.uploads import upload_emoji_namer
+from core.uris import AutoAbsoluteUrl, RelativeAbsoluteUrl, StaticAbsoluteUrl
+from stator.models import State, StateField, StateGraph, StatorModel
+from users.models import Domain
+
+
+class EmojiStates(StateGraph):
+ outdated = State(try_interval=300, force_initial=True)
+ updated = State()
+
+ outdated.transitions_to(updated)
+
+ @classmethod
+ async def handle_outdated(cls, instance: "Emoji"):
+ """
+ Fetches remote emoji and uploads to file for local caching
+ """
+ if instance.remote_url and not instance.file:
+ file, mimetype = await get_remote_file(
+ instance.remote_url,
+ timeout=settings.SETUP.REMOTE_TIMEOUT,
+ max_size=settings.SETUP.EMOJI_MAX_IMAGE_FILESIZE_KB * 1024,
+ )
+ if file:
+ instance.file = file
+ instance.mimetype = mimetype
+ await sync_to_async(instance.save)()
+
+ return cls.updated
+
+
+class EmojiQuerySet(models.QuerySet):
+ def usable(self, domain: Domain | None = None):
+ public_q = models.Q(public=True)
+ if Config.system.emoji_unreviewed_are_public:
+ public_q |= models.Q(public__isnull=True)
+
+ qs = self.filter(public_q)
+ if domain:
+ if domain.local:
+ qs = qs.filter(local=True)
+ else:
+ qs = qs.filter(domain=domain)
+ return qs
+
+
+class EmojiManager(models.Manager):
+ def get_queryset(self):
+ return EmojiQuerySet(self.model, using=self._db)
+
+ def usable(self, domain: Domain | None = None):
+ return self.get_queryset().usable(domain)
+
+
+class Emoji(StatorModel):
+
+ # Normalized Emoji without the ':'
+ shortcode = models.SlugField(max_length=100, db_index=True)
+
+ domain = models.ForeignKey(
+ "users.Domain", null=True, blank=True, on_delete=models.CASCADE
+ )
+ local = models.BooleanField(default=True)
+
+ # Should this be shown in the public UI?
+ public = models.BooleanField(null=True)
+
+ object_uri = models.CharField(max_length=500, blank=True, null=True, unique=True)
+
+ mimetype = models.CharField(max_length=200)
+
+ # Files may not be populated if it's remote and not cached on our side yet
+ file = models.ImageField(
+ upload_to=partial(upload_emoji_namer, "emoji"),
+ null=True,
+ blank=True,
+ )
+
+ # A link to the custom emoji
+ remote_url = models.CharField(max_length=500, blank=True, null=True)
+
+ # Used for sorting custom emoji in the picker
+ category = models.CharField(max_length=100, blank=True, null=True)
+
+ # State of this Emoji
+ state = StateField(EmojiStates)
+
+ created = models.DateTimeField(auto_now_add=True)
+ updated = models.DateTimeField(auto_now=True)
+
+ objects = EmojiManager()
+
+ # Cache of the local emojis {shortcode: Emoji}
+ locals: ClassVar["dict[str, Emoji]"]
+
+ class Meta:
+ unique_together = ("domain", "shortcode")
+
+ class urls(urlman.Urls):
+ root = "/admin/emoji/"
+ create = "{root}/create/"
+ edit = "{root}{self.Emoji}/"
+ delete = "{edit}delete/"
+
+ emoji_regex = re.compile(r"\B:([a-zA-Z0-9(_)-]+):\B")
+
+ def clean(self):
+ super().clean()
+ if self.local ^ (self.domain is None):
+ raise ValidationError("Must be local or have a domain")
+
+ def __str__(self):
+ return f"{self.id}-{self.shortcode}"
+
+ @classmethod
+ def load_locals(cls) -> dict[str, "Emoji"]:
+ return {x.shortcode: x for x in Emoji.objects.usable().filter(local=True)}
+
+ @property
+ def fullcode(self):
+ return f":{self.shortcode}:"
+
+ @property
+ def is_usable(self) -> bool:
+ """
+ Return True if this Emoji is usable.
+ """
+ return self.public or (
+ self.public is None and Config.system.emoji_unreviewed_are_public
+ )
+
+ def full_url(self) -> RelativeAbsoluteUrl:
+ if self.is_usable:
+ if self.file:
+ return AutoAbsoluteUrl(self.file.url)
+ elif self.remote_url:
+ return AutoAbsoluteUrl(f"/proxy/emoji/{self.pk}/")
+ return StaticAbsoluteUrl("img/blank-emoji-128.png")
+
+ def as_html(self):
+ if self.is_usable:
+ return mark_safe(
+ f'<img src="{self.full_url().relative}" class="emoji" alt="Emoji {self.shortcode}">'
+ )
+ return self.fullcode
+
+ @classmethod
+ def imageify_emojis(
+ cls,
+ content: str,
+ *,
+ emojis: list["Emoji"] | EmojiQuerySet | None = None,
+ include_local: bool = True,
+ ):
+ """
+ Find :emoji: in content and convert to <img>. If include_local is True,
+ the local emoji will be used as a fallback for any shortcodes not defined
+ by emojis.
+ """
+ emoji_set = (
+ cast(list[Emoji], list(cls.locals.values())) if include_local else []
+ )
+
+ if emojis:
+ if isinstance(emojis, (EmojiQuerySet, list)):
+ emoji_set.extend(list(emojis))
+ else:
+ raise TypeError("Unsupported type for emojis")
+
+ possible_matches = {
+ emoji.shortcode: emoji.as_html() for emoji in emoji_set if emoji.is_usable
+ }
+
+ def replacer(match):
+ fullcode = match.group(1).lower()
+ if fullcode in possible_matches:
+ return possible_matches[fullcode]
+ return match.group()
+
+ return mark_safe(Emoji.emoji_regex.sub(replacer, content))
+
+ @classmethod
+ def emojis_from_content(cls, content: str, domain: Domain) -> list[str]:
+ """
+ Return a parsed and sanitized of emoji found in content without
+ the surrounding ':'.
+ """
+ emoji_hits = cls.emoji_regex.findall(strip_html(content))
+ emojis = sorted({emoji.lower() for emoji in emoji_hits})
+ return list(
+ cls.objects.filter(local=domain is None)
+ .usable(domain)
+ .filter(shortcode__in=emojis)
+ )
+
+ def to_ap_tag(self):
+ """
+ Return this Emoji as an ActivityPub Tag
+ http://joinmastodon.org/ns#Emoji
+ """
+ return {
+ "id": self.object_uri,
+ "type": "Emoji",
+ "name": self.shortcode,
+ "icon": {
+ "type": "Image",
+ "mediaType": self.mimetype,
+ "url": self.full_url().absolute,
+ },
+ }
+
+ @classmethod
+ def by_ap_tag(cls, domain: Domain, data: dict, create: bool = False):
+ """ """
+ try:
+ return cls.objects.get(object_uri=data["id"])
+ except cls.DoesNotExist:
+ if not create:
+ raise KeyError(f"No emoji with ID {data['id']}", data)
+
+ # create
+ shortcode = data["name"].lower().strip(":")
+ icon = data["icon"]
+ category = (icon.get("category") or "")[:100]
+ emoji = cls.objects.create(
+ shortcode=shortcode,
+ domain=None if domain.local else domain,
+ local=domain.local,
+ object_uri=data["id"],
+ mimetype=icon["mediaType"],
+ category=category,
+ remote_url=icon["url"],
+ )
+ return emoji
+
+ ### Mastodon API ###
+
+ def to_mastodon_json(self):
+ url = self.full_url().absolute
+ data = {
+ "shortcode": self.shortcode,
+ "url": url,
+ "static_url": self.remote_url or url,
+ "visible_in_picker": self.public,
+ "category": self.category or "",
+ }
+ return data
diff --git a/activities/models/fan_out.py b/activities/models/fan_out.py
index 8f4f342..fd52acd 100644
--- a/activities/models/fan_out.py
+++ b/activities/models/fan_out.py
@@ -184,8 +184,14 @@ class FanOut(StatorModel):
"""
Returns a version of the object with all relations pre-loaded
"""
- return await FanOut.objects.select_related(
- "identity",
- "subject_post",
- "subject_post_interaction",
- ).aget(pk=self.pk)
+ return (
+ await FanOut.objects.select_related(
+ "identity",
+ "subject_post",
+ "subject_post_interaction",
+ )
+ .prefetch_related(
+ "subject_post__emojis",
+ )
+ .aget(pk=self.pk)
+ )
diff --git a/activities/models/post.py b/activities/models/post.py
index 8e355bf..2b0a7c2 100644
--- a/activities/models/post.py
+++ b/activities/models/post.py
@@ -12,8 +12,10 @@ from django.template.defaultfilters import linebreaks_filter
from django.utils import timezone
from django.utils.safestring import mark_safe
+from activities.models.emoji import Emoji
from activities.models.fan_out import FanOut
from activities.models.hashtag import Hashtag
+from activities.templatetags.emoji_tags import imageify_emojis
from core.html import sanitize_post, strip_html
from core.ld import canonicalise, format_ld_date, get_list, parse_ld_date
from stator.models import State, StateField, StateGraph, StatorModel
@@ -218,6 +220,12 @@ class Post(StatorModel):
# Hashtags in the post
hashtags = models.JSONField(blank=True, null=True)
+ emojis = models.ManyToManyField(
+ "activities.Emoji",
+ related_name="posts_using_emoji",
+ blank=True,
+ )
+
# When the post was originally created (as opposed to when we received it)
published = models.DateTimeField(default=timezone.now)
@@ -328,8 +336,11 @@ class Post(StatorModel):
"""
Returns the content formatted for local display
"""
- return Hashtag.linkify_hashtags(
- self.linkify_mentions(sanitize_post(self.content), local=True)
+ return imageify_emojis(
+ Hashtag.linkify_hashtags(
+ self.linkify_mentions(sanitize_post(self.content), local=True)
+ ),
+ self.author.domain,
)
def safe_content_remote(self):
@@ -379,6 +390,8 @@ class Post(StatorModel):
visibility = reply_to.Visibilities.local_only
# Find hashtags in this post
hashtags = Hashtag.hashtags_from_content(content) or None
+ # Find emoji in this post
+ emojis = Emoji.emojis_from_content(content, author.domain)
# Strip all HTML and apply linebreaks filter
content = linebreaks_filter(strip_html(content))
# Make the Post object
@@ -395,6 +408,7 @@ class Post(StatorModel):
post.object_uri = post.urls.object_uri
post.url = post.absolute_object_uri()
post.mentions.set(mentions)
+ post.emojis.set(emojis)
if attachments:
post.attachments.set(attachments)
post.save()
@@ -416,6 +430,7 @@ class Post(StatorModel):
self.edited = timezone.now()
self.hashtags = Hashtag.hashtags_from_content(content) or None
self.mentions.set(self.mentions_from_content(content, self.author))
+ self.emojis.set(Emoji.emojis_from_content(content, self.author.domain))
self.attachments.set(attachments or [])
self.save()
@@ -520,14 +535,11 @@ class Post(StatorModel):
value["updated"] = format_ld_date(self.edited)
# Mentions
for mention in self.mentions.all():
- value["tag"].append(
- {
- "href": mention.actor_uri,
- "name": "@" + mention.handle,
- "type": "Mention",
- }
- )
+ value["tag"].append(mention.to_ap_tag())
value["cc"].append(mention.actor_uri)
+ # Emoji
+ for emoji in self.emojis.all():
+ value["tag"].append(emoji.to_ap_tag())
# Attachments
for attachment in self.attachments.all():
value["attachment"].append(attachment.to_ap())
@@ -616,7 +628,9 @@ class Post(StatorModel):
# Do we have one with the right ID?
created = False
try:
- post = cls.objects.get(object_uri=data["id"])
+ post = cls.objects.select_related("author__domain").get(
+ object_uri=data["id"]
+ )
except cls.DoesNotExist:
if create:
# Resolve the author
@@ -645,10 +659,10 @@ class Post(StatorModel):
mention_identity = Identity.by_actor_uri(tag["href"], create=True)
post.mentions.add(mention_identity)
elif tag["type"].lower() == "as:hashtag":
- post.hashtags.append(tag["name"].lstrip("#"))
+ post.hashtags.append(tag["name"].lower().lstrip("#"))
elif tag["type"].lower() == "http://joinmastodon.org/ns#emoji":
- # TODO: Handle incoming emoji
- pass
+ emoji = Emoji.by_ap_tag(post.author.domain, tag, create=True)
+ post.emojis.add(emoji)
else:
raise ValueError(f"Unknown tag type {tag['type']}")
# Visibility and to
@@ -818,7 +832,7 @@ class Post(StatorModel):
if self.hashtags
else []
),
- "emojis": [],
+ "emojis": [emoji.to_mastodon_json() for emoji in self.emojis.usable()],
"reblogs_count": self.interactions.filter(type="boost").count(),
"favourites_count": self.interactions.filter(type="like").count(),
"replies_count": 0,
diff --git a/activities/templatetags/emoji_tags.py b/activities/templatetags/emoji_tags.py
new file mode 100644
index 0000000..ad221db
--- /dev/null
+++ b/activities/templatetags/emoji_tags.py
@@ -0,0 +1,27 @@
+from cachetools import TTLCache, cached
+from django import template
+
+from activities.models import Emoji
+from users.models import Domain
+
+register = template.Library()
+
+
+@cached(cache=TTLCache(maxsize=1000, ttl=60))
+def emoji_from_domain(domain: Domain | None) -> list[Emoji]:
+ if not domain:
+ return list(Emoji.locals.values())
+ return list(Emoji.objects.usable(domain))
+
+
+@register.filter
+def imageify_emojis(value: str, arg: Domain | None = None):
+ """
+ Convert hashtags in content in to /tags/<hashtag>/ links.
+ """
+ if not value:
+ return ""
+
+ emojis = emoji_from_domain(arg)
+
+ return Emoji.imageify_emojis(value, emojis=emojis)
diff --git a/activities/views/posts.py b/activities/views/posts.py
index e285c7e..1b8676d 100644
--- a/activities/views/posts.py
+++ b/activities/views/posts.py
@@ -67,6 +67,8 @@ class Individual(TemplateView):
in_reply_to=self.post_obj.object_uri,
)
.distinct()
+ .select_related("author__domain")
+ .prefetch_related("emojis")
.order_by("published", "created"),
}
diff --git a/activities/views/timelines.py b/activities/views/timelines.py
index 0c1b693..f55e331 100644
--- a/activities/views/timelines.py
+++ b/activities/views/timelines.py
@@ -98,8 +98,8 @@ class Local(ListView):
def get_queryset(self):
return (
Post.objects.local_public()
- .select_related("author")
- .prefetch_related("attachments", "mentions")
+ .select_related("author", "author__domain")
+ .prefetch_related("attachments", "mentions", "emojis")
.order_by("-created")[:50]
)
@@ -126,8 +126,8 @@ class Federated(ListView):
Post.objects.filter(
visibility=Post.Visibilities.public, in_reply_to__isnull=True
)
- .select_related("author")
- .prefetch_related("attachments", "mentions")
+ .select_related("author", "author__domain")
+ .prefetch_related("attachments", "mentions", "emojis")
.order_by("-created")[:50]
)
@@ -173,7 +173,13 @@ class Notifications(ListView):
return (
TimelineEvent.objects.filter(identity=self.request.identity, type__in=types)
.order_by("-created")[:50]
- .select_related("subject_post", "subject_post__author", "subject_identity")
+ .select_related(
+ "subject_post",
+ "subject_post__author",
+ "subject_post__author__domain",
+ "subject_identity",
+ )
+ .prefetch_related("subject_post__emojis")
)
def get_context_data(self, **kwargs):
diff --git a/core/files.py b/core/files.py
index a04cef9..3ef79aa 100644
--- a/core/files.py
+++ b/core/files.py
@@ -1,7 +1,10 @@
import io
import blurhash
+import httpx
+from django.conf import settings
from django.core.files import File
+from django.core.files.base import ContentFile
from PIL import Image, ImageOps
@@ -37,3 +40,28 @@ def blurhash_image(file) -> str:
Returns the blurhash for an image
"""
return blurhash.encode(file, 4, 4)
+
+
+async def get_remote_file(
+ url: str,
+ *,
+ timeout: float = settings.SETUP.REMOTE_TIMEOUT,
+ max_size: int | None = None,
+) -> tuple[File | None, str | None]:
+ """
+ Download a URL and return the File and content-type.
+ """
+ async with httpx.AsyncClient() as client:
+ async with client.stream("GET", url, timeout=timeout) as stream:
+ allow_download = max_size is None
+ if max_size:
+ try:
+ content_length = int(stream.headers["content-length"])
+ allow_download = content_length <= max_size
+ except TypeError:
+ pass
+ if allow_download:
+ file = ContentFile(await stream.aread(), name=url)
+ return file, stream.headers["content-type"]
+
+ return None, None
diff --git a/core/models/config.py b/core/models/config.py
index 8f5dc31..9670c11 100644
--- a/core/models/config.py
+++ b/core/models/config.py
@@ -224,6 +224,8 @@ class Config(models.Model):
hashtag_unreviewed_are_public: bool = True
hashtag_stats_max_age: int = 60 * 60
+ emoji_unreviewed_are_public: bool = False
+
cache_timeout_page_default: int = 60
cache_timeout_page_timeline: int = 60 * 3
cache_timeout_page_post: int = 60 * 2
diff --git a/core/uploads.py b/core/uploads.py
index 41b6e94..f6c0e89 100644
--- a/core/uploads.py
+++ b/core/uploads.py
@@ -1,10 +1,14 @@
import os
import secrets
+from typing import TYPE_CHECKING
from django.utils import timezone
from storages.backends.gcloud import GoogleCloudStorage
from storages.backends.s3boto3 import S3Boto3Storage
+if TYPE_CHECKING:
+ from activities.models import Emoji
+
def upload_namer(prefix, instance, filename):
"""
@@ -16,6 +20,18 @@ def upload_namer(prefix, instance, filename):
return f"{prefix}/{now.year}/{now.month}/{now.day}/{new_filename}{old_extension}"
+def upload_emoji_namer(prefix, instance: "Emoji", filename):
+ """
+ Names uploaded emoji per domain
+ """
+ _, old_extension = os.path.splitext(filename)
+ if instance.domain is None:
+ domain = "_default"
+ else:
+ domain = instance.domain.domain
+ return f"{prefix}/{domain}/{instance.shortcode}{old_extension}"
+
+
class TakaheS3Storage(S3Boto3Storage):
"""
Custom override backend that makes webp files store correctly
diff --git a/mediaproxy/views.py b/mediaproxy/views.py
index 57257f3..4fc09b1 100644
--- a/mediaproxy/views.py
+++ b/mediaproxy/views.py
@@ -5,7 +5,7 @@ from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django.views.generic import View
-from activities.models import PostAttachment
+from activities.models import Emoji, PostAttachment
from users.models import Identity
@@ -57,6 +57,21 @@ class BaseCacheView(View):
raise NotImplementedError()
+class EmojiCacheView(BaseCacheView):
+ """
+ Caches Emoji
+ """
+
+ item_timeout = 86400 * 7 # One week
+
+ def get_remote_url(self):
+ self.emoji = get_object_or_404(Emoji, pk=self.kwargs["emoji_id"])
+
+ if not self.emoji.remote_url:
+ raise Http404()
+ return self.emoji.remote_url
+
+
class IdentityIconCacheView(BaseCacheView):
"""
Caches identity icons (avatars)
diff --git a/requirements.txt b/requirements.txt
index d24b45d..628b19a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
bleach~=5.0.1
blurhash-python~=1.1.3
+cachetools~=5.2.0
cryptography~=38.0
dj_database_url~=1.0.0
django-cache-url~=3.4.2
diff --git a/static/css/style.css b/static/css/style.css
index ce01793..9b509b8 100644
--- a/static/css/style.css
+++ b/static/css/style.css
@@ -358,6 +358,10 @@ nav a i {
width: auto;
}
+.icon-menu .option img.emoji {
+ height: 22px;
+}
+
.icon-menu .option i {
display: inline-block;
text-align: center;
@@ -740,6 +744,10 @@ h1.identity .icon {
margin: 0 20px 0 0;
}
+h1.identity .emoji {
+ height: 22px;
+}
+
h1.identity small {
display: block;
font-size: 60%;
@@ -752,6 +760,10 @@ h1.identity small {
margin: 0 0 20px 0;
}
+.bio .emoji {
+ height: 22px;
+}
+
.system-note {
background: var(--color-bg-menu);
color: var(--color-text-dull);
@@ -789,6 +801,10 @@ table.metadata td.name {
gap: 1em;
}
+table.metadata td .emoji {
+ height: 22px;
+}
+
/* Timelines */
.left-column .timeline-name {
@@ -857,6 +873,10 @@ table.metadata td.name {
float: left;
}
+.post .emoji {
+ height: 22px;
+}
+
.post .handle {
display: block;
padding: 7px 0 0 64px;
@@ -1014,6 +1034,13 @@ table.metadata td.name {
padding: 0 0 3px 5px;
}
+.boost-banner .emoji,
+.mention-banner .emoji,
+.follow-banner .emoji,
+.like-banner .emoji {
+ height: 22px;
+}
+
.boost-banner a,
.mention-banner a,
.follow-banner a,
diff --git a/static/img/blank-emoji-128.png b/static/img/blank-emoji-128.png
new file mode 100644
index 0000000..e1bc56c
--- /dev/null
+++ b/static/img/blank-emoji-128.png
Binary files differ
diff --git a/takahe/settings.py b/takahe/settings.py
index 91bfe7b..98bc9dd 100644
--- a/takahe/settings.py
+++ b/takahe/settings.py
@@ -107,6 +107,11 @@ class Settings(BaseSettings):
#: is necessary for compatibility with Mastodon’s image proxy.
MEDIA_MAX_IMAGE_FILESIZE_MB: int = 10
+ #: Maximum filesize for Emoji. Attempting to upload Local Emoji larger than this size will be
+ #: blocked. Remote Emoji larger than this size will not be fetched and served from media, but
+ #: served through the image proxy.
+ EMOJI_MAX_IMAGE_FILESIZE_KB: int = 200
+
#: Request timeouts to use when talking to other servers Either
#: float or tuple of floats for (connect, read, write, pool)
REMOTE_TIMEOUT: float | tuple[float, float, float, float] = 5.0
@@ -194,6 +199,7 @@ MIDDLEWARE = [
"core.middleware.ConfigLoadingMiddleware",
"api.middleware.ApiTokenMiddleware",
"users.middleware.IdentityMiddleware",
+ "activities.middleware.EmojiDefaultsLoadingMiddleware",
]
ROOT_URLCONF = "takahe.urls"
diff --git a/takahe/urls.py b/takahe/urls.py
index e8aa359..d6d893f 100644
--- a/takahe/urls.py
+++ b/takahe/urls.py
@@ -194,6 +194,11 @@ urlpatterns = [
mediaproxy.PostAttachmentCacheView.as_view(),
name="proxy_post_attachment",
),
+ path(
+ "proxy/emoji/<emoji_id>/",
+ mediaproxy.EmojiCacheView.as_view(),
+ name="proxy_emoji",
+ ),
# Well-known endpoints and system actor
path(".well-known/webfinger", activitypub.Webfinger.as_view()),
path(".well-known/host-meta", activitypub.HostMeta.as_view()),
diff --git a/templates/activities/_event.html b/templates/activities/_event.html
index 0c94aad..10d1ce4 100644
--- a/templates/activities/_event.html
+++ b/templates/activities/_event.html
@@ -3,14 +3,14 @@
{% if event.type == "followed" %}
<div class="follow-banner">
<a href="{{ event.subject_identity.urls.view }}">
- {{ event.subject_identity.name_or_handle }}
+ {{ event.subject_identity.html_name_or_handle }}
</a> followed you
</div>
{% include "activities/_identity.html" with identity=event.subject_identity created=event.created %}
{% elif event.type == "liked" %}
<div class="like-banner">
<a href="{{ event.subject_identity.urls.view }}">
- {{ event.subject_identity.name_or_handle }}
+ {{ event.subject_identity.html_name_or_handle }}
</a> liked your post
</div>
{% if not event.collapsed %}
@@ -19,7 +19,7 @@
{% elif event.type == "mentioned" %}
<div class="mention-banner">
<a href="{{ event.subject_identity.urls.view }}">
- {{ event.subject_identity.name_or_handle }}
+ {{ event.subject_identity.html_name_or_handle }}
</a> mentioned you
</div>
{% if not event.collapsed %}
@@ -28,7 +28,7 @@
{% elif event.type == "boosted" %}
<div class="boost-banner">
<a href="{{ event.subject_identity.urls.view }}">
- {{ event.subject_identity.name_or_handle }}
+ {{ event.subject_identity.html_name_or_handle }}
</a> boosted your post
</div>
{% if not event.collapsed %}
diff --git a/templates/activities/_identity.html b/templates/activities/_identity.html
index feb3178..791f05f 100644
--- a/templates/activities/_identity.html
+++ b/templates/activities/_identity.html
@@ -12,6 +12,6 @@
{% endif %}
<a href="{{ identity.urls.view }}" class="handle">
- {{ identity.name_or_handle }} <small>@{{ identity.handle }}</small>
+ {{ identity.html_name_or_handle }} <small>@{{ identity.handle }}</small>
</a>
</div>
diff --git a/templates/activities/_mini_post.html b/templates/activities/_mini_post.html
index 9f83333..335a8a4 100644
--- a/templates/activities/_mini_post.html
+++ b/templates/activities/_mini_post.html
@@ -7,7 +7,7 @@
</a>
<a href="{{ post.author.urls.view }}" class="handle">
- {{ post.author.name_or_handle }}
+ {{ post.author.html_name_or_handle }}
</a>
<div class="content">
diff --git a/templates/activities/_post.html b/templates/activities/_post.html
index 5802acd..5d75b78 100644
--- a/templates/activities/_post.html
+++ b/templates/activities/_post.html
@@ -59,7 +59,7 @@
{% endif %}
<a href="{{ post.author.urls.view }}" class="handle">
- {{ post.author.name_or_handle }} <small>@{{ post.author.handle }}</small>
+ {{ post.author.html_name_or_handle }} <small>@{{ post.author.handle }}</small>
</a>
{% if post.summary %}
diff --git a/templates/activities/follows.html b/templates/activities/follows.html
index 27a13d3..18c7811 100644
--- a/templates/activities/follows.html
+++ b/templates/activities/follows.html
@@ -8,7 +8,7 @@
<a class="option" href="{{ identity.urls.view }}">
<img src="{{ identity.local_icon_url.relative }}">
<span class="handle">
- {{ identity.name_or_handle }}
+ {{ identity.html_name_or_handle }}
<small>@{{ identity.handle }}</small>
</span>
{% if details.outbound %}
diff --git a/templates/activities/home.html b/templates/activities/home.html
index d574e2a..546da0d 100644
--- a/templates/activities/home.html
+++ b/templates/activities/home.html
@@ -10,7 +10,7 @@
{% elif event.type == "boost" %}
<div class="boost-banner">
<a href="{{ event.subject_identity.urls.view }}">
- {{ event.subject_identity.name_or_handle }}
+ {{ event.subject_identity.html_name_or_handle }}
</a> boosted
<time>
{{ event.subject_post_interaction.published | timedeltashort }} ago
diff --git a/templates/activities/post.html b/templates/activities/post.html
index fd717ad..6205064 100644
--- a/templates/activities/post.html
+++ b/templates/activities/post.html
@@ -1,6 +1,6 @@
{% extends "base.html" %}
-{% block title %}Post by {{ post.author.name_or_handle }}{% endblock %}
+{% block title %}Post by {{ post.author.html_name_or_handle }}{% endblock %}
{% block content %}
{% if parent %}
diff --git a/templates/identity/select.html b/templates/identity/select.html
index b3fc41d..5f73464 100644
--- a/templates/identity/select.html
+++ b/templates/identity/select.html
@@ -8,7 +8,7 @@
<a class="option" href="{{ identity.urls.activate }}">
<img src="{{ identity.local_icon_url.relative }}">
<span class="handle">
- {{ identity.name_or_handle }}
+ {{ identity.html_name_or_handle }}
<small>@{{ identity.handle }}</small>
</span>
</a>
diff --git a/templates/identity/view.html b/templates/identity/view.html
index 7e2c8d4..145a0ef 100644
--- a/templates/identity/view.html
+++ b/templates/identity/view.html
@@ -1,4 +1,5 @@
{% extends "base.html" %}
+{% load emoji_tags %}
{% block title %}{{ identity }}{% endblock %}
@@ -40,7 +41,7 @@
{% endif %}
{% endif %}
- {{ identity.name_or_handle }}
+ {{ identity.html_name_or_handle }}
<small>
@{{ identity.handle }}
<a title="Copy handle"
diff --git a/tests/activities/models/test_post.py b/tests/activities/models/test_post.py
index a5f2f79..ff74a9e 100644
--- a/tests/activities/models/test_post.py
+++ b/tests/activities/models/test_post.py
@@ -100,7 +100,9 @@ def test_linkify_mentions_remote(
@pytest.mark.django_db
-def test_linkify_mentions_local(identity, identity2, remote_identity):
+def test_linkify_mentions_local(
+ config_system, emoji_locals, identity, identity2, remote_identity
+):
"""
Tests that we can linkify post mentions properly for local use
"""
diff --git a/tests/conftest.py b/tests/conftest.py
index f2b9d64..4466130 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,6 +2,7 @@ import time
import pytest
+from activities.models import Emoji
from api.models import Application, Token
from core.models import Config
from stator.runner import StatorModel, StatorRunner
@@ -69,6 +70,16 @@ def config_system(keypair):
@pytest.fixture
@pytest.mark.django_db
+def emoji_locals():
+ Emoji.locals = Emoji.load_locals()
+ Emoji.__forced__ = True
+ yield Emoji.locals
+ Emoji.__forced__ = False
+ del Emoji.locals
+
+
+@pytest.fixture
+@pytest.mark.django_db
def user() -> User:
return User.objects.create(email="test@example.com")
diff --git a/users/models/identity.py b/users/models/identity.py
index 1239ca1..7898b4a 100644
--- a/users/models/identity.py
+++ b/users/models/identity.py
@@ -169,16 +169,20 @@ class Identity(StatorModel):
@property
def safe_summary(self):
- return sanitize_post(self.summary)
+ from activities.templatetags.emoji_tags import imageify_emojis
+
+ return imageify_emojis(sanitize_post(self.summary), self.domain)
@property
def safe_metadata(self):
+ from activities.templatetags.emoji_tags import imageify_emojis
+
if not self.metadata:
return []
return [
{
"name": data["name"],
- "value": strip_html(data["value"]),
+ "value": imageify_emojis(strip_html(data["value"]), self.domain),
}
for data in self.metadata
]
@@ -240,6 +244,15 @@ class Identity(StatorModel):
def name_or_handle(self):
return self.name or self.handle
+ @cached_property
+ def html_name_or_handle(self):
+ """
+ Return the name_or_handle with any HTML substitutions made
+ """
+ from activities.templatetags.emoji_tags import imageify_emojis
+
+ return imageify_emojis(self.name_or_handle, self.domain)
+
@property
def handle(self):
if self.username is None:
@@ -303,6 +316,17 @@ class Identity(StatorModel):
}
return response
+ def to_ap_tag(self):
+ """
+ Return this Identity as an ActivityPub Tag
+ http://joinmastodon.org/ns#Mention
+ """
+ return {
+ "href": self.actor_uri,
+ "name": "@" + self.handle,
+ "type": "Mention",
+ }
+
### ActivityPub (inbound) ###
@classmethod
@@ -470,7 +494,15 @@ class Identity(StatorModel):
### Mastodon Client API ###
def to_mastodon_json(self):
+ from activities.models import Emoji
+
header_image = self.local_image_url()
+ metadata_value_text = (
+ " ".join([m["value"] for m in self.metadata]) if self.metadata else ""
+ )
+ emojis = Emoji.emojis_from_content(
+ f"{self.name} {self.summary} {metadata_value_text}", self.domain
+ )
return {
"id": self.pk,
"username": self.username,
@@ -491,7 +523,7 @@ class Identity(StatorModel):
if self.metadata
else []
),
- "emojis": [],
+ "emojis": [emoji.to_mastodon_json() for emoji in emojis],
"bot": False,
"group": False,
"discoverable": self.discoverable,
diff --git a/users/views/admin/settings.py b/users/views/admin/settings.py
index a4e0190..b9e2543 100644
--- a/users/views/admin/settings.py
+++ b/users/views/admin/settings.py
@@ -85,6 +85,10 @@ class BasicSettings(AdminSettingsPage):
"title": "Unreviewed Hashtags Are Public",
"help_text": "Public Hashtags may appear in Trending and have a Tags timeline",
},
+ "emoji_unreviewed_are_public": {
+ "title": "Unreviewed Emoji Are Public",
+ "help_text": "Public Emoji may appear as images, instead of shortcodes",
+ },
}
layout = {
@@ -100,6 +104,7 @@ class BasicSettings(AdminSettingsPage):
"post_length",
"content_warning_text",
"hashtag_unreviewed_are_public",
+ "emoji_unreviewed_are_public",
],
"Identities": [
"identity_max_per_user",