Audio federation

api/funkwhale_api/federation/activity.py

@@ -2,11 +2,12 @@ import uuid
 import logging
 
 from django.db import transaction, IntegrityError
-from django.utils import timezone
+from django.db.models import Q
 
 from funkwhale_api.common import channels
 from funkwhale_api.common import utils as funkwhale_utils
 
+
 logger = logging.getLogger(__name__)
 PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
 
@@ -83,18 +84,21 @@ def receive(activity, on_behalf_of):
             serializer.validated_data.get("id"),
         )
         return
-    # we create inbox items for further delivery
-    items = [
-        models.InboxItem(activity=copy, actor=r, type="to")
-        for r in serializer.validated_data["recipients"]["to"]
-        if hasattr(r, "fid")
-    ]
-    items += [
-        models.InboxItem(activity=copy, actor=r, type="cc")
-        for r in serializer.validated_data["recipients"]["cc"]
-        if hasattr(r, "fid")
-    ]
-    models.InboxItem.objects.bulk_create(items)
+
+    local_to_recipients = get_actors_from_audience(activity.get("to", []))
+    local_to_recipients = local_to_recipients.exclude(user=None)
+
+    local_cc_recipients = get_actors_from_audience(activity.get("cc", []))
+    local_cc_recipients = local_cc_recipients.exclude(user=None)
+
+    inbox_items = []
+    for recipients, type in [(local_to_recipients, "to"), (local_cc_recipients, "cc")]:
+
+        for r in recipients.values_list("pk", flat=True):
+            inbox_items.append(models.InboxItem(actor_id=r, type=type, activity=copy))
+
+    models.InboxItem.objects.bulk_create(inbox_items)
+
     # at this point, we have the activity in database. Even if we crash, it's
     # okay, as we can retry later
     funkwhale_utils.on_commit(tasks.dispatch_inbox.delay, activity_id=copy.pk)
@@ -153,6 +157,16 @@ class InboxRouter(Router):
                 inbox_items = context.get(
                     "inbox_items", models.InboxItem.objects.none()
                 )
+                inbox_items = (
+                    inbox_items.select_related()
+                    .select_related("actor__user")
+                    .prefetch_related(
+                        "activity__object",
+                        "activity__target",
+                        "activity__related_object",
+                    )
+                )
+
                 for ii in inbox_items:
                     user = ii.actor.get_user()
                     if not user:
@@ -169,7 +183,6 @@ class InboxRouter(Router):
                             },
                         },
                     )
-                inbox_items.update(is_delivered=True, last_delivery_date=timezone.now())
                 return
 
 
@@ -185,73 +198,203 @@ class OutboxRouter(Router):
         from . import tasks
 
         for route, handler in self.routes:
-            if match_route(route, routing):
-                activities_data = []
-                for e in handler(context):
-                    # a route can yield zero, one or more activity payloads
-                    if e:
-                        activities_data.append(e)
-                inbox_items_by_activity_uuid = {}
-                prepared_activities = []
-                for activity_data in activities_data:
-                    to = activity_data["payload"].pop("to", [])
-                    cc = activity_data["payload"].pop("cc", [])
-                    a = models.Activity(**activity_data)
-                    a.uuid = uuid.uuid4()
-                    to_items, new_to = prepare_inbox_items(to, "to")
-                    cc_items, new_cc = prepare_inbox_items(cc, "cc")
-                    if not to_items and not cc_items:
-                        continue
-                    inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
-                    if new_to:
-                        a.payload["to"] = new_to
-                    if new_cc:
-                        a.payload["cc"] = new_cc
-                    prepared_activities.append(a)
+            if not match_route(route, routing):
+                continue
 
-                activities = models.Activity.objects.bulk_create(prepared_activities)
+            activities_data = []
+            for e in handler(context):
+                # a route can yield zero, one or more activity payloads
+                if e:
+                    activities_data.append(e)
+            inbox_items_by_activity_uuid = {}
+            deliveries_by_activity_uuid = {}
+            prepared_activities = []
+            for activity_data in activities_data:
+                activity_data["payload"]["actor"] = activity_data["actor"].fid
+                to = activity_data["payload"].pop("to", [])
+                cc = activity_data["payload"].pop("cc", [])
+                a = models.Activity(**activity_data)
+                a.uuid = uuid.uuid4()
+                to_inbox_items, to_deliveries, new_to = prepare_deliveries_and_inbox_items(
+                    to, "to"
+                )
+                cc_inbox_items, cc_deliveries, new_cc = prepare_deliveries_and_inbox_items(
+                    cc, "cc"
+                )
+                if not any(
+                    [to_inbox_items, to_deliveries, cc_inbox_items, cc_deliveries]
+                ):
+                    continue
+                deliveries_by_activity_uuid[str(a.uuid)] = to_deliveries + cc_deliveries
+                inbox_items_by_activity_uuid[str(a.uuid)] = (
+                    to_inbox_items + cc_inbox_items
+                )
+                if new_to:
+                    a.payload["to"] = new_to
+                if new_cc:
+                    a.payload["cc"] = new_cc
+                prepared_activities.append(a)
 
-                final_inbox_items = []
-                for a in activities:
-                    try:
-                        prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
-                    except KeyError:
-                        continue
+            activities = models.Activity.objects.bulk_create(prepared_activities)
 
-                    for ii in prepared_inbox_items:
-                        ii.activity = a
-                        final_inbox_items.append(ii)
+            for activity in activities:
+                if str(activity.uuid) in deliveries_by_activity_uuid:
+                    for obj in deliveries_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
 
-                # create all inbox items, in bulk
-                models.InboxItem.objects.bulk_create(final_inbox_items)
+                if str(activity.uuid) in inbox_items_by_activity_uuid:
+                    for obj in inbox_items_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
 
-                for a in activities:
-                    funkwhale_utils.on_commit(
-                        tasks.dispatch_outbox.delay, activity_id=a.pk
-                    )
-                return activities
+            # create all deliveries and items, in bulk
+            models.Delivery.objects.bulk_create(
+                [
+                    obj
+                    for collection in deliveries_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+            models.InboxItem.objects.bulk_create(
+                [
+                    obj
+                    for collection in inbox_items_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+
+            for a in activities:
+                funkwhale_utils.on_commit(tasks.dispatch_outbox.delay, activity_id=a.pk)
+            return activities
+
+
+def recursive_gettattr(obj, key):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        v = v.get(k)
+        if v is None:
+            return
+
+    return v
 
 
 def match_route(route, payload):
     for key, value in route.items():
-        if payload.get(key) != value:
+        payload_value = recursive_gettattr(payload, key)
+        if payload_value != value:
             return False
 
     return True
 
 
-def prepare_inbox_items(recipient_list, type):
+def prepare_deliveries_and_inbox_items(recipient_list, type):
+    """
+    Given a list of recipients (
+        either actor instances, public adresses, a dictionnary with a "type" and "target"
+        keys for followers collections)
+    returns a list of deliveries, alist of inbox_items and a list
+    of urls to persist in the activity in place of the initial recipient list.
+    """
     from . import models
 
-    items = []
-    new_list = []  # we return a list of actors url instead
+    local_recipients = set()
+    remote_inbox_urls = set()
+    urls = []
 
     for r in recipient_list:
-        if r != PUBLIC_ADDRESS:
-            item = models.InboxItem(actor=r, type=type)
-            items.append(item)
-            new_list.append(r.fid)
-        else:
-            new_list.append(r)
+        if isinstance(r, models.Actor):
+            if r.is_local:
+                local_recipients.add(r)
+            else:
+                remote_inbox_urls.add(r.shared_inbox_url or r.inbox_url)
+            urls.append(r.fid)
+        elif r == PUBLIC_ADDRESS:
+            urls.append(r)
+        elif isinstance(r, dict) and r["type"] == "followers":
+            received_follows = (
+                r["target"]
+                .received_follows.filter(approved=True)
+                .select_related("actor__user")
+            )
+            for follow in received_follows:
+                actor = follow.actor
+                if actor.is_local:
+                    local_recipients.add(actor)
+                else:
+                    remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
+            urls.append(r["target"].followers_url)
 
-    return items, new_list
+    deliveries = [models.Delivery(inbox_url=url) for url in remote_inbox_urls]
+    inbox_items = [
+        models.InboxItem(actor=actor, type=type) for actor in local_recipients
+    ]
+
+    return inbox_items, deliveries, urls
+
+
+def join_queries_or(left, right):
+    if left:
+        return left | right
+    else:
+        return right
+
+
+def get_actors_from_audience(urls):
+    """
+    Given a list of urls such as [
+        "https://hello.world/@bob/followers",
+        "https://eldritch.cafe/@alice/followers",
+        "https://funkwhale.demo/libraries/uuid/followers",
+    ]
+    Returns a queryset of actors that are member of the collections
+    listed in the given urls. The urls may contain urls referring
+    to an actor, an actor followers collection or an library followers
+    collection.
+
+    Urls that don't match anything are simply discarded
+    """
+    from . import models
+
+    queries = {"followed": None, "actors": []}
+    for url in urls:
+        if url == PUBLIC_ADDRESS:
+            continue
+        queries["actors"].append(url)
+        queries["followed"] = join_queries_or(
+            queries["followed"], Q(target__followers_url=url)
+        )
+    final_query = None
+    if queries["actors"]:
+        final_query = join_queries_or(final_query, Q(fid__in=queries["actors"]))
+    if queries["followed"]:
+        actor_follows = models.Follow.objects.filter(queries["followed"], approved=True)
+        final_query = join_queries_or(
+            final_query, Q(pk__in=actor_follows.values_list("actor", flat=True))
+        )
+
+        library_follows = models.LibraryFollow.objects.filter(
+            queries["followed"], approved=True
+        )
+        final_query = join_queries_or(
+            final_query, Q(pk__in=library_follows.values_list("actor", flat=True))
+        )
+    if not final_query:
+        return models.Actor.objects.none()
+    return models.Actor.objects.filter(final_query)
+
+
+def get_inbox_urls(actor_queryset):
+    """
+    Given an actor queryset, returns a deduplicated set containing
+    all inbox or shared inbox urls where we should deliver our payloads for
+    those actors
+    """
+    values = actor_queryset.values("inbox_url", "shared_inbox_url")
+
+    urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
+    return sorted(urls)