Attachments

api/funkwhale_api/common/admin.py

@@ -45,3 +45,20 @@ class MutationAdmin(ModelAdmin):
     search_fields = ["created_by__preferred_username"]
     list_filter = ["type", "is_approved", "is_applied"]
     actions = [apply]
+
+
+@register(models.Attachment)
+class AttachmentAdmin(ModelAdmin):
+    list_display = [
+        "uuid",
+        "actor",
+        "url",
+        "file",
+        "size",
+        "mimetype",
+        "creation_date",
+        "last_fetch_date",
+    ]
+    list_select_related = True
+    search_fields = ["actor__domain__name"]
+    list_filter = ["mimetype"]

api/funkwhale_api/common/factories.py

@@ -23,3 +23,14 @@ class MutationFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
             return
         self.target = extracted
         self.save()
+
+
+@registry.register
+class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    url = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    actor = factory.SubFactory(federation_factories.ActorFactory)
+    file = factory.django.ImageField()
+
+    class Meta:
+        model = "common.Attachment"

api/funkwhale_api/common/middleware.py

@@ -1,6 +1,5 @@
 import html
 import io
-import requests
 import time
 import xml.sax.saxutils
 
@@ -11,6 +10,7 @@ from django import urls
 from rest_framework import views
 
 from . import preferences
+from . import session
 from . import throttling
 from . import utils
 
@@ -76,10 +76,7 @@ def get_spa_html(spa_url):
     if cached:
         return cached
 
-    response = requests.get(
-        utils.join_url(spa_url, "index.html"),
-        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-    )
+    response = session.get_session().get(utils.join_url(spa_url, "index.html"),)
     response.raise_for_status()
     content = response.text
     caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)

api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.2.6 on 2019-11-11 13:38
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.common.models
+import funkwhale_api.common.validators
+import uuid
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0003_cit_extension'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Attachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(max_length=500, unique=True, null=True)),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('last_fetch_date', models.DateTimeField(blank=True, null=True)),
+                ('size', models.IntegerField(blank=True, null=True)),
+                ('mimetype', models.CharField(blank=True, max_length=200, null=True)),
+                ('file', versatileimagefield.fields.VersatileImageField(max_length=255, upload_to=funkwhale_api.common.models.get_file_path, validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg'], max_size=5242880)])),
+                ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='federation.Actor', null=True)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/models.py

@@ -1,4 +1,6 @@
 import uuid
+import magic
+import mimetypes
 
 from django.contrib.postgres.fields import JSONField
 from django.contrib.contenttypes.fields import GenericForeignKey
@@ -9,11 +11,18 @@ from django.db import connections, models, transaction
 from django.db.models import Lookup
 from django.db.models.fields import Field
 from django.db.models.sql.compiler import SQLCompiler
+from django.dispatch import receiver
 from django.utils import timezone
 from django.urls import reverse
 
+from versatileimagefield.fields import VersatileImageField
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+
 from funkwhale_api.federation import utils as federation_utils
 
+from . import utils
+from . import validators
+
 
 @Field.register_lookup
 class NotEqual(Lookup):
@@ -150,3 +159,102 @@ class Mutation(models.Model):
         self.applied_date = timezone.now()
         self.save(update_fields=["is_applied", "applied_date", "previous_state"])
         return previous_state
+
+
+def get_file_path(instance, filename):
+    return utils.ChunkedPath("attachments")(instance, filename)
+
+
+class AttachmentQuerySet(models.QuerySet):
+    def attached(self, include=True):
+        related_fields = ["covered_album"]
+        query = None
+        for field in related_fields:
+            field_query = ~models.Q(**{field: None})
+            query = query | field_query if query else field_query
+
+        if include is False:
+            query = ~query
+
+        return self.filter(query)
+
+
+class Attachment(models.Model):
+    # Remote URL where the attachment can be fetched
+    url = models.URLField(max_length=500, unique=True, null=True)
+    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
+    # Actor associated with the attachment
+    actor = models.ForeignKey(
+        "federation.Actor",
+        related_name="attachments",
+        on_delete=models.CASCADE,
+        null=True,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    last_fetch_date = models.DateTimeField(null=True, blank=True)
+    # File size
+    size = models.IntegerField(null=True, blank=True)
+    mimetype = models.CharField(null=True, blank=True, max_length=200)
+
+    file = VersatileImageField(
+        upload_to=get_file_path,
+        max_length=255,
+        validators=[
+            validators.ImageDimensionsValidator(min_width=50, min_height=50),
+            validators.FileValidator(
+                allowed_extensions=["png", "jpg", "jpeg"], max_size=1024 * 1024 * 5,
+            ),
+        ],
+    )
+
+    objects = AttachmentQuerySet.as_manager()
+
+    def save(self, **kwargs):
+        if self.file and not self.size:
+            self.size = self.file.size
+
+        if self.file and not self.mimetype:
+            self.mimetype = self.guess_mimetype()
+
+        return super().save()
+
+    @property
+    def is_local(self):
+        return federation_utils.is_local(self.fid)
+
+    def guess_mimetype(self):
+        f = self.file
+        b = min(1000000, f.size)
+        t = magic.from_buffer(f.read(b), mime=True)
+        if not t.startswith("image/"):
+            # failure, we try guessing by extension
+            mt, _ = mimetypes.guess_type(f.name)
+            if mt:
+                t = mt
+        return t
+
+    @property
+    def download_url_original(self):
+        if self.file:
+            return federation_utils.full_url(self.file.url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=original")
+
+    @property
+    def download_url_medium_square_crop(self):
+        if self.file:
+            return federation_utils.full_url(self.file.crop["200x200"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=medium_square_crop")
+
+
+@receiver(models.signals.post_save, sender=Attachment)
+def warm_attachment_thumbnails(sender, instance, **kwargs):
+    if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS:
+        return
+    warmer = VersatileImageFieldWarmer(
+        instance_or_queryset=instance,
+        rendition_key_set="attachment_square",
+        image_attr="file",
+    )
+    num_created, failed_to_create = warmer.warm()

api/funkwhale_api/common/scripts/create_image_variations.py

@@ -4,11 +4,16 @@ Compute different sizes of image used for Album covers and User avatars
 
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
+from funkwhale_api.common.models import Attachment
 from funkwhale_api.music.models import Album
 from funkwhale_api.users.models import User
 
 
-MODELS = [(Album, "cover", "square"), (User, "avatar", "square")]
+MODELS = [
+    (Album, "cover", "square"),
+    (User, "avatar", "square"),
+    (Attachment, "file", "attachment_square"),
+]
 
 
 def main(command, **kwargs):

api/funkwhale_api/common/serializers.py

@@ -272,3 +272,38 @@ class APIMutationSerializer(serializers.ModelSerializer):
         if value not in self.context["registry"]:
             raise serializers.ValidationError("Invalid mutation type {}".format(value))
         return value
+
+
+class AttachmentSerializer(serializers.Serializer):
+    uuid = serializers.UUIDField(read_only=True)
+    size = serializers.IntegerField(read_only=True)
+    mimetype = serializers.CharField(read_only=True)
+    creation_date = serializers.DateTimeField(read_only=True)
+    file = StripExifImageField(write_only=True)
+    urls = serializers.SerializerMethodField()
+
+    def get_urls(self, o):
+        urls = {}
+        urls["source"] = o.url
+        urls["original"] = o.download_url_original
+        urls["medium_square_crop"] = o.download_url_medium_square_crop
+        return urls
+
+    def to_representation(self, o):
+        repr = super().to_representation(o)
+        # XXX: BACKWARD COMPATIBILITY
+        # having the attachment urls in a nested JSON obj is better,
+        # but we can't do this without breaking clients
+        # So we extract the urls and include these in the parent payload
+        repr.update({k: v for k, v in repr["urls"].items() if k != "source"})
+        # also, our legacy images had lots of variations (400x400, 200x200, 50x50)
+        # but we removed some of these, so we emulate these by hand (by redirecting)
+        # to actual, existing attachment variations
+        repr["square_crop"] = repr["medium_square_crop"]
+        repr["small_square_crop"] = repr["medium_square_crop"]
+        return repr
+
+    def create(self, validated_data):
+        return models.Attachment.objects.create(
+            file=validated_data["file"], actor=validated_data["actor"]
+        )

api/funkwhale_api/common/session.py

@@ -4,6 +4,13 @@ from django.conf import settings
 import funkwhale_api
 
 
+class FunkwhaleSession(requests.Session):
+    def request(self, *args, **kwargs):
+        kwargs.setdefault("verify", settings.EXTERNAL_REQUESTS_VERIFY_SSL)
+        kwargs.setdefault("timeout", settings.EXTERNAL_REQUESTS_TIMEOUT)
+        return super().request(*args, **kwargs)
+
+
 def get_user_agent():
     return "python-requests (funkwhale/{}; +{})".format(
         funkwhale_api.__version__, settings.FUNKWHALE_URL
@@ -11,6 +18,6 @@ def get_user_agent():
 
 
 def get_session():
-    s = requests.Session()
+    s = FunkwhaleSession()
     s.headers["User-Agent"] = get_user_agent()
     return s

api/funkwhale_api/common/tasks.py

@@ -1,14 +1,23 @@
+import datetime
+import logging
+import tempfile
+
+from django.conf import settings
+from django.core.files import File
 from django.db import transaction
 from django.dispatch import receiver
-
+from django.utils import timezone
 
 from funkwhale_api.common import channels
 from funkwhale_api.taskapp import celery
 
 from . import models
 from . import serializers
+from . import session
 from . import signals
 
+logger = logging.getLogger(__name__)
+
 
 @celery.app.task(name="common.apply_mutation")
 @transaction.atomic
@@ -57,3 +66,35 @@ def broadcast_mutation_update(mutation, old_is_approved, new_is_approved, **kwar
             },
         },
     )
+
+
+def fetch_remote_attachment(attachment, filename=None, save=True):
+    if attachment.file:
+        # already there, no need to fetch
+        return
+
+    s = session.get_session()
+    attachment.last_fetch_date = timezone.now()
+    with tempfile.TemporaryFile() as tf:
+        with s.get(attachment.url, timeout=5, stream=True) as r:
+            for chunk in r.iter_content():
+                tf.write(chunk)
+            tf.seek(0)
+            attachment.file.save(
+                filename or attachment.url.split("/")[-1], File(tf), save=save
+            )
+
+
+@celery.app.task(name="common.prune_unattached_attachments")
+def prune_unattached_attachments():
+    limit = timezone.now() - datetime.timedelta(
+        seconds=settings.ATTACHMENTS_UNATTACHED_PRUNE_DELAY
+    )
+    candidates = models.Attachment.objects.attached(False).filter(
+        creation_date__lte=limit
+    )
+
+    total = candidates.count()
+    logger.info("Deleting %s unattached attachments…", total)
+    result = candidates.delete()
+    logger.info("Deletion done: %s", result)

api/funkwhale_api/common/views.py

@@ -11,6 +11,8 @@ from rest_framework import response
 from rest_framework import views
 from rest_framework import viewsets
 
+from funkwhale_api.users.oauth import permissions as oauth_permissions
+
 from . import filters
 from . import models
 from . import mutations
@@ -140,3 +142,40 @@ class RateLimitView(views.APIView):
             "scopes": throttling.get_status(ident, time.time()),
         }
         return response.Response(data, status=200)
+
+
+class AttachmentViewSet(
+    mixins.RetrieveModelMixin,
+    mixins.CreateModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = models.Attachment.objects.all()
+    serializer_class = serializers.AttachmentSerializer
+    permission_classes = [oauth_permissions.ScopePermission]
+    required_scope = "libraries"
+    anonymous_policy = "setting"
+
+    @action(detail=True, methods=["get"])
+    @transaction.atomic
+    def proxy(self, request, *args, **kwargs):
+        instance = self.get_object()
+
+        size = request.GET.get("next", "original").lower()
+        if size not in ["original", "medium_square_crop"]:
+            size = "original"
+
+        tasks.fetch_remote_attachment(instance)
+        data = self.serializer_class(instance).data
+        redirect = response.Response(status=302)
+        redirect["Location"] = data["urls"][size]
+        return redirect
+
+    def perform_create(self, serializer):
+        return serializer.save(actor=self.request.user.actor)
+
+    def perform_destroy(self, instance):
+        if instance.actor is None or instance.actor != self.request.user.actor:
+            raise exceptions.PermissionDenied()
+        instance.delete()

api/funkwhale_api/favorites/views.py

@@ -54,7 +54,9 @@ class TrackFavoriteViewSet(
         )
         tracks = Track.objects.with_playable_uploads(
             music_utils.get_actor_from_request(self.request)
-        ).select_related("artist", "album__artist", "attributed_to")
+        ).select_related(
+            "artist", "album__artist", "attributed_to", "album__attachment_cover"
+        )
         queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
         return queryset
 

api/funkwhale_api/federation/actors.py

@@ -14,10 +14,7 @@ logger = logging.getLogger(__name__)
 
 def get_actor_data(actor_url):
     response = session.get_session().get(
-        actor_url,
-        timeout=5,
-        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-        headers={"Accept": "application/activity+json"},
+        actor_url, headers={"Accept": "application/activity+json"},
     )
     response.raise_for_status()
     try:

api/funkwhale_api/federation/library.py

@@ -1,5 +1,4 @@
 import requests
-from django.conf import settings
 
 from funkwhale_api.common import session
 
@@ -12,8 +11,6 @@ def get_library_data(library_url, actor):
         response = session.get_session().get(
             library_url,
             auth=auth,
-            timeout=5,
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
             headers={"Content-Type": "application/activity+json"},
         )
     except requests.ConnectionError:
@@ -35,11 +32,7 @@ def get_library_data(library_url, actor):
 def get_library_page(library, page_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     response = session.get_session().get(
-        page_url,
-        auth=auth,
-        timeout=5,
-        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-        headers={"Content-Type": "application/activity+json"},
+        page_url, auth=auth, headers={"Content-Type": "application/activity+json"},
     )
     serializer = serializers.CollectionPageSerializer(
         data=response.json(),

api/funkwhale_api/federation/models.py

@@ -541,7 +541,6 @@ class LibraryTrack(models.Model):
             auth=auth,
             stream=True,
             timeout=20,
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
             headers={"Content-Type": "application/activity+json"},
         )
         with remote_response as r:

api/funkwhale_api/federation/serializers.py

@@ -824,8 +824,8 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
 
     def get_tags_repr(self, instance):
         return [
-            {"type": "Hashtag", "name": "#{}".format(tag)}
-            for tag in sorted(instance.tagged_items.values_list("tag__name", flat=True))
+            {"type": "Hashtag", "name": "#{}".format(item.tag.name)}
+            for item in sorted(instance.tagged_items.all(), key=lambda i: i.tag.name)
         ]
 
 
@@ -902,12 +902,11 @@ class AlbumSerializer(MusicEntitySerializer):
             else None,
             "tag": self.get_tags_repr(instance),
         }
-        if instance.cover:
+        if instance.attachment_cover:
             d["cover"] = {
                 "type": "Link",
-                "href": utils.full_url(instance.cover.url),
-                "mediaType": mimetypes.guess_type(instance.cover_path)[0]
-                or "image/jpeg",
+                "href": instance.attachment_cover.download_url_original,
+                "mediaType": instance.attachment_cover.mimetype or "image/jpeg",
             }
         if self.context.get("include_ap_context", self.parent is None):
             d["@context"] = jsonld.get_default_context()

api/funkwhale_api/federation/tasks.py

@@ -88,7 +88,7 @@ def dispatch_inbox(activity, call_handlers=True):
         context={
             "activity": activity,
             "actor": activity.actor,
-            "inbox_items": activity.inbox_items.filter(is_read=False),
+            "inbox_items": activity.inbox_items.filter(is_read=False).order_by("id"),
         },
         call_handlers=call_handlers,
     )
@@ -142,8 +142,6 @@ def deliver_to_remote(delivery):
             auth=auth,
             json=delivery.activity.payload,
             url=delivery.inbox_url,
-            timeout=5,
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
             headers={"Content-Type": "application/activity+json"},
         )
         logger.debug("Remote answered with %s", response.status_code)
@@ -163,9 +161,7 @@ def deliver_to_remote(delivery):
 def fetch_nodeinfo(domain_name):
     s = session.get_session()
     wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name)
-    response = s.get(
-        url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
-    )
+    response = s.get(url=wellknown_url)
     response.raise_for_status()
     serializer = serializers.NodeInfoSerializer(data=response.json())
     serializer.is_valid(raise_exception=True)
@@ -175,9 +171,7 @@ def fetch_nodeinfo(domain_name):
             nodeinfo_url = link["href"]
             break
 
-    response = s.get(
-        url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
-    )
+    response = s.get(url=nodeinfo_url)
     response.raise_for_status()
     return response.json()
 
@@ -308,8 +302,6 @@ def fetch(fetch):
         response = session.get_session().get(
             auth=auth,
             url=fetch.url,
-            timeout=5,
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
             headers={"Content-Type": "application/activity+json"},
         )
         logger.debug("Remote answered with %s", response.status_code)

api/funkwhale_api/federation/utils.py

@@ -84,8 +84,6 @@ def retrieve_ap_object(
     response = session.get_session().get(
         fid,
         auth=auth,
-        timeout=5,
-        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
         headers={
             "Accept": "application/activity+json",
             "Content-Type": "application/activity+json",

api/funkwhale_api/federation/views.py

@@ -1,5 +1,6 @@
 from django import forms
 from django.core import paginator
+from django.db.models import Prefetch
 from django.http import HttpResponse
 from django.urls import reverse
 from rest_framework import exceptions, mixins, permissions, response, viewsets
@@ -163,7 +164,7 @@ class MusicLibraryViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    renderer_classes = renderers.get_ap_renderers()
+    # renderer_classes = renderers.get_ap_renderers()
     serializer_class = serializers.LibrarySerializer
     queryset = music_models.Library.objects.all().select_related("actor")
     lookup_field = "uuid"
@@ -176,7 +177,25 @@ class MusicLibraryViewSet(
             "actor": lb.actor,
             "name": lb.name,
             "summary": lb.description,
-            "items": lb.uploads.for_federation().order_by("-creation_date"),
+            "items": lb.uploads.for_federation()
+            .order_by("-creation_date")
+            .prefetch_related(
+                Prefetch(
+                    "track",
+                    queryset=music_models.Track.objects.select_related(
+                        "album__artist__attributed_to",
+                        "artist__attributed_to",
+                        "album__attributed_to",
+                        "attributed_to",
+                        "album__attachment_cover",
+                    ).prefetch_related(
+                        "tagged_items__tag",
+                        "album__tagged_items__tag",
+                        "album__artist__tagged_items__tag",
+                        "artist__tagged_items__tag",
+                    ),
+                )
+            ),
             "item_serializer": serializers.UploadSerializer,
         }
         page = request.GET.get("page")
@@ -219,7 +238,10 @@ class MusicUploadViewSet(
     authentication_classes = [authentication.SignatureAuthentication]
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Upload.objects.local().select_related(
-        "library__actor", "track__artist", "track__album__artist"
+        "library__actor",
+        "track__artist",
+        "track__album__artist",
+        "track__album__attachment_cover",
     )
     serializer_class = serializers.UploadSerializer
     lookup_field = "uuid"

api/funkwhale_api/federation/webfinger.py

@@ -41,9 +41,7 @@ def get_resource(resource_string):
     url = "https://{}/.well-known/webfinger?resource={}".format(
         hostname, resource_string
     )
-    response = session.get_session().get(
-        url, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, timeout=5
-    )
+    response = session.get_session().get(url)
     response.raise_for_status()
     serializer = serializers.ActorWebfingerSerializer(data=response.json())
     serializer.is_valid(raise_exception=True)

api/funkwhale_api/manage/views.py

@@ -69,9 +69,9 @@ class ManageArtistViewSet(
             "tracks",
             Prefetch(
                 "albums",
-                queryset=music_models.Album.objects.annotate(
-                    tracks_count=Count("tracks")
-                ),
+                queryset=music_models.Album.objects.select_related(
+                    "attachment_cover"
+                ).annotate(tracks_count=Count("tracks")),
             ),
             music_views.TAG_PREFETCH,
         )
@@ -110,7 +110,7 @@ class ManageAlbumViewSet(
     queryset = (
         music_models.Album.objects.all()
         .order_by("-id")
-        .select_related("attributed_to", "artist")
+        .select_related("attributed_to", "artist", "attachment_cover")
         .prefetch_related("tracks", music_views.TAG_PREFETCH)
     )
     serializer_class = serializers.ManageAlbumSerializer
@@ -153,7 +153,9 @@ class ManageTrackViewSet(
     queryset = (
         music_models.Track.objects.all()
         .order_by("-id")
-        .select_related("attributed_to", "artist", "album__artist")
+        .select_related(
+            "attributed_to", "artist", "album__artist", "album__attachment_cover"
+        )
         .annotate(uploads_count=Coalesce(Subquery(uploads_subquery), 0))
         .prefetch_related(music_views.TAG_PREFETCH)
     )

api/funkwhale_api/moderation/management/commands/mrf_check.py

@@ -6,8 +6,6 @@ import logging
 from django.core.management.base import BaseCommand, CommandError
 from django.core import validators
 
-from django.conf import settings
-
 from funkwhale_api.common import session
 from funkwhale_api.federation import models
 from funkwhale_api.moderation import mrf
@@ -84,10 +82,7 @@ class Command(BaseCommand):
             content = models.Activity.objects.get(uuid=input).payload
         elif is_url(input):
             response = session.get_session().get(
-                input,
-                timeout=5,
-                verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-                headers={"Content-Type": "application/activity+json"},
+                input, headers={"Content-Type": "application/activity+json"},
             )
             response.raise_for_status()
             content = response.json()

api/funkwhale_api/music/factories.py

@@ -4,6 +4,7 @@ import factory
 
 from funkwhale_api.factories import registry, NoUpdateOnCreate
 
+from funkwhale_api.common import factories as common_factories
 from funkwhale_api.federation import factories as federation_factories
 from funkwhale_api.music import licenses
 from funkwhale_api.tags import factories as tags_factories
@@ -81,7 +82,7 @@ class AlbumFactory(
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
     release_date = factory.Faker("date_object")
-    cover = factory.django.ImageField()
+    attachment_cover = factory.SubFactory(common_factories.AttachmentFactory)
     artist = factory.SubFactory(ArtistFactory)
     release_group_id = factory.Faker("uuid4")
     fid = factory.Faker("federation_url")

api/funkwhale_api/music/migrations/0042_album_attachment_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.6 on 2019-11-12 09:56
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0004_auto_20191111_1338'),
+        ('music', '0041_auto_20191021_1705'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='attachment_cover',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='common.Attachment', blank=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0043_album_cover_attachment.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def create_attachments(apps, schema_editor):
+    Album = apps.get_model("music", "Album")
+    Attachment = apps.get_model("common", "Attachment")
+
+    album_attachment_mapping = {}
+    def get_mimetype(path):
+        if path.lower().endswith('.png'):
+            return "image/png"
+        return "image/jpeg"
+
+    for album in Album.objects.filter(attachment_cover=None).exclude(cover="").exclude(cover=None):
+        album_attachment_mapping[album] = Attachment(
+            file=album.cover,
+            size=album.cover.size,
+            mimetype=get_mimetype(album.cover.path),
+        )
+
+    Attachment.objects.bulk_create(album_attachment_mapping.values(), batch_size=2000)
+    # map each attachment to the corresponding album
+    # and bulk save
+    for album, attachment in album_attachment_mapping.items():
+        album.attachment_cover = attachment
+
+    Album.objects.bulk_update(album_attachment_mapping.keys(), fields=['attachment_cover'], batch_size=2000)
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("music", "0042_album_attachment_cover")]
+
+    operations = [migrations.RunPython(create_attachments, rewind)]

api/funkwhale_api/music/models.py

@@ -20,7 +20,6 @@ from django.urls import reverse
 from django.utils import timezone
 
 from versatileimagefield.fields import VersatileImageField
-from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
 from funkwhale_api import musicbrainz
 from funkwhale_api.common import fields
@@ -286,9 +285,17 @@ class Album(APIModelMixin):
     artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
     release_date = models.DateField(null=True, blank=True, db_index=True)
     release_group_id = models.UUIDField(null=True, blank=True)
+    # XXX: 1.0 clean this uneeded field in favor of attachment_cover
     cover = VersatileImageField(
         upload_to="albums/covers/%Y/%m/%d", null=True, blank=True
     )
+    attachment_cover = models.ForeignKey(
+        "common.Attachment",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="covered_album",
+    )
     TYPE_CHOICES = (("album", "Album"),)
     type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")
 
@@ -334,40 +341,46 @@ class Album(APIModelMixin):
     objects = AlbumQuerySet.as_manager()
 
     def get_image(self, data=None):
+        from funkwhale_api.common import tasks as common_tasks
+
+        attachment = None
         if data:
             extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
             extension = extensions.get(data["mimetype"], "jpg")
+            attachment = common_models.Attachment(mimetype=data["mimetype"])
             f = None
+            filename = "{}.{}".format(self.uuid, extension)
             if data.get("content"):
                 # we have to cover itself
                 f = ContentFile(data["content"])
+                attachment.file.save(filename, f, save=False)
             elif data.get("url"):
+                attachment.url = data.get("url")
                 # we can fetch from a url
                 try:
-                    response = session.get_session().get(
-                        data.get("url"),
-                        timeout=3,
-                        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+                    common_tasks.fetch_remote_attachment(
+                        attachment, filename=filename, save=False
                     )
-                    response.raise_for_status()
                 except Exception as e:
                     logger.warn(
                         "Cannot download cover at url %s: %s", data.get("url"), e
                     )
                     return
-                else:
-                    f = ContentFile(response.content)
-            if f:
-                self.cover.save("{}.{}".format(self.uuid, extension), f, save=False)
-                self.save(update_fields=["cover"])
-                return self.cover.file
-        if self.mbid:
+
+        elif self.mbid:
             image_data = musicbrainz.api.images.get_front(str(self.mbid))
             f = ContentFile(image_data)
-            self.cover.save("{0}.jpg".format(self.mbid), f, save=False)
-            self.save(update_fields=["cover"])
-        if self.cover:
-            return self.cover.file
+            attachment = common_models.Attachment(mimetype="image/jpeg")
+            attachment.file.save("{0}.jpg".format(self.mbid), f, save=False)
+        if attachment and attachment.file:
+            attachment.save()
+            self.attachment_cover = attachment
+            self.save(update_fields=["attachment_cover"])
+            return self.attachment_cover.file
+
+    @property
+    def cover(self):
+        return self.attachment_cover
 
     def __str__(self):
         return self.title
@@ -378,16 +391,6 @@ class Album(APIModelMixin):
     def get_moderation_url(self):
         return "/manage/library/albums/{}".format(self.pk)
 
-    @property
-    def cover_path(self):
-        if not self.cover:
-            return None
-        try:
-            return self.cover.path
-        except NotImplementedError:
-            # external storage
-            return self.cover.name
-
     @classmethod
     def get_or_create_from_title(cls, title, **kwargs):
         kwargs.update({"title": title})
@@ -415,7 +418,9 @@ def import_album(v):
 
 class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
     def for_nested_serialization(self):
-        return self.prefetch_related("artist", "album__artist")
+        return self.prefetch_related(
+            "artist", "album__artist", "album__attachment_cover"
+        )
 
     def annotate_playable_by_actor(self, actor):
 
@@ -729,7 +734,6 @@ class Upload(models.Model):
         return parsed.hostname
 
     def download_audio_from_remote(self, actor):
-        from funkwhale_api.common import session
         from funkwhale_api.federation import signing
 
         if actor:
@@ -743,7 +747,6 @@ class Upload(models.Model):
             stream=True,
             timeout=20,
             headers={"Content-Type": "application/octet-stream"},
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
         )
         with remote_response as r:
             remote_response.raise_for_status()
@@ -1307,13 +1310,3 @@ def update_request_status(sender, instance, created, **kwargs):
         # let's mark the request as imported since the import is over
         instance.import_request.status = "imported"
         return instance.import_request.save(update_fields=["status"])
-
-
-@receiver(models.signals.post_save, sender=Album)
-def warm_album_covers(sender, instance, **kwargs):
-    if not instance.cover or not settings.CREATE_IMAGE_THUMBNAILS:
-        return
-    album_covers_warmer = VersatileImageFieldWarmer(
-        instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"
-    )
-    num_created, failed_to_create = album_covers_warmer.warm()

api/funkwhale_api/music/serializers.py

@@ -4,7 +4,6 @@ from django.db import transaction
 from django import urls
 from django.conf import settings
 from rest_framework import serializers
-from versatileimagefield.serializers import VersatileImageFieldSerializer
 
 from funkwhale_api.activity import serializers as activity_serializers
 from funkwhale_api.common import serializers as common_serializers
@@ -17,7 +16,25 @@ from funkwhale_api.tags.models import Tag
 from . import filters, models, tasks
 
 
-cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
+class NullToEmptDict(object):
+    def get_attribute(self, o):
+        attr = super().get_attribute(o)
+        if attr is None:
+            return {}
+        return attr
+
+    def to_representation(self, v):
+        if not v:
+            return v
+        return super().to_representation(v)
+
+
+class CoverField(NullToEmptDict, common_serializers.AttachmentSerializer):
+    # XXX: BACKWARD COMPATIBILITY
+    pass
+
+
+cover_field = CoverField()
 
 
 def serialize_attributed_to(self, obj):
@@ -450,12 +467,12 @@ class OembedSerializer(serializers.Serializer):
             embed_type = "track"
             embed_id = track.pk
             data["title"] = "{} by {}".format(track.title, track.artist.name)
-            if track.album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    track.album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if track.album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = track.album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["description"] = track.full_name
             data["author_name"] = track.artist.name
             data["height"] = 150
@@ -476,12 +493,12 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "album"
             embed_id = album.pk
-            if album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = "{} by {}".format(album.title, album.artist.name)
             data["description"] = "{} by {}".format(album.title, album.artist.name)
             data["author_name"] = album.artist.name
@@ -501,19 +518,14 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "artist"
             embed_id = artist.pk
-            album = (
-                artist.albums.filter(cover__isnull=False)
-                .exclude(cover="")
-                .order_by("-id")
-                .first()
-            )
+            album = artist.albums.exclude(attachment_cover=None).order_by("-id").first()
 
-            if album and album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if album and album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = artist.name
             data["description"] = artist.name
             data["author_name"] = artist.name
@@ -533,19 +545,22 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "playlist"
             embed_id = obj.pk
-            playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
-            playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
-            playlist_tracks = playlist_tracks.select_related("track__album").order_by(
-                "index"
+            playlist_tracks = obj.playlist_tracks.exclude(
+                track__album__attachment_cover=None
             )
+            playlist_tracks = playlist_tracks.select_related(
+                "track__album__attachment_cover"
+            ).order_by("index")
             first_playlist_track = playlist_tracks.first()
 
             if first_playlist_track:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    first_playlist_track.track.album.cover.crop["400x400"].url
+                data[
+                    "thumbnail_url"
+                ] = (
+                    first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop
                 )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = obj.name
             data["description"] = obj.name
             data["author_name"] = obj.name

api/funkwhale_api/music/spa_views.py

@@ -57,14 +57,12 @@ def library_track(request, pk):
             ),
         },
     ]
-    if obj.album.cover:
+    if obj.album.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, obj.album.cover.crop["400x400"].url
-                ),
+                "content": obj.album.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -126,14 +124,12 @@ def library_album(request, pk):
             }
         )
 
-    if obj.cover:
+    if obj.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, obj.cover.crop["400x400"].url
-                ),
+                "content": obj.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -166,7 +162,7 @@ def library_artist(request, pk):
     )
     # we use latest album's cover as artist image
     latest_album = (
-        obj.albums.exclude(cover="").exclude(cover=None).order_by("release_date").last()
+        obj.albums.exclude(attachment_cover=None).order_by("release_date").last()
     )
     metas = [
         {"tag": "meta", "property": "og:url", "content": artist_url},
@@ -174,14 +170,12 @@ def library_artist(request, pk):
         {"tag": "meta", "property": "og:type", "content": "profile"},
     ]
 
-    if latest_album and latest_album.cover:
+    if latest_album and latest_album.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, latest_album.cover.crop["400x400"].url
-                ),
+                "content": latest_album.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -217,8 +211,7 @@ def library_playlist(request, pk):
         utils.spa_reverse("library_playlist", kwargs={"pk": obj.pk}),
     )
     # we use the first playlist track's album's cover as image
-    playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
-    playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
+    playlist_tracks = obj.playlist_tracks.exclude(track__album__attachment_cover=None)
     playlist_tracks = playlist_tracks.select_related("track__album").order_by("index")
     first_playlist_track = playlist_tracks.first()
     metas = [
@@ -232,10 +225,7 @@ def library_playlist(request, pk):
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL,
-                    first_playlist_track.track.album.cover.crop["400x400"].url,
-                ),
+                "content": first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop,
             }
         )
 

api/funkwhale_api/music/tasks.py

@@ -21,7 +21,6 @@ from . import licenses
 from . import models
 from . import metadata
 from . import signals
-from . import serializers
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +28,7 @@ logger = logging.getLogger(__name__)
 def update_album_cover(
     album, source=None, cover_data=None, musicbrainz=True, replace=False
 ):
-    if album.cover and not replace:
+    if album.attachment_cover and not replace:
         return
     if cover_data:
         return album.get_image(data=cover_data)
@@ -257,7 +256,7 @@ def process_upload(upload, update_denormalization=True):
         )
 
     # update album cover, if needed
-    if not track.album.cover:
+    if not track.album.attachment_cover:
         update_album_cover(
             track.album,
             source=final_metadata.get("upload_source"),
@@ -404,7 +403,7 @@ def sort_candidates(candidates, important_fields):
 @transaction.atomic
 def get_track_from_import_metadata(data, update_cover=False, attributed_to=None):
     track = _get_track(data, attributed_to=attributed_to)
-    if update_cover and track and not track.album.cover:
+    if update_cover and track and not track.album.attachment_cover:
         update_album_cover(
             track.album,
             source=data.get("upload_source"),
@@ -584,6 +583,8 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
     if not user:
         return
 
+    from . import serializers
+
     group = "user.{}.imports".format(user.pk)
     channels.group_send(
         group,

api/funkwhale_api/music/views.py

@@ -128,7 +128,9 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        albums = models.Album.objects.with_tracks_count()
+        albums = models.Album.objects.with_tracks_count().select_related(
+            "attachment_cover"
+        )
         albums = albums.annotate_playable_by_actor(
             utils.get_actor_from_request(self.request)
         )
@@ -149,7 +151,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
     queryset = (
         models.Album.objects.all()
         .order_by("-creation_date")
-        .prefetch_related("artist", "attributed_to")
+        .prefetch_related("artist", "attributed_to", "attachment_cover")
     )
     serializer_class = serializers.AlbumSerializer
     permission_classes = [oauth_permissions.ScopePermission]

api/funkwhale_api/playlists/models.py

@@ -17,7 +17,8 @@ class PlaylistQuerySet(models.QuerySet):
 
     def with_covers(self):
         album_prefetch = models.Prefetch(
-            "album", queryset=music_models.Album.objects.only("cover", "artist_id")
+            "album",
+            queryset=music_models.Album.objects.select_related("attachment_cover"),
         )
         track_prefetch = models.Prefetch(
             "track",
@@ -29,8 +30,7 @@ class PlaylistQuerySet(models.QuerySet):
         plt_prefetch = models.Prefetch(
             "playlist_tracks",
             queryset=PlaylistTrack.objects.all()
-            .exclude(track__album__cover=None)
-            .exclude(track__album__cover="")
+            .exclude(track__album__attachment_cover=None)
             .order_by("index")
             .only("id", "playlist_id", "track_id")
             .prefetch_related(track_prefetch),
@@ -179,7 +179,9 @@ class Playlist(models.Model):
 class PlaylistTrackQuerySet(models.QuerySet):
     def for_nested_serialization(self, actor=None):
         tracks = music_models.Track.objects.with_playable_uploads(actor)
-        tracks = tracks.select_related("artist", "album__artist")
+        tracks = tracks.select_related(
+            "artist", "album__artist", "album__attachment_cover", "attributed_to"
+        )
         return self.prefetch_related(
             models.Prefetch("track", queryset=tracks, to_attr="_prefetched_track")
         )

api/funkwhale_api/playlists/serializers.py

@@ -145,7 +145,7 @@ class PlaylistSerializer(serializers.ModelSerializer):
         for plt in plts:
             if plt.track.album.artist_id in excluded_artists:
                 continue
-            url = plt.track.album.cover.crop["200x200"].url
+            url = plt.track.album.attachment_cover.download_url_medium_square_crop
             if url in covers:
                 continue
             covers.append(url)

api/funkwhale_api/subsonic/serializers.py

@@ -89,7 +89,7 @@ class GetArtistSerializer(serializers.Serializer):
                 "created": to_subsonic_date(album.creation_date),
                 "songCount": len(album.tracks.all()),
             }
-            if album.cover:
+            if album.attachment_cover_id:
                 album_data["coverArt"] = "al-{}".format(album.id)
             if album.release_date:
                 album_data["year"] = album.release_date.year
@@ -122,7 +122,7 @@ def get_track_data(album, track, upload):
         "artistId": album.artist.pk,
         "type": "music",
     }
-    if track.album.cover:
+    if track.album.attachment_cover_id:
         data["coverArt"] = "al-{}".format(track.album.id)
     if upload.bitrate:
         data["bitrate"] = int(upload.bitrate / 1000)
@@ -141,7 +141,7 @@ def get_album2_data(album):
         "artist": album.artist.name,
         "created": to_subsonic_date(album.creation_date),
     }
-    if album.cover:
+    if album.attachment_cover_id:
         payload["coverArt"] = "al-{}".format(album.id)
 
     try:

api/funkwhale_api/subsonic/views.py

@@ -16,7 +16,12 @@ from rest_framework.serializers import ValidationError
 
 import funkwhale_api
 from funkwhale_api.activity import record
-from funkwhale_api.common import fields, preferences, utils as common_utils
+from funkwhale_api.common import (
+    fields,
+    preferences,
+    utils as common_utils,
+    tasks as common_tasks,
+)
 from funkwhale_api.favorites.models import TrackFavorite
 from funkwhale_api.moderation import filters as moderation_filters
 from funkwhale_api.music import models as music_models
@@ -732,20 +737,23 @@ class SubsonicViewSet(viewsets.GenericViewSet):
             try:
                 album_id = int(id.replace("al-", ""))
                 album = (
-                    music_models.Album.objects.exclude(cover__isnull=True)
-                    .exclude(cover="")
+                    music_models.Album.objects.exclude(attachment_cover=None)
+                    .select_related("attachment_cover")
                     .get(pk=album_id)
                 )
             except (TypeError, ValueError, music_models.Album.DoesNotExist):
                 return response.Response(
                     {"error": {"code": 70, "message": "cover art not found."}}
                 )
-            cover = album.cover
+            attachment = album.attachment_cover
         else:
             return response.Response(
                 {"error": {"code": 70, "message": "cover art not found."}}
             )
 
+        if not attachment.file:
+            common_tasks.fetch_remote_attachment(attachment)
+        cover = attachment.file
         mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"}
         path = music_views.get_file_path(cover)
         file_header = mapping[settings.REVERSE_PROXY_TYPE]