Upgraded celery to 4.1, added endpoint logic for fingerprinting audio files

api/funkwhale_api/music/migrations/0016_trackfile_acoustid_track_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0 on 2017-12-26 16:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0015_bind_track_file_to_import_job'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfile',
+            name='acoustid_track_id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -15,11 +15,9 @@ from django.utils import timezone
 from taggit.managers import TaggableManager
 from versatileimagefield.fields import VersatileImageField
 
-from funkwhale_api.taskapp import celery
 from funkwhale_api import downloader
 from funkwhale_api import musicbrainz
 from . import importers
-from . import lyrics as lyrics_utils
 
 
 class APIModelMixin(models.Model):
@@ -255,14 +253,6 @@ class Lyrics(models.Model):
     url = models.URLField(unique=True)
     content = models.TextField(null=True, blank=True)
 
-    @celery.app.task(name='Lyrics.fetch_content', filter=celery.task_method)
-    def fetch_content(self):
-        html = lyrics_utils._get_html(self.url)
-        content = lyrics_utils.extract_content(html)
-        cleaned_content = lyrics_utils.clean_content(content)
-        self.content = cleaned_content
-        self.save()
-
     @property
     def content_rendered(self):
         return markdown.markdown(
@@ -362,6 +352,7 @@ class TrackFile(models.Model):
     audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
     source = models.URLField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
+    acoustid_track_id = models.UUIDField(null=True, blank=True)
 
     def download_file(self):
         # import the track file, since there is not any
@@ -429,26 +420,3 @@ class ImportJob(models.Model):
 
     class Meta:
         ordering = ('id', )
-    @celery.app.task(name='ImportJob.run', filter=celery.task_method)
-    def run(self, replace=False):
-        try:
-            track, created = Track.get_or_create_from_api(mbid=self.mbid)
-            track_file = None
-            if replace:
-                track_file = track.files.first()
-            elif track.files.count() > 0:
-                return
-
-            track_file = track_file or TrackFile(
-                track=track, source=self.source)
-            track_file.download_file()
-            track_file.save()
-            self.status = 'finished'
-            self.track_file = track_file
-            self.save()
-            return track.pk
-
-        except Exception as exc:
-            if not settings.DEBUG:
-                raise ImportJob.run.retry(args=[self], exc=exc, countdown=30, max_retries=3)
-            raise

api/funkwhale_api/music/tasks.py

@@ -0,0 +1,56 @@
+from funkwhale_api.taskapp import celery
+from funkwhale_api.providers.acoustid import get_acoustid_client
+
+from django.conf import settings
+from . import models
+from . import lyrics as lyrics_utils
+
+
+@celery.app.task(name='acoustid.set_on_track_file')
+@celery.require_instance(models.TrackFile, 'track_file')
+def set_acoustid_on_track_file(track_file):
+    client = get_acoustid_client()
+    result = client.get_best_match(track_file.audio_file.path)
+
+    def update(id):
+        track_file.acoustid_track_id = id
+        track_file.save(update_fields=['acoustid_track_id'])
+        return id
+    if result:
+        return update(result['id'])
+
+
+@celery.app.task(name='ImportJob.run', bind=True)
+@celery.require_instance(models.ImportJob, 'import_job')
+def import_job_run(self, import_job, replace=False):
+    try:
+        track, created = models.Track.get_or_create_from_api(mbid=import_job.mbid)
+        track_file = None
+        if replace:
+            track_file = track.files.first()
+        elif track.files.count() > 0:
+            return
+
+        track_file = track_file or models.TrackFile(
+            track=track, source=import_job.source)
+        track_file.download_file()
+        track_file.save()
+        import_job.status = 'finished'
+        import_job.track_file = track_file
+        import_job.save()
+        return track.pk
+
+    except Exception as exc:
+        if not settings.DEBUG:
+            raise import_job_run.retry(args=[self], exc=exc, countdown=30, max_retries=3)
+        raise
+
+
+@celery.app.task(name='Lyrics.fetch_content')
+@celery.require_instance(models.Lyrics, 'lyrics')
+def fetch_content(lyrics):
+    html = lyrics_utils._get_html(lyrics.url)
+    content = lyrics_utils.extract_content(html)
+    cleaned_content = lyrics_utils.clean_content(content)
+    lyrics.content = cleaned_content
+    lyrics.save(update_fields=['content'])

api/funkwhale_api/music/views.py

@@ -22,6 +22,7 @@ from . import models
 from . import serializers
 from . import importers
 from . import filters
+from . import tasks
 from . import utils
 
 
@@ -129,7 +130,8 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
         lyrics = work.fetch_lyrics()
         try:
             if not lyrics.content:
-                lyrics.fetch_content()
+                tasks.fetch_content(lyrics_id=lyrics.pk)
+                lyrics.refresh_from_db()
         except AttributeError:
             return Response({'error': 'unavailable lyrics'}, status=404)
         serializer = serializers.LyricsSerializer(lyrics)
@@ -244,7 +246,7 @@ class SubmitViewSet(viewsets.ViewSet):
             pass
         batch = models.ImportBatch.objects.create(submitted_by=request.user)
         job = models.ImportJob.objects.create(mbid=request.POST['mbid'], batch=batch, source=request.POST['import_url'])
-        job.run.delay()
+        tasks.import_job_run.delay(import_job_id=job.pk)
         serializer = serializers.ImportBatchSerializer(batch)
         return Response(serializer.data)
 
@@ -272,7 +274,7 @@ class SubmitViewSet(viewsets.ViewSet):
                 models.TrackFile.objects.get(track__mbid=row['mbid'])
             except models.TrackFile.DoesNotExist:
                 job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
-                job.run.delay()
+                tasks.import_job_run.delay(import_job_id=job.pk)
         serializer = serializers.ImportBatchSerializer(batch)
         return serializer.data, batch