Upgraded celery to 4.1, added endpoint logic for fingerprinting audio files

api/tests/conftest.py

@@ -1,6 +1,10 @@
 import tempfile
 import shutil
 import pytest
+from django.core.cache import cache as django_cache
+from dynamic_preferences.registries import global_preferences_registry
+
+from funkwhale_api.taskapp import celery
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -11,12 +15,23 @@ def factories_autodiscover():
     factories.registry.autodiscover(app_names)
 
 
+@pytest.fixture(autouse=True)
+def cache():
+    yield django_cache
+    django_cache.clear()
+
+
 @pytest.fixture
 def factories(db):
     from funkwhale_api import factories
     yield factories.registry
 
 
+@pytest.fixture
+def preferences(db):
+    yield global_preferences_registry.manager()
+
+
 @pytest.fixture
 def tmpdir():
     d = tempfile.mkdtemp()

api/tests/music/test_api.py

@@ -34,11 +34,11 @@ def test_can_submit_youtube_url_for_track_import(mocker, superuser_client):
     assert track.album.title == 'Marsupial Madness'
 
 
-def test_import_creates_an_import_with_correct_data(superuser_client, settings):
+def test_import_creates_an_import_with_correct_data(mocker, superuser_client):
+    mocker.patch('funkwhale_api.music.tasks.import_job_run')
     mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed'
     video_id = 'tPEE9ZwTmy0'
     url = reverse('api:v1:submit-single')
-    settings.CELERY_ALWAYS_EAGER = False
     response = superuser_client.post(
         url,
         {'import_url': 'https://www.youtube.com/watch?v={0}'.format(video_id),
@@ -54,7 +54,8 @@ def test_import_creates_an_import_with_correct_data(superuser_client, settings):
     assert job.source == 'https://www.youtube.com/watch?v={0}'.format(video_id)
 
 
-def test_can_import_whole_album(mocker, superuser_client, settings):
+def test_can_import_whole_album(mocker, superuser_client):
+    mocker.patch('funkwhale_api.music.tasks.import_job_run')
     mocker.patch(
         'funkwhale_api.musicbrainz.api.artists.get',
         return_value=api_data.artists['get']['soad'])
@@ -82,7 +83,6 @@ def test_can_import_whole_album(mocker, superuser_client, settings):
         ]
     }
     url = reverse('api:v1:submit-album')
-    settings.CELERY_ALWAYS_EAGER = False
     response = superuser_client.post(
         url, json.dumps(payload), content_type="application/json")
 
@@ -109,7 +109,8 @@ def test_can_import_whole_album(mocker, superuser_client, settings):
         assert job.source == row['source']
 
 
-def test_can_import_whole_artist(mocker, superuser_client, settings):
+def test_can_import_whole_artist(mocker, superuser_client):
+    mocker.patch('funkwhale_api.music.tasks.import_job_run')
     mocker.patch(
         'funkwhale_api.musicbrainz.api.artists.get',
         return_value=api_data.artists['get']['soad'])
@@ -142,7 +143,6 @@ def test_can_import_whole_artist(mocker, superuser_client, settings):
         ]
     }
     url = reverse('api:v1:submit-artist')
-    settings.CELERY_ALWAYS_EAGER = False
     response = superuser_client.post(
         url, json.dumps(payload), content_type="application/json")
 

api/tests/music/test_lyrics.py

@@ -4,6 +4,7 @@ from django.urls import reverse
 from funkwhale_api.music import models
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.music import serializers
+from funkwhale_api.music import tasks
 from funkwhale_api.music import lyrics as lyrics_utils
 
 from .mocking import lyricswiki
@@ -18,7 +19,8 @@ def test_works_import_lyrics_if_any(mocker, factories):
     lyrics = factories['music.Lyrics'](
         url='http://lyrics.wikia.com/System_Of_A_Down:Chop_Suey!')
 
-    lyrics.fetch_content()
+    tasks.fetch_content(lyrics_id=lyrics.pk)
+    lyrics.refresh_from_db()
     self.assertIn(
         'Grab a brush and put on a little makeup',
         lyrics.content,

api/tests/music/test_models.py

@@ -2,6 +2,7 @@ import pytest
 
 from funkwhale_api.music import models
 from funkwhale_api.music import importers
+from funkwhale_api.music import tasks
 
 
 def test_can_store_release_group_id_on_album(factories):
@@ -44,6 +45,6 @@ def test_import_job_is_bound_to_track_file(factories, mocker):
     job = factories['music.ImportJob'](mbid=track.mbid)
 
     mocker.patch('funkwhale_api.music.models.TrackFile.download_file')
-    job.run()
+    tasks.import_job_run(import_job_id=job.pk)
     job.refresh_from_db()
     assert job.track_file.track == track

api/tests/music/test_tasks.py

@@ -0,0 +1,42 @@
+from funkwhale_api.providers.acoustid import get_acoustid_client
+
+from funkwhale_api.music import tasks
+
+
+def test_set_acoustid_on_track_file(factories, mocker):
+    track_file = factories['music.TrackFile'](acoustid_track_id=None)
+    id = 'e475bf79-c1ce-4441-bed7-1e33f226c0a2'
+    payload = {
+        'results': [
+            {'id': id,
+             'recordings': [
+                {'artists': [
+                    {'id': '9c6bddde-6228-4d9f-ad0d-03f6fcb19e13',
+                     'name': 'Binärpilot'}],
+                 'duration': 268,
+                 'id': 'f269d497-1cc0-4ae4-a0c4-157ec7d73fcb',
+                 'title': 'Bend'}],
+            'score': 0.860825}],
+        'status': 'ok'
+    }
+    m = mocker.patch('acoustid.match', return_value=payload)
+    r = tasks.set_acoustid_on_track_file(track_file_id=track_file.pk)
+    track_file.refresh_from_db()
+
+    assert str(track_file.acoustid_track_id) == id
+    assert r == id
+    m.assert_called_once_with('', track_file.audio_file.path, parse=False)
+
+
+def test_set_acoustid_on_track_file_required_high_score(factories, mocker):
+    track_file = factories['music.TrackFile'](acoustid_track_id=None)
+    id = 'e475bf79-c1ce-4441-bed7-1e33f226c0a2'
+    payload = {
+        'results': [{'score': 0.79}],
+        'status': 'ok'
+    }
+    m = mocker.patch('acoustid.match', return_value=payload)
+    r = tasks.set_acoustid_on_track_file(track_file_id=track_file.pk)
+    track_file.refresh_from_db()
+
+    assert track_file.acoustid_track_id is None

api/tests/test_acoustid.py

@@ -0,0 +1,34 @@
+from funkwhale_api.providers.acoustid import get_acoustid_client
+
+
+def test_client_is_configured_with_correct_api_key(preferences):
+    api_key = 'hello world'
+    preferences['providers_acoustid__api_key'] = api_key
+
+    client = get_acoustid_client()
+    assert client.api_key == api_key
+
+
+def test_client_returns_raw_results(db, mocker, preferences):
+    api_key = 'test'
+    preferences['providers_acoustid__api_key'] = api_key
+    payload = {
+        'results': [
+            {'id': 'e475bf79-c1ce-4441-bed7-1e33f226c0a2',
+             'recordings': [
+                {'artists': [
+                    {'id': '9c6bddde-6228-4d9f-ad0d-03f6fcb19e13',
+                     'name': 'Binärpilot'}],
+                 'duration': 268,
+                 'id': 'f269d497-1cc0-4ae4-a0c4-157ec7d73fcb',
+                 'title': 'Bend'}],
+           'score': 0.860825}],
+       'status': 'ok'
+    }
+
+    m = mocker.patch('acoustid.match', return_value=payload)
+    client = get_acoustid_client()
+    response = client.match('/tmp/noopfile.mp3')
+
+    assert response == payload
+    m.assert_called_once_with('test', '/tmp/noopfile.mp3', parse=False)

api/tests/test_import_audio_file.py

@@ -1,6 +1,8 @@
-import os
+import acoustid
 import datetime
+import os
 
+from .music import data as api_data
 from funkwhale_api.providers.audiofile import tasks
 
 DATA_DIR = os.path.join(
@@ -9,7 +11,36 @@ DATA_DIR = os.path.join(
 )
 
 
-def test_can_import_single_audio_file(db, mocker):
+def test_import_file_with_acoustid(db, mocker, preferences):
+    mbid = api_data.tracks['get']['8bitadventures']['recording']['id']
+    payload = {
+        'results': [{
+            'id': 'e475bf79-c1ce-4441-bed7-1e33f226c0a2',
+            'recordings': [{'id': mbid}],
+            'score': 0.86
+        }]
+    }
+    path = os.path.join(DATA_DIR, 'dummy_file.ogg')
+    m = mocker.patch('acoustid.match', return_value=payload)
+    mocker.patch(
+        'funkwhale_api.musicbrainz.api.artists.get',
+        return_value=api_data.artists['get']['adhesive_wombat'])
+    mocker.patch(
+        'funkwhale_api.musicbrainz.api.releases.get',
+        return_value=api_data.albums['get']['marsupial'])
+    mocker.patch(
+        'funkwhale_api.musicbrainz.api.recordings.get',
+        return_value=api_data.tracks['get']['8bitadventures'])
+    track_file = tasks.from_path(path)
+    result = payload['results'][0]
+
+    assert track_file.acoustid_track_id == result['id']
+    assert track_file.track.mbid == result['recordings'][0]['id']
+    m.assert_called_once_with('', path, parse=False)
+
+
+def test_can_import_single_audio_file_without_acoustid(db, mocker):
+    mocker.patch('acoustid.match', side_effect=acoustid.WebServiceError('test'))
     metadata = {
         'artist': ['Test artist'],
         'album': ['Test album'],
@@ -20,7 +51,6 @@ def test_can_import_single_audio_file(db, mocker):
         'musicbrainz_trackid': ['bd21ac48-46d8-4e78-925f-d9cc2a294656'],
         'musicbrainz_artistid': ['013c8e5b-d72a-4cd3-8dee-6c64d6125823'],
     }
-
     m1 = mocker.patch('mutagen.File', return_value=metadata)
     m2 = mocker.patch(
         'funkwhale_api.music.metadata.Metadata.get_file_type',

api/tests/test_tasks.py

@@ -0,0 +1,33 @@
+import pytest
+
+from funkwhale_api.taskapp import celery
+
+
+class Dummy:
+    @staticmethod
+    def noop(instance):
+        pass
+
+
+def test_require_instance_decorator(factories, mocker):
+    user = factories['users.User']()
+
+    @celery.require_instance(user.__class__, 'user')
+    def t(user):
+        Dummy.noop(user)
+
+    m = mocker.patch.object(Dummy, 'noop')
+    t(user_id=user.pk)
+
+    m.assert_called_once_with(user)
+
+
+def test_require_instance_decorator_accepts_qs(factories, mocker):
+    user = factories['users.User'](is_active=False)
+    qs = user.__class__.objects.filter(is_active=True)
+
+    @celery.require_instance(qs, 'user')
+    def t(user):
+        pass
+    with pytest.raises(user.__class__.DoesNotExist):
+        t(user_id=user.pk)