yetangitu/funkwhale
1
0
Fork 0
mirror of https://code.eliotberriot.com/funkwhale/funkwhale.git synced 2025-10-04 09:59:18 +02:00
Code Issues Releases Wiki Activity

Scan task can now handle until

Browse source
This commit is contained in:
Eliot Berriot 2018-04-12 18:41:43 +02:00
parent 520fb9d078
commit 077a17b093
No known key found for this signature in database
GPG key ID: DD6965E2476E5C27
2 changed files with 69 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

27
api/funkwhale_api/federation/tasks.py
View file

@ -1,27 +1,44 @@
from requests.exceptions import RequestException
from funkwhale_api.taskapp import celery
from . import library as lb
from . import models
@celery.app.task(name='federation.scan_library')
@celery.app.task(
name='federation.scan_library',
autoretry_for=[RequestException],
retry_backoff=30,
max_retries=5)
@celery.require_instance(models.Library, 'library')
def scan_library(library):
def scan_library(library, until=None):
if not library.federation_enabled:
return
data = lb.get_library_data(library.url)
scan_library_page.delay(
library_id=library.id, page_url=data['first'])
library_id=library.id, page_url=data['first'], until=until)
@celery.app.task(name='federation.scan_library_page')
@celery.app.task(
name='federation.scan_library_page',
autoretry_for=[RequestException],
retry_backoff=30,
max_retries=5)
@celery.require_instance(models.Library, 'library')
def scan_library_page(library, page_url):
def scan_library_page(library, page_url, until=None):
if not library.federation_enabled:
return
data = lb.get_library_page(library, page_url)
lts = []
for item_serializer in data['items']:
item_date = item_serializer.validated_data['published']
if until and item_date < until:
return
lts.append(item_serializer.save())
next_page = data.get('next')
if next_page and next_page != page_url:
scan_library_page.delay(library_id=library.id, page_url=next_page)