Merge branch 'feature/export-progress' of https://github.com/vabene1111/recipes into feature/export-progress

This commit is contained in:
vabene1111
2022-02-03 15:59:18 +01:00
26 changed files with 1240 additions and 100 deletions

View File

@@ -32,11 +32,12 @@ class Default(Integration):
return None
def get_file_from_recipe(self, recipe):
export = RecipeExportSerializer(recipe).data
return 'recipe.json', JSONRenderer().render(export).decode("utf-8")
def get_files_from_recipes(self, recipes, cookie):
def get_files_from_recipes(self, recipes, el, cookie):
export_zip_stream = BytesIO()
export_zip_obj = ZipFile(export_zip_stream, 'w')
@@ -50,13 +51,20 @@ class Default(Integration):
recipe_stream.write(data)
recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
recipe_stream.close()
try:
recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
except ValueError:
pass
recipe_zip_obj.close()
export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
el.exported_recipes += 1
el.msg += self.get_recipe_processed_msg(r)
el.save()
export_zip_obj.close()
return [[ 'export.zip', export_zip_stream.getvalue() ]]
return [[ self.get_export_file_name(), export_zip_stream.getvalue() ]]

View File

@@ -1,9 +1,12 @@
import time
import datetime
import json
import traceback
import uuid
from io import BytesIO, StringIO
from zipfile import BadZipFile, ZipFile
from django.core.cache import cache
import datetime
from bs4 import Tag
from django.core.exceptions import ObjectDoesNotExist
@@ -18,6 +21,7 @@ from cookbook.forms import ImportExportBase
from cookbook.helper.image_processing import get_filetype, handle_image
from cookbook.models import Keyword, Recipe
from recipes.settings import DEBUG
from recipes.settings import EXPORT_FILE_CACHE_DURATION
class Integration:
@@ -61,35 +65,44 @@ class Integration:
space=request.space
)
def do_export(self, recipes):
"""
Perform the export based on a list of recipes
:param recipes: list of recipe objects
:return: HttpResponse with the file of the requested export format that is directly downloaded (When that format involve multiple files they are zipped together)
"""
files = self.get_files_from_recipes(recipes, self.request.COOKIES)
if len(files) == 1:
filename, file = files[0]
export_filename = filename
export_file = file
def do_export(self, recipes, el):
else:
export_filename = "export.zip"
export_stream = BytesIO()
export_obj = ZipFile(export_stream, 'w')
with scope(space=self.request.space):
el.total_recipes = len(recipes)
el.cache_duration = EXPORT_FILE_CACHE_DURATION
el.save()
for filename, file in files:
export_obj.writestr(filename, file)
files = self.get_files_from_recipes(recipes, el, self.request.COOKIES)
export_obj.close()
export_file = export_stream.getvalue()
if len(files) == 1:
filename, file = files[0]
export_filename = filename
export_file = file
else:
#zip the files if there is more then one file
export_filename = self.get_export_file_name()
export_stream = BytesIO()
export_obj = ZipFile(export_stream, 'w')
for filename, file in files:
export_obj.writestr(filename, file)
export_obj.close()
export_file = export_stream.getvalue()
cache.set('export_file_'+str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
el.running = False
el.save()
response = HttpResponse(export_file, content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
return response
def import_file_name_filter(self, zip_info_object):
"""
Since zipfile.namelist() returns all files in all subdirectories this function allows filtering of files
@@ -126,7 +139,7 @@ class Integration:
for d in data_list:
recipe = self.get_recipe_from_file(d)
recipe.keywords.add(self.keyword)
il.msg += f'{recipe.pk} - {recipe.name} \n'
il.msg += self.get_recipe_processed_msg(recipe)
self.handle_duplicates(recipe, import_duplicates)
il.imported_recipes += 1
il.save()
@@ -151,7 +164,7 @@ class Integration:
else:
recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
recipe.keywords.add(self.keyword)
il.msg += f'{recipe.pk} - {recipe.name} \n'
il.msg += self.get_recipe_processed_msg(recipe)
self.handle_duplicates(recipe, import_duplicates)
il.imported_recipes += 1
il.save()
@@ -166,7 +179,7 @@ class Integration:
try:
recipe = self.get_recipe_from_file(d)
recipe.keywords.add(self.keyword)
il.msg += f'{recipe.pk} - {recipe.name} \n'
il.msg += self.get_recipe_processed_msg(recipe)
self.handle_duplicates(recipe, import_duplicates)
il.imported_recipes += 1
il.save()
@@ -183,7 +196,7 @@ class Integration:
try:
recipe = self.get_recipe_from_file(d)
recipe.keywords.add(self.keyword)
il.msg += f'{recipe.pk} - {recipe.name} \n'
il.msg += self.get_recipe_processed_msg(recipe)
self.handle_duplicates(recipe, import_duplicates)
il.imported_recipes += 1
il.save()
@@ -193,7 +206,7 @@ class Integration:
else:
recipe = self.get_recipe_from_file(f['file'])
recipe.keywords.add(self.keyword)
il.msg += f'{recipe.pk} - {recipe.name} \n'
il.msg += self.get_recipe_processed_msg(recipe)
self.handle_duplicates(recipe, import_duplicates)
except BadZipFile:
il.msg += 'ERROR ' + _(
@@ -260,7 +273,7 @@ class Integration:
"""
raise NotImplementedError('Method not implemented in integration')
def get_files_from_recipes(self, recipes, cookie):
def get_files_from_recipes(self, recipes, el, cookie):
"""
Takes a list of recipe object and converts it to a array containing each file.
Each file is represented as an array [filename, data] where data is a string of the content of the file.
@@ -279,3 +292,10 @@ class Integration:
log.msg += exception.msg
if DEBUG:
traceback.print_exc()
def get_export_file_name(self, format='zip'):
return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
def get_recipe_processed_msg(self, recipe):
return f'{recipe.pk} - {recipe.name} \n'

View File

@@ -11,22 +11,25 @@ from cookbook.helper.image_processing import get_filetype
from cookbook.integration.integration import Integration
from cookbook.serializer import RecipeExportSerializer
import django.core.management.commands.runserver as runserver
from cookbook.models import ExportLog
from asgiref.sync import sync_to_async
import django.core.management.commands.runserver as runserver
import logging
class PDFexport(Integration):
def get_recipe_from_file(self, file):
raise NotImplementedError('Method not implemented in storage integration')
async def get_files_from_recipes_async(self, recipes, cookie):
async def get_files_from_recipes_async(self, recipes, el, cookie):
cmd = runserver.Command()
browser = await launch(
handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False,
ignoreHTTPSErrors=True
ignoreHTTPSErrors=True,
)
cookies = {'domain': cmd.default_addr, 'name': 'sessionid', 'value': cookie['sessionid'], }
@@ -39,17 +42,28 @@ class PDFexport(Integration):
}
}
page = await browser.newPage()
await page.emulateMedia('print')
await page.setCookie(cookies)
files = []
for recipe in recipes:
await page.goto('http://' + cmd.default_addr + ':' + cmd.default_port + '/view/recipe/' + str(recipe.id), {'waitUntil': 'networkidle0', })
page = await browser.newPage()
await page.emulateMedia('print')
await page.setCookie(cookies)
await page.goto('http://'+cmd.default_addr+':'+cmd.default_port+'/view/recipe/'+str(recipe.id), {'waitUntil': 'domcontentloaded'})
await page.waitForSelector('#printReady');
files.append([recipe.name + '.pdf', await page.pdf(options)])
await page.close();
el.exported_recipes += 1
el.msg += self.get_recipe_processed_msg(recipe)
await sync_to_async(el.save, thread_sensitive=True)()
await browser.close()
return files
def get_files_from_recipes(self, recipes, cookie):
return asyncio.run(self.get_files_from_recipes_async(recipes, cookie))
def get_files_from_recipes(self, recipes, el, cookie):
return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

View File

@@ -88,12 +88,16 @@ class RecipeSage(Integration):
return data
def get_files_from_recipes(self, recipes, cookie):
def get_files_from_recipes(self, recipes, el, cookie):
json_list = []
for r in recipes:
json_list.append(self.get_file_from_recipe(r))
return [['export.json', json.dumps(json_list)]]
el.exported_recipes += 1
el.msg += self.get_recipe_processed_msg(r)
el.save()
return [[self.get_export_file_name('json'), json.dumps(json_list)]]
def split_recipe_file(self, file):
return json.loads(file.read().decode("utf-8"))

View File

@@ -87,10 +87,14 @@ class Saffron(Integration):
return recipe.name+'.txt', data
def get_files_from_recipes(self, recipes, cookie):
def get_files_from_recipes(self, recipes, el, cookie):
files = []
for r in recipes:
filename, data = self.get_file_from_recipe(r)
files.append([ filename, data ])
el.exported_recipes += 1
el.msg += self.get_recipe_processed_msg(r)
el.save()
return files

View File

@@ -0,0 +1,32 @@
# Generated by Django 3.2.11 on 2022-01-07 20:29
import cookbook.models
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('cookbook', '0163_auto_20220105_0758'),
]
operations = [
migrations.CreateModel(
name='ExportLog',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('type', models.CharField(max_length=32)),
('running', models.BooleanField(default=True)),
('msg', models.TextField(default='')),
('total_recipes', models.IntegerField(default=0)),
('exported_recipes', models.IntegerField(default=0)),
('created_at', models.DateTimeField(auto_now_add=True)),
('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')),
],
bases=(models.Model, cookbook.models.PermissionModelMixin),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.11 on 2022-01-08 00:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cookbook', '0164_exportlog'),
]
operations = [
migrations.AddField(
model_name='exportlog',
name='cache_duration',
field=models.IntegerField(default=0),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.11 on 2022-01-08 00:43
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cookbook', '0165_exportlog_cache_duration'),
]
operations = [
migrations.AddField(
model_name='exportlog',
name='possibly_not_expired',
field=models.BooleanField(default=True),
),
]

View File

@@ -1002,6 +1002,25 @@ class ImportLog(models.Model, PermissionModelMixin):
def __str__(self):
return f"{self.created_at}:{self.type}"
class ExportLog(models.Model, PermissionModelMixin):
type = models.CharField(max_length=32)
running = models.BooleanField(default=True)
msg = models.TextField(default="")
total_recipes = models.IntegerField(default=0)
exported_recipes = models.IntegerField(default=0)
cache_duration = models.IntegerField(default=0)
possibly_not_expired = models.BooleanField(default=True)
created_at = models.DateTimeField(auto_now_add=True)
created_by = models.ForeignKey(User, on_delete=models.CASCADE)
objects = ScopedManager(space='space')
space = models.ForeignKey(Space, on_delete=models.CASCADE)
def __str__(self):
return f"{self.created_at}:{self.type}"
class BookmarkletImport(ExportModelOperationsMixin('bookmarklet_import'), models.Model, PermissionModelMixin):
html = models.TextField()

View File

@@ -15,7 +15,7 @@ from rest_framework.fields import empty
from cookbook.helper.HelperFunctions import str2bool
from cookbook.helper.shopping_helper import RecipeShoppingEditor
from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Food,
FoodInheritField, ImportLog, Ingredient, Keyword, MealPlan, MealType,
FoodInheritField, ImportLog, ExportLog, Ingredient, Keyword, MealPlan, MealType,
NutritionInformation, Recipe, RecipeBook, RecipeBookEntry,
RecipeImport, ShareLink, ShoppingList, ShoppingListEntry,
ShoppingListRecipe, Step, Storage, Supermarket, SupermarketCategory,
@@ -850,6 +850,20 @@ class ImportLogSerializer(serializers.ModelSerializer):
read_only_fields = ('created_by',)
class ExportLogSerializer(serializers.ModelSerializer):
def create(self, validated_data):
validated_data['created_by'] = self.context['request'].user
validated_data['space'] = self.context['request'].space
return super().create(validated_data)
class Meta:
model = ExportLog
fields = ('id', 'type', 'msg', 'running', 'total_recipes', 'exported_recipes', 'cache_duration', 'possibly_not_expired', 'created_by', 'created_at')
read_only_fields = ('created_by',)
class AutomationSerializer(serializers.ModelSerializer):
def create(self, validated_data):

View File

@@ -1140,3 +1140,10 @@
min-width: 28rem;
}
}
@media print{
#switcher{
display: none;
}
}

View File

@@ -1,25 +1,33 @@
{% extends "base.html" %}
{% load crispy_forms_filters %}
{% load i18n %}
{% load render_bundle from webpack_loader %}
{% load static %}
{% load i18n %}
{% load l10n %}
{% block title %}{% trans 'Export Recipes' %}{% endblock %}
{% block extra_head %}
{{ form.media }}
{% block content %}
<div id="app">
<export-view></export-view>
</div>
{% endblock %}
{% block script %}
{% if debug %}
<script src="{% url 'js_reverse' %}"></script>
{% else %}
<script src="{% static 'django_js_reverse/reverse.js' %}"></script>
{% endif %}
<script type="application/javascript">
window.EXPORT_ID = {{pk}};
window.CUSTOM_LOCALE = '{{ request.LANGUAGE_CODE }}';
</script>
{% render_bundle 'export_view' %}
{% endblock %}
{% block content %}
<h2>{% trans 'Export' %}</h2>
<div class="row">
<div class="col col-md-12">
<form action="." method="post">
{% csrf_token %}
{{ form|crispy }}
<button class="btn btn-success" type="submit"><i class="fas fa-file-export"></i> {% trans 'Export' %}
</button>
</form>
</div>
</div>
{% endblock %}

View File

@@ -0,0 +1,32 @@
{% extends "base.html" %}
{% load render_bundle from webpack_loader %}
{% load static %}
{% load i18n %}
{% load l10n %}
{% block title %}{% trans 'Export' %}{% endblock %}
{% block content %}
<div id="app">
<export-response-view></export-response-view>
</div>
{% endblock %}
{% block script %}
{% if debug %}
<script src="{% url 'js_reverse' %}"></script>
{% else %}
<script src="{% static 'django_js_reverse/reverse.js' %}"></script>
{% endif %}
<script type="application/javascript">
window.EXPORT_ID = {{pk}};
window.CUSTOM_LOCALE = '{{ request.LANGUAGE_CODE }}'
</script>
{% render_bundle 'export_response_view' %}
{% endblock %}

View File

@@ -21,6 +21,7 @@ router.register(r'cook-log', api.CookLogViewSet)
router.register(r'food', api.FoodViewSet)
router.register(r'food-inherit-field', api.FoodInheritFieldViewSet)
router.register(r'import-log', api.ImportLogViewSet)
router.register(r'export-log', api.ExportLogViewSet)
router.register(r'ingredient', api.IngredientViewSet)
router.register(r'keyword', api.KeywordViewSet)
router.register(r'meal-plan', api.MealPlanViewSet)
@@ -72,8 +73,10 @@ urlpatterns = [
path('abuse/<slug:token>', views.report_share_abuse, name='view_report_share_abuse'),
path('import/', import_export.import_recipe, name='view_import'),
path('import-response/<int:pk>/', import_export.import_response, name='view_import_response'),
path('import-response/<int:pk>/', import_export.import_response, name='view_import_response'),\
path('export/', import_export.export_recipe, name='view_export'),
path('export-response/<int:pk>/', import_export.export_response, name='view_export_response'),
path('export-file/<int:pk>/', import_export.export_file, name='view_export_file'),
path('view/recipe/<int:pk>', views.recipe_view, name='view_recipe'),
path('view/recipe/<int:pk>/<slug:share>', views.recipe_view, name='view_recipe'),

View File

@@ -43,7 +43,7 @@ from cookbook.helper.recipe_search import RecipeFacet, RecipeSearch, old_search
from cookbook.helper.recipe_url_import import get_from_scraper
from cookbook.helper.shopping_helper import RecipeShoppingEditor, shopping_helper
from cookbook.models import (Automation, BookmarkletImport, CookLog, Food, FoodInheritField,
ImportLog, Ingredient, Keyword, MealPlan, MealType, Recipe, RecipeBook,
ImportLog, ExportLog, Ingredient, Keyword, MealPlan, MealType, Recipe, RecipeBook,
RecipeBookEntry, ShareLink, ShoppingList, ShoppingListEntry,
ShoppingListRecipe, Step, Storage, Supermarket, SupermarketCategory,
SupermarketCategoryRelation, Sync, SyncLog, Unit, UserFile,
@@ -54,7 +54,7 @@ from cookbook.provider.nextcloud import Nextcloud
from cookbook.schemas import FilterSchema, QueryParam, QueryParamAutoSchema, TreeSchema
from cookbook.serializer import (AutomationSerializer, BookmarkletImportSerializer,
CookLogSerializer, FoodInheritFieldSerializer, FoodSerializer,
FoodShoppingUpdateSerializer, ImportLogSerializer,
FoodShoppingUpdateSerializer, ImportLogSerializer, ExportLogSerializer,
IngredientSerializer, KeywordSerializer, MealPlanSerializer,
MealTypeSerializer, RecipeBookEntrySerializer,
RecipeBookSerializer, RecipeImageSerializer,
@@ -861,6 +861,17 @@ class ImportLogViewSet(viewsets.ModelViewSet):
return self.queryset.filter(space=self.request.space)
class ExportLogViewSet(viewsets.ModelViewSet):
queryset = ExportLog.objects
serializer_class = ExportLogSerializer
permission_classes = [CustomIsUser]
pagination_class = DefaultPagination
def get_queryset(self):
return self.queryset.filter(space=self.request.space)
class BookmarkletImportViewSet(viewsets.ModelViewSet):
queryset = BookmarkletImport.objects
serializer_class = BookmarkletImportSerializer

View File

@@ -1,9 +1,10 @@
import re
import threading
from io import BytesIO
from django.core.cache import cache
from django.contrib import messages
from django.http import HttpResponseRedirect, JsonResponse
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import render
from django.urls import reverse
from django.utils.translation import gettext as _
@@ -29,7 +30,7 @@ from cookbook.integration.recipesage import RecipeSage
from cookbook.integration.rezkonv import RezKonv
from cookbook.integration.saffron import Saffron
from cookbook.integration.pdfexport import PDFexport
from cookbook.models import Recipe, ImportLog, UserPreference
from cookbook.models import Recipe, ImportLog, ExportLog, UserPreference
from recipes import settings
@@ -123,25 +124,55 @@ def export_recipe(request):
if form.cleaned_data['all']:
recipes = Recipe.objects.filter(space=request.space, internal=True).all()
if form.cleaned_data['type'] == ImportExportBase.PDF and not settings.ENABLE_PDF_EXPORT:
messages.add_message(request, messages.ERROR, _('The PDF Exporter is not enabled on this instance as it is still in an experimental state.'))
return render(request, 'export.html', {'form': form})
integration = get_integration(request, form.cleaned_data['type'])
return integration.do_export(recipes)
el = ExportLog.objects.create(type=form.cleaned_data['type'], created_by=request.user, space=request.space)
t = threading.Thread(target=integration.do_export, args=[recipes, el])
t.setDaemon(True)
t.start()
return JsonResponse({'export_id': el.pk})
except NotImplementedError:
messages.add_message(request, messages.ERROR, _('Exporting is not implemented for this provider'))
return JsonResponse(
{
'error': True,
'msg': _('Importing is not implemented for this provider')
},
status=400
)
else:
form = ExportForm(space=request.space)
pk = ''
recipe = request.GET.get('r')
if recipe:
if re.match(r'^([0-9])+$', recipe):
if recipe := Recipe.objects.filter(pk=int(recipe), space=request.space).first():
form = ExportForm(initial={'recipes': recipe}, space=request.space)
pk = Recipe.objects.filter(pk=int(recipe), space=request.space).first().pk
return render(request, 'export.html', {'form': form})
return render(request, 'export.html', {'pk': pk})
@group_required('user')
def import_response(request, pk):
return render(request, 'import_response.html', {'pk': pk})
@group_required('user')
def export_response(request, pk):
return render(request, 'export_response.html', {'pk': pk})
@group_required('user')
def export_file(request, pk):
cacheData = cache.get('export_file_'+str(pk))
if cacheData is None:
el = ExportLog.objects.get(pk=pk)
el.possibly_not_expired = False;
el.save()
return render(request, 'export_response.html', {'pk': pk})
response = HttpResponse(cacheData['file'], content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="'+cacheData['filename']+'"'
return response