From e5f0c19cdcceb042ca24b2831483ba38348482c3 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:05:34 +0100 Subject: [PATCH 01/54] Add ConnectorManager component which allows for Connectors to listen to triggers and do actions on them. Also add HomeAssistantConfig which stores the configuration for the HomeAssistantConnector --- cookbook/admin.py | 10 +- cookbook/connectors/__init__.py | 0 cookbook/connectors/connector.py | 41 ++++++++ cookbook/connectors/connector_manager.py | 98 +++++++++++++++++++ cookbook/connectors/homeassistant.py | 62 ++++++++++++ cookbook/forms.py | 41 +++++++- ..._alter_storage_path_homeassistantconfig.py | 30 ++++++ ...ing_list_entry_created_enabled_and_more.py | 33 +++++++ cookbook/models.py | 78 ++++++++++----- cookbook/serializer.py | 63 ++++++++---- cookbook/signals.py | 8 +- cookbook/tables.py | 11 ++- cookbook/urls.py | 6 +- cookbook/views/api.py | 10 ++ cookbook/views/delete.py | 25 ++++- cookbook/views/edit.py | 61 ++++++++++-- cookbook/views/lists.py | 30 ++++-- cookbook/views/new.py | 29 +++++- 18 files changed, 566 insertions(+), 70 deletions(-) create mode 100644 cookbook/connectors/__init__.py create mode 100644 cookbook/connectors/connector.py create mode 100644 cookbook/connectors/connector_manager.py create mode 100644 cookbook/connectors/homeassistant.py create mode 100644 cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py create mode 100644 cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py diff --git a/cookbook/admin.py b/cookbook/admin.py index 823246283..4a2b26971 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog) + ViewLog, HomeAssistantConfig) class CustomUserAdmin(UserAdmin): @@ -95,6 +95,14 @@ class StorageAdmin(admin.ModelAdmin): admin.site.register(Storage, StorageAdmin) +class HomeAssistantConfigAdmin(admin.ModelAdmin): + list_display = ('name',) + search_fields = ('name',) + + +admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) + + class SyncAdmin(admin.ModelAdmin): list_display = ('storage', 'path', 'active', 'last_checked') search_fields = ('storage__name', 'path') diff --git a/cookbook/connectors/__init__.py b/cookbook/connectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py new file mode 100644 index 000000000..d2a435d73 --- /dev/null +++ b/cookbook/connectors/connector.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod + +from cookbook.models import ShoppingListEntry, Space + + +class Connector(ABC): + @abstractmethod + async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + @abstractmethod + async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + @abstractmethod + async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + # @abstractmethod + # def on_recipe_created(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_recipe_updated(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_recipe_deleted(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_created(self, instance: MealPlan, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_updated(self, instance: MealPlan, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_deleted(self, instance: MealPlan, **kwargs) -> None: + # pass diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py new file mode 100644 index 000000000..0d3df8cf2 --- /dev/null +++ b/cookbook/connectors/connector_manager.py @@ -0,0 +1,98 @@ +import asyncio +from enum import Enum +from types import UnionType +from typing import List, Any, Dict + +from django_scopes import scope + +from cookbook.connectors.connector import Connector +from cookbook.connectors.homeassistant import HomeAssistant +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space + + +class ActionType(Enum): + CREATED = 1 + UPDATED = 2 + DELETED = 3 + + +class ConnectorManager: + _connectors: Dict[str, List[Connector]] + _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + max_concurrent_tasks = 2 + + def __init__(self): + self._connectors = dict() + + def __call__(self, instance: Any, **kwargs) -> None: + if not isinstance(instance, self._listening_to_classes): + return + + # If a Connector was changed/updated, refresh connector from the database for said space + purge_connector_cache = isinstance(instance, Connector) + + space: Space = instance.space + if space.name in self._connectors and not purge_connector_cache: + connectors: List[Connector] = self._connectors[space.name] + else: + with scope(space=space): + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all()] + self._connectors[space.name] = connectors + + if len(connectors) == 0 or purge_connector_cache: + return + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.run_connectors(connectors, space, instance, **kwargs)) + loop.close() + + @staticmethod + async def run_connectors(connectors: List[Connector], space: Space, instance: Any, **kwargs): + action_type: ActionType + if "created" in kwargs and kwargs["created"]: + action_type = ActionType.CREATED + elif "created" in kwargs and not kwargs["created"]: + action_type = ActionType.UPDATED + elif "origin" in kwargs: + action_type = ActionType.DELETED + else: + return + + tasks: List[asyncio.Task] = list() + + if isinstance(instance, ShoppingListEntry): + shopping_list_entry: ShoppingListEntry = instance + + match action_type: + case ActionType.CREATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) + case ActionType.UPDATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) + case ActionType.DELETED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException as e: + print("received an exception from one of the tasks: ", e) + # if isinstance(instance, Recipe): + # if "created" in kwargs and kwargs["created"]: + # for connector in self._connectors: + # connector.on_recipe_created(instance, **kwargs) + # return + # for connector in self._connectors: + # connector.on_recipe_updated(instance, **kwargs) + # return + # + # if isinstance(instance, MealPlan): + # if "created" in kwargs and kwargs["created"]: + # for connector in self._connectors: + # connector.on_meal_plan_created(instance, **kwargs) + # return + # for connector in self._connectors: + # connector.on_meal_plan_updated(instance, **kwargs) + # return diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py new file mode 100644 index 000000000..6d504d6c1 --- /dev/null +++ b/cookbook/connectors/homeassistant.py @@ -0,0 +1,62 @@ +import logging + +from homeassistant_api import Client, HomeassistantAPIError + +from cookbook.connectors.connector import Connector +from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space + + +class HomeAssistant(Connector): + _config: HomeAssistantConfig + + def __init__(self, config: HomeAssistantConfig): + self._config = config + self._logger = logging.getLogger("connector.HomeAssistant") + + async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_created_enabled: + return + + item, description = _format_shopping_list_entry(shopping_list_entry) + async with Client(self._config.url, self._config.token, use_async=True) as client: + try: + todo_domain = await client.async_get_domain('todo') + await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_updated_enabled: + return + pass + + async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_deleted_enabled: + return + + item, description = _format_shopping_list_entry(shopping_list_entry) + async with Client(self._config.url, self._config.token, use_async=True) as client: + try: + todo_domain = await client.async_get_domain('todo') + await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + +def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): + item = shopping_list_entry.food.name + if shopping_list_entry.amount > 0: + if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0: + item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.base_unit})" + elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0: + item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.name})" + else: + item += f" ({shopping_list_entry.amount})" + + description = "Imported by TandoorRecipes" + if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0: + description += f", created by {shopping_list_entry.created_by.first_name}" + else: + description += f", created by {shopping_list_entry.created_by.username}" + + return item, description diff --git a/cookbook/forms.py b/cookbook/forms.py index dd62aa0bb..76d4a17fc 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference) + SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig) class SelectWidget(widgets.Select): @@ -188,6 +188,45 @@ class StorageForm(forms.ModelForm): } +class HomeAssistantConfigForm(forms.ModelForm): + token = forms.CharField( + widget=forms.TextInput( + attrs={'autocomplete': 'new-password', 'type': 'password'} + ), + required=True, + help_text=_('Long Lived Access Token for your HomeAssistant instance') + ) + + url = forms.URLField( + required=True, + help_text=_('Something like http://homeassistant.local:8123/api'), + ) + + on_shopping_list_entry_created_enabled = forms.BooleanField( + help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", + required=False, + ) + + on_shopping_list_entry_updated_enabled = forms.BooleanField( + help_text="PLACEHOLDER", + required=False, + ) + + on_shopping_list_entry_deleted_enabled = forms.BooleanField( + help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List -- Warning: Might have negative performance impact", + required=False, + ) + + class Meta: + model = HomeAssistantConfig + fields = ( + 'name', 'url', 'token', 'todo_entity', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + + help_texts = { + 'url': _('http://homeassistant.local:8123/api for example'), + } + + # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py b/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py new file mode 100644 index 000000000..acbec2378 --- /dev/null +++ b/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.7 on 2024-01-10 21:28 + +import cookbook.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cookbook', '0205_alter_food_fdc_id_alter_propertytype_fdc_id'), + ] + + operations = [ + migrations.CreateModel( + name='HomeAssistantConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('url', models.URLField(blank=True, help_text='Something like http://homeassistant:8123/api')), + ('token', models.CharField(blank=True, max_length=512)), + ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), + ] diff --git a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py new file mode 100644 index 000000000..67f93b18f --- /dev/null +++ b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.7 on 2024-01-11 19:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0206_alter_storage_path_homeassistantconfig'), + ] + + operations = [ + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_created_enabled', + field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List'), + ), + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_deleted_enabled', + field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List'), + ), + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_updated_enabled', + field=models.BooleanField(default=False, help_text='PLACEHOLDER'), + ), + migrations.AlterField( + model_name='homeassistantconfig', + name='url', + field=models.URLField(blank=True), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index 0614c9749..bb9eb7162 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -49,14 +49,16 @@ def get_active_space(self): def get_shopping_share(self): # get list of users that shared shopping list with user. Django ORM forbids this type of query, so raw is required - return User.objects.raw(' '.join([ - 'SELECT auth_user.id FROM auth_user', - 'INNER JOIN cookbook_userpreference', - 'ON (auth_user.id = cookbook_userpreference.user_id)', - 'INNER JOIN cookbook_userpreference_shopping_share', - 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', - 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) - ])) + return User.objects.raw( + ' '.join( + [ + 'SELECT auth_user.id FROM auth_user', + 'INNER JOIN cookbook_userpreference', + 'ON (auth_user.id = cookbook_userpreference.user_id)', + 'INNER JOIN cookbook_userpreference_shopping_share', + 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', + 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) + ])) auth.models.User.add_to_class('get_user_display_name', get_user_display_name) @@ -339,6 +341,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): SyncLog.objects.filter(sync__space=self).delete() Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() + HomeAssistantConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -363,6 +366,24 @@ class Space(ExportModelOperationsMixin('space'), models.Model): return self.name +class HomeAssistantConfig(models.Model, PermissionModelMixin): + name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) + + url = models.URLField(blank=True) + token = models.CharField(max_length=512, blank=True) + + todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + + on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") + on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") + on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") + + created_by = models.ForeignKey(User, on_delete=models.PROTECT) + + space = models.ForeignKey(Space, on_delete=models.CASCADE) + objects = ScopedManager(space='space') + + class UserPreference(models.Model, PermissionModelMixin): # Themes BOOTSTRAP = 'BOOTSTRAP' @@ -674,10 +695,11 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): if len(inherit) > 0: # ManyToMany cannot be updated through an UPDATE operation for i in inherit: - trough.objects.bulk_create([ - trough(food_id=x, foodinheritfield_id=i['id']) - for x in Food.objects.filter(tree_filter).values_list('id', flat=True) - ]) + trough.objects.bulk_create( + [ + trough(food_id=x, foodinheritfield_id=i['id']) + for x in Food.objects.filter(tree_filter).values_list('id', flat=True) + ]) inherit = [x['field'] for x in inherit] for field in ['ignore_shopping', 'substitute_children', 'substitute_siblings']: @@ -804,8 +826,9 @@ class PropertyType(models.Model, PermissionModelMixin): unit = models.CharField(max_length=64, blank=True, null=True) order = models.IntegerField(default=0) description = models.CharField(max_length=512, blank=True, null=True) - category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), - (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) + category = models.CharField( + max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), + (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) fdc_id = models.IntegerField(null=True, default=None, blank=True) @@ -1368,19 +1391,20 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis UNIT_REPLACE = 'UNIT_REPLACE' NAME_REPLACE = 'NAME_REPLACE' - type = models.CharField(max_length=128, - choices=( - (FOOD_ALIAS, _('Food Alias')), - (UNIT_ALIAS, _('Unit Alias')), - (KEYWORD_ALIAS, _('Keyword Alias')), - (DESCRIPTION_REPLACE, _('Description Replace')), - (INSTRUCTION_REPLACE, _('Instruction Replace')), - (NEVER_UNIT, _('Never Unit')), - (TRANSPOSE_WORDS, _('Transpose Words')), - (FOOD_REPLACE, _('Food Replace')), - (UNIT_REPLACE, _('Unit Replace')), - (NAME_REPLACE, _('Name Replace')), - )) + type = models.CharField( + max_length=128, + choices=( + (FOOD_ALIAS, _('Food Alias')), + (UNIT_ALIAS, _('Unit Alias')), + (KEYWORD_ALIAS, _('Keyword Alias')), + (DESCRIPTION_REPLACE, _('Description Replace')), + (INSTRUCTION_REPLACE, _('Instruction Replace')), + (NEVER_UNIT, _('Never Unit')), + (TRANSPOSE_WORDS, _('Transpose Words')), + (FOOD_REPLACE, _('Food Replace')), + (UNIT_REPLACE, _('Unit Replace')), + (NAME_REPLACE, _('Name Replace')), + )) name = models.CharField(max_length=128, default='') description = models.TextField(blank=True, null=True) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index d4f7bd471..8e338c177 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -34,7 +34,7 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Cu ShareLink, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserPreference, UserSpace, ViewLog) + UserFile, UserPreference, UserSpace, ViewLog, HomeAssistantConfig) from cookbook.templatetags.custom_tags import markdown from recipes.settings import AWS_ENABLED, MEDIA_URL @@ -413,6 +413,27 @@ class StorageSerializer(SpacedModelSerializer): } +class HomeAssistantConfigSerializer(SpacedModelSerializer): + + def create(self, validated_data): + validated_data['created_by'] = self.context['request'].user + return super().create(validated_data) + + class Meta: + model = HomeAssistantConfig + fields = ( + 'id', 'name', 'url', 'token', 'todo_entity', + 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled', 'created_by' + ) + + read_only_fields = ('created_by',) + + extra_kwargs = { + 'token': {'write_only': True}, + } + + class SyncSerializer(SpacedModelSerializer): class Meta: model = Sync @@ -665,8 +686,9 @@ class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedR properties = validated_data.pop('properties', None) - obj, created = Food.objects.get_or_create(name=name, plural_name=plural_name, space=space, properties_food_unit=properties_food_unit, - defaults=validated_data) + obj, created = Food.objects.get_or_create( + name=name, plural_name=plural_name, space=space, properties_food_unit=properties_food_unit, + defaults=validated_data) if properties and len(properties) > 0: for p in properties: @@ -1254,8 +1276,9 @@ class InviteLinkSerializer(WritableNestedModelSerializer): if obj.email: try: - if InviteLink.objects.filter(space=self.context['request'].space, - created_at__gte=datetime.now() - timedelta(hours=4)).count() < 20: + if InviteLink.objects.filter( + space=self.context['request'].space, + created_at__gte=datetime.now() - timedelta(hours=4)).count() < 20: message = _('Hello') + '!\n\n' + _('You have been invited by ') + escape( self.context['request'].user.get_user_display_name()) message += _(' to join their Tandoor Recipes space ') + escape( @@ -1410,12 +1433,15 @@ class RecipeExportSerializer(WritableNestedModelSerializer): class RecipeShoppingUpdateSerializer(serializers.ModelSerializer): - list_recipe = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("Existing shopping list to update")) - ingredients = serializers.IntegerField(write_only=True, allow_null=True, required=False, help_text=_( - "List of ingredient IDs from the recipe to add, if not provided all ingredients will be added.")) - servings = serializers.IntegerField(default=1, write_only=True, allow_null=True, required=False, help_text=_( - "Providing a list_recipe ID and servings of 0 will delete that shopping list.")) + list_recipe = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("Existing shopping list to update")) + ingredients = serializers.IntegerField( + write_only=True, allow_null=True, required=False, help_text=_( + "List of ingredient IDs from the recipe to add, if not provided all ingredients will be added.")) + servings = serializers.IntegerField( + default=1, write_only=True, allow_null=True, required=False, help_text=_( + "Providing a list_recipe ID and servings of 0 will delete that shopping list.")) class Meta: model = Recipe @@ -1423,12 +1449,15 @@ class RecipeShoppingUpdateSerializer(serializers.ModelSerializer): class FoodShoppingUpdateSerializer(serializers.ModelSerializer): - amount = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("Amount of food to add to the shopping list")) - unit = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("ID of unit to use for the shopping list")) - delete = serializers.ChoiceField(choices=['true'], write_only=True, allow_null=True, allow_blank=True, - help_text=_("When set to true will delete all food from active shopping lists.")) + amount = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("Amount of food to add to the shopping list")) + unit = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("ID of unit to use for the shopping list")) + delete = serializers.ChoiceField( + choices=['true'], write_only=True, allow_null=True, allow_blank=True, + help_text=_("When set to true will delete all food from active shopping lists.")) class Meta: model = Recipe diff --git a/cookbook/signals.py b/cookbook/signals.py index a93ffba1e..2d94b1d06 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -4,11 +4,12 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.search import SearchVector from django.core.cache import caches -from django.db.models.signals import post_save +from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from django.utils import translation from django_scopes import scope, scopes_disabled +from cookbook.connectors.connector_manager import ConnectorManager from cookbook.helper.cache_helper import CacheHelper from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY @@ -161,3 +162,8 @@ def clear_unit_cache(sender, instance=None, created=False, **kwargs): def clear_property_type_cache(sender, instance=None, created=False, **kwargs): if instance: caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) + + +handler = ConnectorManager() +post_save.connect(handler, dispatch_uid="connector_manager") +post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/tables.py b/cookbook/tables.py index 6392f791e..8ffe2a53a 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -3,7 +3,7 @@ from django.utils.html import format_html from django.utils.translation import gettext as _ from django_tables2.utils import A -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig class StorageTable(tables.Table): @@ -15,6 +15,15 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') +class HomeAssistantConfigTable(tables.Table): + id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) + + class Meta: + model = HomeAssistantConfig + template_name = 'generic/table_template.html' + fields = ('id', 'name', 'url') + + class ImportLogTable(tables.Table): sync_id = tables.LinkColumn('edit_sync', args=[A('sync_id')]) diff --git a/cookbook/urls.py b/cookbook/urls.py index 9566541a2..96735037e 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,7 +12,7 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name) + UserFile, UserSpace, get_model_name, HomeAssistantConfig) from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -51,6 +51,7 @@ router.register(r'shopping-list-recipe', api.ShoppingListRecipeViewSet) router.register(r'space', api.SpaceViewSet) router.register(r'step', api.StepViewSet) router.register(r'storage', api.StorageViewSet) +router.register(r'home-assistant-config', api.HomeAssistantConfigViewSet) router.register(r'supermarket', api.SupermarketViewSet) router.register(r'supermarket-category', api.SupermarketCategoryViewSet) router.register(r'supermarket-category-relation', api.SupermarketCategoryRelationViewSet) @@ -114,6 +115,7 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), + path('edit/home-assistant-config//', edit.edit_home_assistant_config, name='edit_home_assistant_config'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -166,7 +168,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, HomeAssistantConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 98b4d9bc1..b5f165684 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -460,6 +460,16 @@ class StorageViewSet(viewsets.ModelViewSet): return self.queryset.filter(space=self.request.space) +class HomeAssistantConfigViewSet(viewsets.ModelViewSet): + # TODO handle delete protect error and adjust test + queryset = HomeAssistantConfig.objects + serializer_class = HomeAssistantConfigSerializer + permission_classes = [CustomIsAdmin & CustomTokenHasReadWriteScope] + + def get_queryset(self): + return self.queryset.filter(space=self.request.space) + + class SyncViewSet(viewsets.ModelViewSet): queryset = Sync.objects serializer_class = SyncSerializer diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index 411eb323c..b6cf857b0 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace) + RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -122,6 +122,29 @@ class StorageDelete(GroupRequiredMixin, DeleteView): return HttpResponseRedirect(reverse('list_storage')) +class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): + groups_required = ['admin'] + template_name = "generic/delete_template.html" + model = HomeAssistantConfig + success_url = reverse_lazy('list_storage') + + def get_context_data(self, **kwargs): + context = super(HomeAssistantConfigDelete, self).get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + def post(self, request, *args, **kwargs): + try: + return self.delete(request, *args, **kwargs) + except ProtectedError: + messages.add_message( + request, + messages.WARNING, + _('Could not delete this storage backend as it is used in at least one monitor.') # noqa: E501 + ) + return HttpResponseRedirect(reverse('list_storage')) + + class CommentDelete(OwnerRequiredMixin, DeleteView): template_name = "generic/delete_template.html" model = Comment diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index 1726b2b59..ce4aac27f 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -1,3 +1,4 @@ +import copy import os from django.contrib import messages @@ -8,15 +9,17 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud from recipes import settings +VALUE_NOT_CHANGED = '__NO__CHANGE__' + @group_required('guest') def switch_recipe(request, pk): @@ -76,7 +79,7 @@ class SyncUpdate(GroupRequiredMixin, UpdateView, SpaceFormMixing): @group_required('admin') def edit_storage(request, pk): - instance = get_object_or_404(Storage, pk=pk, space=request.space) + instance: Storage = get_object_or_404(Storage, pk=pk, space=request.space) if not (instance.created_by == request.user or request.user.is_superuser): messages.add_message(request, messages.ERROR, _('You cannot edit this storage!')) @@ -87,17 +90,18 @@ def edit_storage(request, pk): return redirect('index') if request.method == "POST": - form = StorageForm(request.POST, instance=instance) + form = StorageForm(request.POST, instance=copy.deepcopy(instance)) if form.is_valid(): instance.name = form.cleaned_data['name'] instance.method = form.cleaned_data['method'] instance.username = form.cleaned_data['username'] instance.url = form.cleaned_data['url'] + instance.path = form.cleaned_data['path'] - if form.cleaned_data['password'] != '__NO__CHANGE__': + if form.cleaned_data['password'] != VALUE_NOT_CHANGED: instance.password = form.cleaned_data['password'] - if form.cleaned_data['token'] != '__NO__CHANGE__': + if form.cleaned_data['token'] != VALUE_NOT_CHANGED: instance.token = form.cleaned_data['token'] instance.save() @@ -113,8 +117,8 @@ def edit_storage(request, pk): ) else: pseudo_instance = instance - pseudo_instance.password = '__NO__CHANGE__' - pseudo_instance.token = '__NO__CHANGE__' + pseudo_instance.password = VALUE_NOT_CHANGED + pseudo_instance.token = VALUE_NOT_CHANGED form = StorageForm(instance=pseudo_instance) return render( @@ -124,6 +128,47 @@ def edit_storage(request, pk): ) +@group_required('admin') +def edit_home_assistant_config(request, pk): + instance: HomeAssistantConfig = get_object_or_404(HomeAssistantConfig, pk=pk, space=request.space) + + if not (instance.created_by == request.user or request.user.is_superuser): + messages.add_message(request, messages.ERROR, _('You cannot edit this homeassistant config!')) + return HttpResponseRedirect(reverse('edit_home_assistant_config')) + + if request.space.demo or settings.HOSTED: + messages.add_message(request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + if request.method == "POST": + form = HomeAssistantConfigForm(request.POST, instance=copy.deepcopy(instance)) + if form.is_valid(): + instance.name = form.cleaned_data['name'] + instance.url = form.cleaned_data['url'] + instance.todo_entity = form.cleaned_data['todo_entity'] + instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] + instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] + instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] + + if form.cleaned_data['token'] != VALUE_NOT_CHANGED: + instance.token = form.cleaned_data['token'] + + instance.save() + + messages.add_message(request, messages.SUCCESS, _('HomeAssistant config saved!')) + else: + messages.add_message(request, messages.ERROR, _('There was an error updating this config!')) + else: + instance.token = VALUE_NOT_CHANGED + form = HomeAssistantConfigForm(instance=instance) + + return render( + request, + 'generic/edit_template.html', + {'form': form, 'title': _('HomeAssistantConfig')} + ) + + class CommentUpdate(OwnerRequiredMixin, UpdateView): template_name = "generic/edit_template.html" model = Comment diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index b9ef9cc71..20dbb73ca 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, HomeAssistantConfig +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, HomeAssistantConfigTable @group_required('admin') @@ -65,17 +65,31 @@ def storage(request): ) +@group_required('admin') +def home_assistant_config(request): + table = HomeAssistantConfigTable(HomeAssistantConfig.objects.filter(space=request.space).all()) + RequestConfig(request, paginate={'per_page': 25}).configure(table) + + return render( + request, 'generic/list_template.html', { + 'title': _("HomeAssistant Config Backend"), + 'table': table, + 'create_url': 'new_home_assistant_config' + }) + + @group_required('admin') def invite_link(request): table = InviteLinkTable( InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=request.space).all()) RequestConfig(request, paginate={'per_page': 25}).configure(table) - return render(request, 'generic/list_template.html', { - 'title': _("Invite Links"), - 'table': table, - 'create_url': 'new_invite_link' - }) + return render( + request, 'generic/list_template.html', { + 'title': _("Invite Links"), + 'table': table, + 'create_url': 'new_invite_link' + }) @group_required('user') @@ -195,7 +209,7 @@ def custom_filter(request): def user_file(request): try: current_file_size_mb = UserFile.objects.filter(space=request.space).aggregate(Sum('file_size_kb'))[ - 'file_size_kb__sum'] / 1000 + 'file_size_kb__sum'] / 1000 except TypeError: current_file_size_mb = 0 diff --git a/cookbook/views/new.py b/cookbook/views/new.py index d4368f67b..263dca092 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -1,4 +1,3 @@ - from django.contrib import messages from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, redirect, render @@ -6,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig from recipes import settings @@ -71,6 +70,30 @@ class StorageCreate(GroupRequiredMixin, CreateView): return context +class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): + groups_required = ['admin'] + template_name = "generic/new_template.html" + model = HomeAssistantConfig + form_class = HomeAssistantConfigForm + success_url = reverse_lazy('list_home_assistant_config') + + def form_valid(self, form): + if self.request.space.demo or settings.HOSTED: + messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + obj = form.save(commit=False) + obj.created_by = self.request.user + obj.space = self.request.space + obj.save() + return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + + def get_context_data(self, **kwargs): + context = super(HomeAssistantConfigCreate, self).get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + @group_required('user') def create_new_external_recipe(request, import_id): if request.method == "POST": From bf0462cd74ed1166cdbe3e3dca48743528e24eb8 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:14:22 +0100 Subject: [PATCH 02/54] add missing from rebase --- cookbook/views/api.py | 216 ++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 91 deletions(-) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index b5f165684..d2cf65f86 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -76,7 +76,7 @@ from cookbook.models import (Automation, BookmarkletImport, CookLog, CustomFilte ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog) + ViewLog, HomeAssistantConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -102,7 +102,7 @@ from cookbook.serializer import (AccessTokenSerializer, AutomationSerializer, SupermarketCategorySerializer, SupermarketSerializer, SyncLogSerializer, SyncSerializer, UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, - UserSerializer, UserSpaceSerializer, ViewLogSerializer) + UserSerializer, UserSpaceSerializer, ViewLogSerializer, HomeAssistantConfigSerializer) from cookbook.views.import_export import get_integration from recipes import settings from recipes.settings import FDC_API_KEY, DRF_THROTTLE_RECIPE_URL_IMPORT @@ -181,9 +181,10 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) query = self.request.query_params.get('query', None) if self.request.user.is_authenticated: - fuzzy = self.request.user.searchpreference.lookup or any([self.model.__name__.lower() in x for x in - self.request.user.searchpreference.trigram.values_list( - 'field', flat=True)]) + fuzzy = self.request.user.searchpreference.lookup or any( + [self.model.__name__.lower() in x for x in + self.request.user.searchpreference.trigram.values_list( + 'field', flat=True)]) else: fuzzy = True @@ -203,8 +204,10 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): filter |= Q(name__unaccent__icontains=query) self.queryset = ( - self.queryset.annotate(starts=Case(When(name__istartswith=query, then=(Value(100))), - default=Value(0))) # put exact matches at the top of the result set + self.queryset.annotate( + starts=Case( + When(name__istartswith=query, then=(Value(100))), + default=Value(0))) # put exact matches at the top of the result set .filter(filter).order_by('-starts', Lower('name').asc()) ) @@ -326,8 +329,9 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True) self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) - return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, - tree=True) + return self.annotate_recipe( + queryset=self.queryset, request=self.request, serializer=self.serializer_class, + tree=True) @decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) @@ -572,8 +576,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): pass self.queryset = super().get_queryset() - shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), - checked=False).values('id') + shopping_status = ShoppingListEntry.objects.filter( + space=self.request.space, food=OuterRef('id'), + checked=False).values('id') # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users])) return self.queryset \ .annotate(shopping_status=Exists(shopping_status)) \ @@ -594,8 +599,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): shared_users = list(self.request.user.get_shopping_share()) shared_users.append(request.user) if request.data.get('_delete', False) == 'true': - ShoppingListEntry.objects.filter(food=obj, checked=False, space=request.space, - created_by__in=shared_users).delete() + ShoppingListEntry.objects.filter( + food=obj, checked=False, space=request.space, + created_by__in=shared_users).delete() content = {'msg': _(f'{obj.name} was removed from the shopping list.')} return Response(content, status=status.HTTP_204_NO_CONTENT) @@ -603,8 +609,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): unit = request.data.get('unit', None) content = {'msg': _(f'{obj.name} was added to the shopping list.')} - ShoppingListEntry.objects.create(food=obj, amount=amount, unit=unit, space=request.space, - created_by=request.user) + ShoppingListEntry.objects.create( + food=obj, amount=amount, unit=unit, space=request.space, + created_by=request.user) return Response(content, status=status.HTTP_204_NO_CONTENT) @decorators.action(detail=True, methods=['POST'], ) @@ -617,8 +624,11 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}') if response.status_code == 429: - return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, - json_dumps_params={'indent': 4}) + return JsonResponse( + {'msg', + 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, + status=429, + json_dumps_params={'indent': 4}) try: data = json.loads(response.content) @@ -634,12 +644,13 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if pt.fdc_id: for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: - food_property_list.append(Property( - property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, - space=self.request.space, - )) + food_property_list.append( + Property( + property_type_id=pt.id, + property_amount=round(fn['amount'], 2), + import_food_id=food.id, + space=self.request.space, + )) Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) @@ -874,12 +885,14 @@ class RecipePagination(PageNumberPagination): return super().paginate_queryset(queryset, request, view) def get_paginated_response(self, data): - return Response(OrderedDict([ - ('count', self.page.paginator.count), - ('next', self.get_next_link()), - ('previous', self.get_previous_link()), - ('results', data), - ])) + return Response( + OrderedDict( + [ + ('count', self.page.paginator.count), + ('next', self.get_next_link()), + ('previous', self.get_previous_link()), + ('results', data), + ])) class RecipeViewSet(viewsets.ModelViewSet): @@ -965,9 +978,10 @@ class RecipeViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if self.request.GET.get('debug', False): - return JsonResponse({ - 'new': str(self.get_queryset().query), - }) + return JsonResponse( + { + 'new': str(self.get_queryset().query), + }) return super().list(request, *args, **kwargs) def get_serializer_class(self): @@ -1137,8 +1151,10 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet): permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope] query_params = [ QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='int'), - QueryParam(name='checked', description=_('Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') - ), + QueryParam( + name='checked', description=_( + 'Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') + ), QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'), ] schema = QueryParamAutoSchema() @@ -1329,25 +1345,28 @@ class CustomAuthToken(ObtainAuthToken): throttle_classes = [AuthTokenThrottle] def post(self, request, *args, **kwargs): - serializer = self.serializer_class(data=request.data, - context={'request': request}) + serializer = self.serializer_class( + data=request.data, + context={'request': request}) serializer.is_valid(raise_exception=True) user = serializer.validated_data['user'] if token := AccessToken.objects.filter(user=user, expires__gt=timezone.now(), scope__contains='read').filter( scope__contains='write').first(): access_token = token else: - access_token = AccessToken.objects.create(user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', - expires=(timezone.now() + timezone.timedelta(days=365 * 5)), - scope='read write app') - return Response({ - 'id': access_token.id, - 'token': access_token.token, - 'scope': access_token.scope, - 'expires': access_token.expires, - 'user_id': access_token.user.pk, - 'test': user.pk - }) + access_token = AccessToken.objects.create( + user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', + expires=(timezone.now() + timezone.timedelta(days=365 * 5)), + scope='read write app') + return Response( + { + 'id': access_token.id, + 'token': access_token.token, + 'scope': access_token.scope, + 'expires': access_token.expires, + 'user_id': access_token.user.pk, + 'test': user.pk + }) class RecipeUrlImportView(APIView): @@ -1376,61 +1395,71 @@ class RecipeUrlImportView(APIView): url = serializer.validated_data.get('url', None) data = unquote(serializer.validated_data.get('data', None)) if not url and not data: - return Response({ - 'error': True, - 'msg': _('Nothing to do.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Nothing to do.') + }, status=status.HTTP_400_BAD_REQUEST) elif url and not data: if re.match('^(https?://)?(www\\.youtube\\.com|youtu\\.be)/.+$', url): if validators.url(url, public=True): - return Response({ - 'recipe_json': get_from_youtube_scraper(url, request), - 'recipe_images': [], - }, status=status.HTTP_200_OK) + return Response( + { + 'recipe_json': get_from_youtube_scraper(url, request), + 'recipe_images': [], + }, status=status.HTTP_200_OK) if re.match( '^(.)*/view/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url): recipe_json = requests.get( - url.replace('/view/recipe/', '/api/recipe/').replace(re.split('/view/recipe/[0-9]+', url)[1], - '') + '?share=' + + url.replace('/view/recipe/', '/api/recipe/').replace( + re.split('/view/recipe/[0-9]+', url)[1], + '') + '?share=' + re.split('/view/recipe/[0-9]+', url)[1].replace('/', '')).json() recipe_json = clean_dict(recipe_json, 'id') serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request}) if serialized_recipe.is_valid(): recipe = serialized_recipe.save() if validators.url(recipe_json['image'], public=True): - recipe.image = File(handle_image(request, - File(io.BytesIO(requests.get(recipe_json['image']).content), - name='image'), - filetype=pathlib.Path(recipe_json['image']).suffix), - name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') + recipe.image = File( + handle_image( + request, + File( + io.BytesIO(requests.get(recipe_json['image']).content), + name='image'), + filetype=pathlib.Path(recipe_json['image']).suffix), + name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') recipe.save() - return Response({ - 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) - }, status=status.HTTP_201_CREATED) + return Response( + { + 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) + }, status=status.HTTP_201_CREATED) else: try: if validators.url(url, public=True): scrape = scrape_me(url_path=url, wild_mode=True) else: - return Response({ - 'error': True, - 'msg': _('Invalid Url') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Invalid Url') + }, status=status.HTTP_400_BAD_REQUEST) except NoSchemaFoundInWildMode: pass except requests.exceptions.ConnectionError: - return Response({ - 'error': True, - 'msg': _('Connection Refused.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Connection Refused.') + }, status=status.HTTP_400_BAD_REQUEST) except requests.exceptions.MissingSchema: - return Response({ - 'error': True, - 'msg': _('Bad URL Schema.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Bad URL Schema.') + }, status=status.HTTP_400_BAD_REQUEST) else: try: data_json = json.loads(data) @@ -1446,16 +1475,18 @@ class RecipeUrlImportView(APIView): scrape = text_scraper(text=data, url=found_url) if scrape: - return Response({ - 'recipe_json': helper.get_from_scraper(scrape, request), - 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), - }, status=status.HTTP_200_OK) + return Response( + { + 'recipe_json': helper.get_from_scraper(scrape, request), + 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), + }, status=status.HTTP_200_OK) else: - return Response({ - 'error': True, - 'msg': _('No usable data could be found.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('No usable data could be found.') + }, status=status.HTTP_400_BAD_REQUEST) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -1547,8 +1578,9 @@ def import_files(request): return Response({'import_id': il.pk}, status=status.HTTP_200_OK) except NotImplementedError: - return Response({'error': True, 'msg': _('Importing is not implemented for this provider')}, - status=status.HTTP_400_BAD_REQUEST) + return Response( + {'error': True, 'msg': _('Importing is not implemented for this provider')}, + status=status.HTTP_400_BAD_REQUEST) else: return Response({'error': True, 'msg': form.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1624,8 +1656,9 @@ def get_recipe_file(request, recipe_id): @group_required('user') def sync_all(request): if request.space.demo or settings.HOSTED: - messages.add_message(request, messages.ERROR, - _('This feature is not yet available in the hosted version of tandoor!')) + messages.add_message( + request, messages.ERROR, + _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') monitors = Sync.objects.filter(active=True).filter(space=request.user.userspace_set.filter(active=1).first().space) @@ -1664,8 +1697,9 @@ def share_link(request, pk): if request.space.allow_sharing and has_group_permission(request.user, ('user',)): recipe = get_object_or_404(Recipe, pk=pk, space=request.space) link = ShareLink.objects.create(recipe=recipe, created_by=request.user, space=request.space) - return JsonResponse({'pk': pk, 'share': link.uuid, - 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) + return JsonResponse( + {'pk': pk, 'share': link.uuid, + 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) else: return JsonResponse({'error': 'sharing_disabled'}, status=403) From 6a393acd265cb397b803d75b4552f9d74c1b7326 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:35:58 +0100 Subject: [PATCH 03/54] redo migration. cleanup commented out code --- .gitignore | 2 +- cookbook/connectors/connector.py | 24 +------------- cookbook/connectors/connector_manager.py | 18 ---------- ...ing_list_entry_created_enabled_and_more.py | 33 ------------------- ...tconfig.py => 0208_homeassistantconfig.py} | 10 ++++-- 5 files changed, 9 insertions(+), 78 deletions(-) delete mode 100644 cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py rename cookbook/migrations/{0206_alter_storage_path_homeassistantconfig.py => 0208_homeassistantconfig.py} (63%) diff --git a/.gitignore b/.gitignore index 553403a5f..4c8df7a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ docs/_build/ target/ \.idea/dataSources/ - +.idea \.idea/dataSources\.xml \.idea/dataSources\.local\.xml diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index d2a435d73..f97905b93 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,26 +16,4 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass - # @abstractmethod - # def on_recipe_created(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_recipe_updated(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_recipe_deleted(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_created(self, instance: MealPlan, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_updated(self, instance: MealPlan, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_deleted(self, instance: MealPlan, **kwargs) -> None: - # pass + # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 0d3df8cf2..93fa637d9 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -19,7 +19,6 @@ class ActionType(Enum): class ConnectorManager: _connectors: Dict[str, List[Connector]] _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector - max_concurrent_tasks = 2 def __init__(self): self._connectors = dict() @@ -79,20 +78,3 @@ class ConnectorManager: await asyncio.gather(*tasks, return_exceptions=False) except BaseException as e: print("received an exception from one of the tasks: ", e) - # if isinstance(instance, Recipe): - # if "created" in kwargs and kwargs["created"]: - # for connector in self._connectors: - # connector.on_recipe_created(instance, **kwargs) - # return - # for connector in self._connectors: - # connector.on_recipe_updated(instance, **kwargs) - # return - # - # if isinstance(instance, MealPlan): - # if "created" in kwargs and kwargs["created"]: - # for connector in self._connectors: - # connector.on_meal_plan_created(instance, **kwargs) - # return - # for connector in self._connectors: - # connector.on_meal_plan_updated(instance, **kwargs) - # return diff --git a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py deleted file mode 100644 index 67f93b18f..000000000 --- a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-11 19:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('cookbook', '0206_alter_storage_path_homeassistantconfig'), - ] - - operations = [ - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_created_enabled', - field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List'), - ), - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_deleted_enabled', - field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List'), - ), - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_updated_enabled', - field=models.BooleanField(default=False, help_text='PLACEHOLDER'), - ), - migrations.AlterField( - model_name='homeassistantconfig', - name='url', - field=models.URLField(blank=True), - ), - ] diff --git a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py similarity index 63% rename from cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py rename to cookbook/migrations/0208_homeassistantconfig.py index acbec2378..6eca066f0 100644 --- a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-10 21:28 +# Generated by Django 4.2.7 on 2024-01-11 21:34 import cookbook.models from django.conf import settings @@ -8,9 +8,10 @@ import django.db.models.deletion class Migration(migrations.Migration): + dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0205_alter_food_fdc_id_alter_propertytype_fdc_id'), + ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), ] operations = [ @@ -19,9 +20,12 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('url', models.URLField(blank=True, help_text='Something like http://homeassistant:8123/api')), + ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), ], From f1b41461db0a373320dc4f82ef08a32e3f78b30c Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:46:29 +0100 Subject: [PATCH 04/54] bugfix for not working space loading --- cookbook/templatetags/theming_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/templatetags/theming_tags.py b/cookbook/templatetags/theming_tags.py index f35096fd5..68695082f 100644 --- a/cookbook/templatetags/theming_tags.py +++ b/cookbook/templatetags/theming_tags.py @@ -11,7 +11,7 @@ register = template.Library() @register.simple_tag def theme_values(request): space = None - if request.space: + if space in request and request.space: space = request.space if not request.user.is_authenticated and UNAUTHENTICATED_THEME_FROM_SPACE > 0: with scopes_disabled(): From a61f79507b031be06c90d740ae74ae851d4e40c8 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 23:09:42 +0100 Subject: [PATCH 05/54] add enabled field --- cookbook/admin.py | 4 ++-- cookbook/connectors/connector.py | 1 + cookbook/connectors/connector_manager.py | 2 +- cookbook/forms.py | 8 +++++++- cookbook/migrations/0208_homeassistantconfig.py | 3 ++- cookbook/models.py | 1 + cookbook/serializer.py | 2 +- cookbook/tables.py | 2 +- cookbook/views/edit.py | 1 + requirements.txt | 1 + 10 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cookbook/admin.py b/cookbook/admin.py index 4a2b26971..3429e63d3 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -96,8 +96,8 @@ admin.site.register(Storage, StorageAdmin) class HomeAssistantConfigAdmin(admin.ModelAdmin): - list_display = ('name',) - search_fields = ('name',) + list_display = ('id', 'name', 'enabled', 'url') + search_fields = ('name', 'url') admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index f97905b93..76699e09a 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,4 +16,5 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass + # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 93fa637d9..8a50f90b1 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -35,7 +35,7 @@ class ConnectorManager: connectors: List[Connector] = self._connectors[space.name] else: with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all()] + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] self._connectors[space.name] = connectors if len(connectors) == 0 or purge_connector_cache: diff --git a/cookbook/forms.py b/cookbook/forms.py index 76d4a17fc..dce4da57d 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -202,6 +202,11 @@ class HomeAssistantConfigForm(forms.ModelForm): help_text=_('Something like http://homeassistant.local:8123/api'), ) + enabled = forms.BooleanField( + help_text="Is the HomeAssistantConnector enabled", + required=False, + ) + on_shopping_list_entry_created_enabled = forms.BooleanField( help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", required=False, @@ -220,7 +225,8 @@ class HomeAssistantConfigForm(forms.ModelForm): class Meta: model = HomeAssistantConfig fields = ( - 'name', 'url', 'token', 'todo_entity', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled') help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py index 6eca066f0..a7140b341 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-11 21:34 +# Generated by Django 4.2.7 on 2024-01-11 22:06 import cookbook.models from django.conf import settings @@ -23,6 +23,7 @@ class Migration(migrations.Migration): ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('enabled', models.BooleanField(default=True, help_text='Is HomeAssistant Connector Enabled')), ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), diff --git a/cookbook/models.py b/cookbook/models.py index bb9eb7162..d6cf50789 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -374,6 +374,7 @@ class HomeAssistantConfig(models.Model, PermissionModelMixin): todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + enabled = models.BooleanField(default=True, help_text="Is HomeAssistant Connector Enabled") on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 8e338c177..53fc8eebc 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -422,7 +422,7 @@ class HomeAssistantConfigSerializer(SpacedModelSerializer): class Meta: model = HomeAssistantConfig fields = ( - 'id', 'name', 'url', 'token', 'todo_entity', + 'id', 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled', 'created_by' ) diff --git a/cookbook/tables.py b/cookbook/tables.py index 8ffe2a53a..4f49690cb 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -21,7 +21,7 @@ class HomeAssistantConfigTable(tables.Table): class Meta: model = HomeAssistantConfig template_name = 'generic/table_template.html' - fields = ('id', 'name', 'url') + fields = ('id', 'name', 'enabled', 'url') class ImportLogTable(tables.Table): diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index ce4aac27f..deb9654bd 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -146,6 +146,7 @@ def edit_home_assistant_config(request, pk): instance.name = form.cleaned_data['name'] instance.url = form.cleaned_data['url'] instance.todo_entity = form.cleaned_data['todo_entity'] + instance.enabled = form.cleaned_data['enabled'] instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] diff --git a/requirements.txt b/requirements.txt index 9cf4efdfb..0611ea423 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,3 +46,4 @@ pytest-factoryboy==2.5.1 pyppeteer==1.0.2 validators==0.20.0 pytube==15.0.0 +homeassistant-api==4.1.1.post2 From d576394c9957d48dbe08e4ac6f48d2275bf2ddd9 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 20:50:23 +0100 Subject: [PATCH 06/54] run everything in a seperate process --- cookbook/connectors/connector_manager.py | 124 +++++++++++++++-------- cookbook/connectors/homeassistant.py | 42 +++++--- 2 files changed, 110 insertions(+), 56 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 8a50f90b1..2be647320 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,7 +1,12 @@ import asyncio +import logging +import multiprocessing +from asyncio import Task +from dataclasses import dataclass from enum import Enum +from multiprocessing import Queue from types import UnionType -from typing import List, Any, Dict +from typing import List, Any, Dict, Optional from django_scopes import scope @@ -9,6 +14,11 @@ from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space +multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 + +QUEUE_MAX_SIZE = 10 +REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + class ActionType(Enum): CREATED = 1 @@ -16,38 +26,25 @@ class ActionType(Enum): DELETED = 3 +@dataclass +class Payload: + instance: REGISTERED_CLASSES + actionType: ActionType + + class ConnectorManager: - _connectors: Dict[str, List[Connector]] - _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + _queue: Queue + _listening_to_classes = REGISTERED_CLASSES def __init__(self): - self._connectors = dict() + self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) + self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) + self._worker.start() def __call__(self, instance: Any, **kwargs) -> None: - if not isinstance(instance, self._listening_to_classes): + if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"): return - # If a Connector was changed/updated, refresh connector from the database for said space - purge_connector_cache = isinstance(instance, Connector) - - space: Space = instance.space - if space.name in self._connectors and not purge_connector_cache: - connectors: List[Connector] = self._connectors[space.name] - else: - with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] - self._connectors[space.name] = connectors - - if len(connectors) == 0 or purge_connector_cache: - return - - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(self.run_connectors(connectors, space, instance, **kwargs)) - loop.close() - - @staticmethod - async def run_connectors(connectors: List[Connector], space: Space, instance: Any, **kwargs): action_type: ActionType if "created" in kwargs and kwargs["created"]: action_type = ActionType.CREATED @@ -58,23 +55,64 @@ class ConnectorManager: else: return - tasks: List[asyncio.Task] = list() + self._queue.put_nowait(Payload(instance, action_type)) - if isinstance(instance, ShoppingListEntry): - shopping_list_entry: ShoppingListEntry = instance + def stop(self): + self._queue.close() + self._worker.join() - match action_type: - case ActionType.CREATED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) - case ActionType.UPDATED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) - case ActionType.DELETED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + @staticmethod + def worker(queue: Queue): + from django.db import connections + connections.close_all() - try: - await asyncio.gather(*tasks, return_exceptions=False) - except BaseException as e: - print("received an exception from one of the tasks: ", e) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + _connectors: Dict[str, List[Connector]] = dict() + + while True: + item: Optional[Payload] = queue.get() + if item is None: + break + + # If a Connector was changed/updated, refresh connector from the database for said space + refresh_connector_cache = isinstance(item.instance, Connector) + + space: Space = item.instance.space + connectors: Optional[List[Connector]] = _connectors.get(space.name, None) + + if connectors is None or refresh_connector_cache: + with scope(space=space): + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] + _connectors[space.name] = connectors + + if len(connectors) == 0 or refresh_connector_cache: + return + + loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) + + loop.close() + + +async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType): + tasks: List[Task] = list() + + if isinstance(instance, ShoppingListEntry): + shopping_list_entry: ShoppingListEntry = instance + + match action_type: + case ActionType.CREATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) + case ActionType.UPDATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) + case ActionType.DELETED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException: + logging.exception("received an exception from one of the tasks") diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index 6d504d6c1..f5ae454fc 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -1,29 +1,41 @@ import logging +from collections import defaultdict +from logging import Logger +from typing import Dict, Any, Optional -from homeassistant_api import Client, HomeassistantAPIError +from homeassistant_api import Client, HomeassistantAPIError, Domain from cookbook.connectors.connector import Connector from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space class HomeAssistant(Connector): + _domains_cache: dict[str, Domain] _config: HomeAssistantConfig + _logger: Logger + _client: Client def __init__(self, config: HomeAssistantConfig): + self._domains_cache = dict() self._config = config self._logger = logging.getLogger("connector.HomeAssistant") + self._client = Client(self._config.url, self._config.token, async_cache_session=False, use_async=True) async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: if not self._config.on_shopping_list_entry_created_enabled: return item, description = _format_shopping_list_entry(shopping_list_entry) - async with Client(self._config.url, self._config.token, use_async=True) as client: - try: - todo_domain = await client.async_get_domain('todo') - await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) - except HomeassistantAPIError as err: - self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + todo_domain = self._domains_cache.get('todo') + try: + if todo_domain is None: + todo_domain = await self._client.async_get_domain('todo') + self._domains_cache['todo'] = todo_domain + + await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: if not self._config.on_shopping_list_entry_updated_enabled: @@ -35,12 +47,16 @@ class HomeAssistant(Connector): return item, description = _format_shopping_list_entry(shopping_list_entry) - async with Client(self._config.url, self._config.token, use_async=True) as client: - try: - todo_domain = await client.async_get_domain('todo') - await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) - except HomeassistantAPIError as err: - self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + todo_domain = self._domains_cache.get('todo') + try: + if todo_domain is None: + todo_domain = await self._client.async_get_domain('todo') + self._domains_cache['todo'] = todo_domain + + await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): From 445e64c71edafef27d46d19c1facec25deab0170 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 22:20:55 +0100 Subject: [PATCH 07/54] add an config toggle for external connectors --- cookbook/connectors/connector_manager.py | 6 ++++- .../migrations/0208_homeassistantconfig.py | 13 ++++++---- cookbook/models.py | 24 +++++++++++-------- cookbook/signals.py | 8 ++++--- cookbook/templates/base.html | 4 ++++ recipes/settings.py | 2 ++ 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 2be647320..532523075 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,6 +1,7 @@ import asyncio import logging import multiprocessing +import queue from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -55,7 +56,10 @@ class ConnectorManager: else: return - self._queue.put_nowait(Payload(instance, action_type)) + try: + self._queue.put_nowait(Payload(instance, action_type)) + except queue.Full: + return def stop(self): self._queue.close() diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py index a7140b341..f21ff06ae 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-11 22:06 +# Generated by Django 4.2.7 on 2024-01-12 21:01 import cookbook.models from django.conf import settings @@ -20,16 +20,19 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), - ('enabled', models.BooleanField(default=True, help_text='Is HomeAssistant Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), ], + options={ + 'abstract': False, + }, bases=(models.Model, cookbook.models.PermissionModelMixin), ), ] diff --git a/cookbook/models.py b/cookbook/models.py index d6cf50789..a28cc5a51 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -366,24 +366,28 @@ class Space(ExportModelOperationsMixin('space'), models.Model): return self.name -class HomeAssistantConfig(models.Model, PermissionModelMixin): +class ConnectorConfig(models.Model, PermissionModelMixin): name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) - url = models.URLField(blank=True) - token = models.CharField(max_length=512, blank=True) - - todo_entity = models.CharField(max_length=128, default='todo.shopping_list') - - enabled = models.BooleanField(default=True, help_text="Is HomeAssistant Connector Enabled") - on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") - on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") - on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") + enabled = models.BooleanField(default=True, help_text="Is Connector Enabled") + on_shopping_list_entry_created_enabled = models.BooleanField(default=False) + on_shopping_list_entry_updated_enabled = models.BooleanField(default=False) + on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False) created_by = models.ForeignKey(User, on_delete=models.PROTECT) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') + class Meta: + abstract = True + + +class HomeAssistantConfig(ConnectorConfig): + url = models.URLField(blank=True) + token = models.CharField(max_length=512, blank=True) + todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + class UserPreference(models.Model, PermissionModelMixin): # Themes diff --git a/cookbook/signals.py b/cookbook/signals.py index 2d94b1d06..ce957bcea 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -15,6 +15,7 @@ from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY from cookbook.models import (Food, MealPlan, PropertyType, Recipe, SearchFields, SearchPreference, Step, Unit, UserPreference) +from recipes.settings import ENABLE_EXTERNAL_CONNECTORS SQLITE = True if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql': @@ -164,6 +165,7 @@ def clear_property_type_cache(sender, instance=None, created=False, **kwargs): caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) -handler = ConnectorManager() -post_save.connect(handler, dispatch_uid="connector_manager") -post_delete.connect(handler, dispatch_uid="connector_manager") +if ENABLE_EXTERNAL_CONNECTORS: + handler = ConnectorManager() + post_save.connect(handler, dispatch_uid="connector_manager") + post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index d84ccbd46..d9e2eda15 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -335,6 +335,10 @@ {% trans 'Space Settings' %} {% endif %} + {% if request.user == request.space.created_by or user.is_superuser %} + {% trans 'External Connectors' %} + {% endif %} {% if user.is_superuser %}
Date: Fri, 12 Jan 2024 23:13:53 +0100 Subject: [PATCH 08/54] write some simple tests --- cookbook/connectors/connector_manager.py | 10 +- .../api/test_api_home_assistant_config.py | 126 ++++++++++++++++++ .../edits/test_edits_home_assistant_config.py | 58 ++++++++ .../tests/other/test_connector_manager.py | 28 ++++ requirements.txt | 1 + 5 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 cookbook/tests/api/test_api_home_assistant_config.py create mode 100644 cookbook/tests/edits/test_edits_home_assistant_config.py create mode 100644 cookbook/tests/other/test_connector_manager.py diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 532523075..d660b6d84 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -28,7 +28,7 @@ class ActionType(Enum): @dataclass -class Payload: +class Work: instance: REGISTERED_CLASSES actionType: ActionType @@ -57,7 +57,7 @@ class ConnectorManager: return try: - self._queue.put_nowait(Payload(instance, action_type)) + self._queue.put_nowait(Work(instance, action_type)) except queue.Full: return @@ -66,7 +66,7 @@ class ConnectorManager: self._worker.join() @staticmethod - def worker(queue: Queue): + def worker(worker_queue: Queue): from django.db import connections connections.close_all() @@ -76,7 +76,7 @@ class ConnectorManager: _connectors: Dict[str, List[Connector]] = dict() while True: - item: Optional[Payload] = queue.get() + item: Optional[Work] = worker_queue.get() if item is None: break @@ -119,4 +119,4 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: - logging.exception("received an exception from one of the tasks") + logging.exception("received an exception from one of the connectors") diff --git a/cookbook/tests/api/test_api_home_assistant_config.py b/cookbook/tests/api/test_api_home_assistant_config.py new file mode 100644 index 000000000..a3de0f453 --- /dev/null +++ b/cookbook/tests/api/test_api_home_assistant_config.py @@ -0,0 +1,126 @@ +import json + +import pytest +from django.contrib import auth +from django.urls import reverse +from django_scopes import scopes_disabled + +from cookbook.models import HomeAssistantConfig + +LIST_URL = 'api:homeassistantconfig-list' +DETAIL_URL = 'api:homeassistantconfig-detail' + + +@pytest.fixture() +def obj_1(space_1, u1_s1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 1', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) + + +@pytest.fixture +def obj_2(space_1, u1_s1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 2', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 200], + ]) +def test_list_permission(arg, request): + c = request.getfixturevalue(arg[0]) + r = c.get(reverse(LIST_URL)) + assert r.status_code == arg[1] + if r.status_code == 200: + response = json.loads(r.content) + assert 'token' not in response + + +def test_list_space(obj_1, obj_2, a1_s1, a1_s2, space_2): + assert len(json.loads(a1_s1.get(reverse(LIST_URL)).content)) == 2 + assert len(json.loads(a1_s2.get(reverse(LIST_URL)).content)) == 0 + + obj_1.space = space_2 + obj_1.save() + + assert len(json.loads(a1_s1.get(reverse(LIST_URL)).content)) == 1 + assert len(json.loads(a1_s2.get(reverse(LIST_URL)).content)) == 1 + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 200], + ['g1_s2', 403], + ['u1_s2', 403], + ['a1_s2', 404], + ]) +def test_update(arg, request, obj_1): + test_token = '1234' + + c = request.getfixturevalue(arg[0]) + r = c.patch( + reverse( + DETAIL_URL, + args={obj_1.id} + ), + {'name': 'new', 'token': test_token}, + content_type='application/json' + ) + response = json.loads(r.content) + assert r.status_code == arg[1] + if r.status_code == 200: + assert response['name'] == 'new' + obj_1.refresh_from_db() + assert obj_1.token == test_token + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 201], + ]) +def test_add(arg, request, a1_s2, obj_1): + c = request.getfixturevalue(arg[0]) + r = c.post( + reverse(LIST_URL), + {'name': 'test', 'url': 'http://localhost:8123/api', 'token': '1234', 'enabled': 'true'}, + content_type='application/json' + ) + response = json.loads(r.content) + print(r.content) + assert r.status_code == arg[1] + if r.status_code == 201: + assert response['name'] == 'test' + r = c.get(reverse(DETAIL_URL, args={response['id']})) + assert r.status_code == 200 + r = a1_s2.get(reverse(DETAIL_URL, args={response['id']})) + assert r.status_code == 404 + + +def test_delete(a1_s1, a1_s2, obj_1): + r = a1_s2.delete( + reverse( + DETAIL_URL, + args={obj_1.id} + ) + ) + assert r.status_code == 404 + + r = a1_s1.delete( + reverse( + DETAIL_URL, + args={obj_1.id} + ) + ) + + assert r.status_code == 204 + with scopes_disabled(): + assert HomeAssistantConfig.objects.count() == 0 diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py new file mode 100644 index 000000000..053972b62 --- /dev/null +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -0,0 +1,58 @@ +from cookbook.models import Storage, HomeAssistantConfig +from django.contrib import auth +from django.urls import reverse +import pytest + +EDIT_VIEW_NAME = 'edit_home_assistant_config' + + +@pytest.fixture +def home_assistant_config_obj(a1_s1, space_1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 1', + token='token', + url='url', + todo_entity='todo.shopping_list', + enabled=True, + created_by=auth.get_user(a1_s1), + space=space_1, + ) + + +def test_edit_home_assistant_config(home_assistant_config_obj, a1_s1, a1_s2): + new_token = '1234_token' + + r = a1_s1.post( + reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), + { + 'name': 'HomeAssistant 1', + 'token': new_token, + } + ) + home_assistant_config_obj.refresh_from_db() + assert r.status_code == 200 + assert home_assistant_config_obj.token == new_token + + r = a1_s2.post( + reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), + { + 'name': 'HomeAssistant 1', + 'token': new_token, + } + ) + assert r.status_code == 404 + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 302], + ['g1_s1', 302], + ['u1_s1', 302], + ['a1_s1', 200], + ['g1_s2', 302], + ['u1_s2', 302], + ['a1_s2', 404], + ]) +def test_view_permission(arg, request, home_assistant_config_obj): + c = request.getfixturevalue(arg[0]) + assert c.get(reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk})).status_code == arg[1] diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py new file mode 100644 index 000000000..c6df778fb --- /dev/null +++ b/cookbook/tests/other/test_connector_manager.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass + +import pytest +from django.contrib import auth +from mock.mock import Mock + +from cookbook.connectors.connector import Connector +from cookbook.connectors.connector_manager import run_connectors, ActionType +from cookbook.models import ShoppingListEntry, ShoppingList, Food + + +@pytest.fixture() +def obj_1(space_1, u1_s1): + e = ShoppingListEntry.objects.create(created_by=auth.get_user(u1_s1), food=Food.objects.get_or_create(name='test 1', space=space_1)[0], space=space_1) + s = ShoppingList.objects.create(created_by=auth.get_user(u1_s1), space=space_1, ) + s.entries.add(e) + return e + + +@pytest.mark.asyncio +async def test_run_connectors(space_1, u1_s1, obj_1) -> None: + connector_mock = Mock(spec=Connector) + + await run_connectors([connector_mock], space_1, obj_1, ActionType.DELETED) + + assert not connector_mock.on_shopping_list_entry_updated.called + assert not connector_mock.on_shopping_list_entry_created.called + connector_mock.on_shopping_list_entry_deleted.assert_called_once_with(space_1, obj_1) diff --git a/requirements.txt b/requirements.txt index 0611ea423..23789d2ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ django-allauth==0.58.1 recipe-scrapers==14.52.0 django-scopes==2.0.0 pytest==7.4.3 +pytest-asyncio==0.23.3 pytest-django==4.6.0 django-treebeard==4.7 django-cors-headers==4.2.0 From 9c804863a815bb83eee5119fff4ea63b5be48e73 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:15:28 +0100 Subject: [PATCH 09/54] undo accidental changes --- cookbook/models.py | 59 +++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/cookbook/models.py b/cookbook/models.py index a28cc5a51..9d6b9c5ae 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -49,16 +49,14 @@ def get_active_space(self): def get_shopping_share(self): # get list of users that shared shopping list with user. Django ORM forbids this type of query, so raw is required - return User.objects.raw( - ' '.join( - [ - 'SELECT auth_user.id FROM auth_user', - 'INNER JOIN cookbook_userpreference', - 'ON (auth_user.id = cookbook_userpreference.user_id)', - 'INNER JOIN cookbook_userpreference_shopping_share', - 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', - 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) - ])) + return User.objects.raw(' '.join([ + 'SELECT auth_user.id FROM auth_user', + 'INNER JOIN cookbook_userpreference', + 'ON (auth_user.id = cookbook_userpreference.user_id)', + 'INNER JOIN cookbook_userpreference_shopping_share', + 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', + 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) + ])) auth.models.User.add_to_class('get_user_display_name', get_user_display_name) @@ -700,11 +698,10 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): if len(inherit) > 0: # ManyToMany cannot be updated through an UPDATE operation for i in inherit: - trough.objects.bulk_create( - [ - trough(food_id=x, foodinheritfield_id=i['id']) - for x in Food.objects.filter(tree_filter).values_list('id', flat=True) - ]) + trough.objects.bulk_create([ + trough(food_id=x, foodinheritfield_id=i['id']) + for x in Food.objects.filter(tree_filter).values_list('id', flat=True) + ]) inherit = [x['field'] for x in inherit] for field in ['ignore_shopping', 'substitute_children', 'substitute_siblings']: @@ -831,9 +828,8 @@ class PropertyType(models.Model, PermissionModelMixin): unit = models.CharField(max_length=64, blank=True, null=True) order = models.IntegerField(default=0) description = models.CharField(max_length=512, blank=True, null=True) - category = models.CharField( - max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), - (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) + category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), + (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) fdc_id = models.IntegerField(null=True, default=None, blank=True) @@ -1396,20 +1392,19 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis UNIT_REPLACE = 'UNIT_REPLACE' NAME_REPLACE = 'NAME_REPLACE' - type = models.CharField( - max_length=128, - choices=( - (FOOD_ALIAS, _('Food Alias')), - (UNIT_ALIAS, _('Unit Alias')), - (KEYWORD_ALIAS, _('Keyword Alias')), - (DESCRIPTION_REPLACE, _('Description Replace')), - (INSTRUCTION_REPLACE, _('Instruction Replace')), - (NEVER_UNIT, _('Never Unit')), - (TRANSPOSE_WORDS, _('Transpose Words')), - (FOOD_REPLACE, _('Food Replace')), - (UNIT_REPLACE, _('Unit Replace')), - (NAME_REPLACE, _('Name Replace')), - )) + type = models.CharField(max_length=128, + choices=( + (FOOD_ALIAS, _('Food Alias')), + (UNIT_ALIAS, _('Unit Alias')), + (KEYWORD_ALIAS, _('Keyword Alias')), + (DESCRIPTION_REPLACE, _('Description Replace')), + (INSTRUCTION_REPLACE, _('Instruction Replace')), + (NEVER_UNIT, _('Never Unit')), + (TRANSPOSE_WORDS, _('Transpose Words')), + (FOOD_REPLACE, _('Food Replace')), + (UNIT_REPLACE, _('Unit Replace')), + (NAME_REPLACE, _('Name Replace')), + )) name = models.CharField(max_length=128, default='') description = models.TextField(blank=True, null=True) From 022439e01711ca3c7b5c7e721dfe8017191e4922 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:40:16 +0100 Subject: [PATCH 10/54] increase queue size to account for recipe adding burst --- cookbook/connectors/connector_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index d660b6d84..332f2f0db 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -17,7 +17,7 @@ from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 -QUEUE_MAX_SIZE = 10 +QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector @@ -116,6 +116,9 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE for connector in connectors: tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + if len(tasks) == 0: + return + try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: From 1a37961ceba786c192d71ea1e7bc864744454514 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:44:15 +0100 Subject: [PATCH 11/54] add mock to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 23789d2ce..16fca7317 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ pyyaml==6.0.1 uritemplate==4.1.1 beautifulsoup4==4.12.2 microdata==0.8.0 +mock==5.1.0 Jinja2==3.1.2 django-webpack-loader==1.8.1 git+https://github.com/BITSOLVER/django-js-reverse@071e304fd600107bc64bbde6f2491f1fe049ec82 From 50eb232fff7eba47f370da31b0aa818e8c02b333 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 00:24:58 +0100 Subject: [PATCH 12/54] update tests and fix small bug in connector_manager --- cookbook/connectors/connector_manager.py | 11 ++++++----- .../edits/test_edits_home_assistant_config.py | 19 +++++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 332f2f0db..833e771ac 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -13,12 +13,13 @@ from django_scopes import scope from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 -REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector +REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan +CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ConnectorConfig class ActionType(Enum): @@ -35,7 +36,7 @@ class Work: class ConnectorManager: _queue: Queue - _listening_to_classes = REGISTERED_CLASSES + _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES def __init__(self): self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) @@ -81,10 +82,10 @@ class ConnectorManager: break # If a Connector was changed/updated, refresh connector from the database for said space - refresh_connector_cache = isinstance(item.instance, Connector) + refresh_connector_cache = isinstance(item.instance, CONNECTOR_UPDATE_CLASSES) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.name, None) + connectors: Optional[List[Connector]] = _connectors.get(space.name) if connectors is None or refresh_connector_cache: with scope(space=space): diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py index 053972b62..7f2e19a91 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -1,7 +1,8 @@ -from cookbook.models import Storage, HomeAssistantConfig +import pytest from django.contrib import auth from django.urls import reverse -import pytest + +from cookbook.models import HomeAssistantConfig EDIT_VIEW_NAME = 'edit_home_assistant_config' @@ -19,25 +20,31 @@ def home_assistant_config_obj(a1_s1, space_1): ) -def test_edit_home_assistant_config(home_assistant_config_obj, a1_s1, a1_s2): +def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConfig, a1_s1, a1_s2): new_token = '1234_token' r = a1_s1.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { - 'name': 'HomeAssistant 1', + 'name': home_assistant_config_obj.name, + 'url': home_assistant_config_obj.url, + 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'enabled': home_assistant_config_obj.enabled, } ) - home_assistant_config_obj.refresh_from_db() assert r.status_code == 200 + home_assistant_config_obj.refresh_from_db() assert home_assistant_config_obj.token == new_token r = a1_s2.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { - 'name': 'HomeAssistant 1', + 'name': home_assistant_config_obj.name, + 'url': home_assistant_config_obj.url, + 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'enabled': home_assistant_config_obj.enabled, } ) assert r.status_code == 404 From 48ac70de95cffdda0c4197ce54437a17632dbc3c Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 11:56:51 +0100 Subject: [PATCH 13/54] make the tests check for any error message --- .../tests/edits/test_edits_home_assistant_config.py | 9 +++++++-- cookbook/tests/edits/test_edits_storage.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py index 7f2e19a91..0df507de8 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -1,5 +1,7 @@ import pytest from django.contrib import auth +from django.contrib import messages +from django.contrib.messages import get_messages from django.urls import reverse from cookbook.models import HomeAssistantConfig @@ -12,7 +14,7 @@ def home_assistant_config_obj(a1_s1, space_1): return HomeAssistantConfig.objects.create( name='HomeAssistant 1', token='token', - url='url', + url='http://localhost:8123/api', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(a1_s1), @@ -28,12 +30,15 @@ def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConf { 'name': home_assistant_config_obj.name, 'url': home_assistant_config_obj.url, - 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'todo_entity': home_assistant_config_obj.todo_entity, 'enabled': home_assistant_config_obj.enabled, } ) assert r.status_code == 200 + r_messages = [m for m in get_messages(r.wsgi_request)] + assert not any(m.level > messages.SUCCESS for m in r_messages) + home_assistant_config_obj.refresh_from_db() assert home_assistant_config_obj.token == new_token diff --git a/cookbook/tests/edits/test_edits_storage.py b/cookbook/tests/edits/test_edits_storage.py index 125445e15..9c4e08e1c 100644 --- a/cookbook/tests/edits/test_edits_storage.py +++ b/cookbook/tests/edits/test_edits_storage.py @@ -1,7 +1,10 @@ -from cookbook.models import Storage -from django.contrib import auth -from django.urls import reverse import pytest +from django.contrib import auth +from django.contrib import messages +from django.contrib.messages import get_messages +from django.urls import reverse + +from cookbook.models import Storage @pytest.fixture @@ -29,6 +32,9 @@ def test_edit_storage(storage_obj, a1_s1, a1_s2): ) storage_obj.refresh_from_db() assert r.status_code == 200 + r_messages = [m for m in get_messages(r.wsgi_request)] + assert not any(m.level > messages.SUCCESS for m in r_messages) + assert storage_obj.password == '1234_pw' assert storage_obj.token == '1234_token' From c7dd61e2391bae009b8e76dd525e114cc793d1ab Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 11:57:08 +0100 Subject: [PATCH 14/54] add caching to the ci-cd workflow --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d699a5641..c4a47812b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,31 +10,65 @@ jobs: max-parallel: 4 matrix: python-version: ['3.10'] + node-version: ['18'] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - uses: awalsh128/cache-apt-pkgs-action@v1.3.1 + with: + packages: libsasl2-dev python3-dev libldap2-dev libssl-dev + version: 1.0 + + # Setup python & dependencies + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + # Build Vue frontend - - uses: actions/setup-node@v3 + - name: Set up Node ${{ matrix.node-version }} + uses: actions/setup-node@v3 with: - node-version: '18' + node-version: ${{ matrix.node-version }} + cache: 'yarn' + cache-dependency-path: ./vue/yarn.lock + - name: Install Vue dependencies working-directory: ./vue run: yarn install + - name: Build Vue dependencies working-directory: ./vue run: yarn build - - name: Install Django dependencies + + # Build backend + - name: Cache Django collectstatic + uses: actions/cache@v2 + with: + path: ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic + + - name: Compile Django StatisFiles run: | - sudo apt-get -y update - sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev - python -m pip install --upgrade pip - pip install -r requirements.txt python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse + - name: Django Testing project - run: | - pytest + run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + comment_mode: off + files: | + junit/test-results-${{ matrix.python-version }}.xml From 87ede4b9cc0e9d26772ad9ddc46af960dcf9610f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 13:43:08 +0100 Subject: [PATCH 15/54] change formatting a bit, and add async close method --- cookbook/connectors/connector.py | 4 +++ cookbook/connectors/connector_manager.py | 31 +++++++++++++++++++----- cookbook/connectors/homeassistant.py | 12 +++++---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 76699e09a..4ed613500 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,5 +16,9 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass + @abstractmethod + async def close(self) -> None: + pass + # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 833e771ac..39919597e 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -5,7 +5,7 @@ import queue from asyncio import Task from dataclasses import dataclass from enum import Enum -from multiprocessing import Queue +from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional @@ -35,11 +35,11 @@ class Work: class ConnectorManager: - _queue: Queue + _queue: JoinableQueue _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES def __init__(self): - self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) + self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() @@ -60,14 +60,16 @@ class ConnectorManager: try: self._queue.put_nowait(Work(instance, action_type)) except queue.Full: + logging.info("queue was full, so skipping %s", instance) return def stop(self): + self._queue.join() self._queue.close() self._worker.join() @staticmethod - def worker(worker_queue: Queue): + def worker(worker_queue: JoinableQueue): from django.db import connections connections.close_all() @@ -77,7 +79,10 @@ class ConnectorManager: _connectors: Dict[str, List[Connector]] = dict() while True: - item: Optional[Work] = worker_queue.get() + try: + item: Optional[Work] = worker_queue.get() + except KeyboardInterrupt: + break if item is None: break @@ -88,18 +93,32 @@ class ConnectorManager: connectors: Optional[List[Connector]] = _connectors.get(space.name) if connectors is None or refresh_connector_cache: + if connectors is not None: + loop.run_until_complete(close_connectors(connectors)) + with scope(space=space): connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: - return + worker_queue.task_done() + continue loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) + worker_queue.task_done() loop.close() +async def close_connectors(connectors: List[Connector]): + tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException: + logging.exception("received an exception while closing one of the connectors") + + async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType): tasks: List[Task] = list() diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index f5ae454fc..4499c7e7b 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -1,7 +1,5 @@ import logging -from collections import defaultdict from logging import Logger -from typing import Dict, Any, Optional from homeassistant_api import Client, HomeassistantAPIError, Domain @@ -58,16 +56,20 @@ class HomeAssistant(Connector): except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + async def close(self) -> None: + await self._client.async_cache_session.close() + def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): item = shopping_list_entry.food.name if shopping_list_entry.amount > 0: + item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.') if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0: - item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.base_unit})" + item += f" {shopping_list_entry.unit.base_unit})" elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0: - item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.name})" + item += f" {shopping_list_entry.unit.name})" else: - item += f" ({shopping_list_entry.amount})" + item += ")" description = "Imported by TandoorRecipes" if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0: From 362c0340fce379a87d83ec05684f1ba5504e14a1 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 16:16:05 +0100 Subject: [PATCH 16/54] skip whole yarn and static files if there was a cache hit --- .github/workflows/ci.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4a47812b..8d70b90f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,20 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Cache StaticFiles + uses: actions/cache@v3 + id: django_cache + with: + path: | + ./cookbook/static + ./vue/webpack-stats.json + ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} + # Build Vue frontend - name: Set up Node ${{ matrix.node-version }} + if: steps.django_cache.outputs.cache-hit != 'true' uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} @@ -40,24 +52,18 @@ jobs: cache-dependency-path: ./vue/yarn.lock - name: Install Vue dependencies + if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn install - name: Build Vue dependencies + if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn build # Build backend - - name: Cache Django collectstatic - uses: actions/cache@v2 - with: - path: ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic - - name: Compile Django StatisFiles + if: steps.django_cache.outputs.cache-hit != 'true' run: | python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse From 17163b0dbadedb0d0411c86984955ceaa3fb08d5 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 16:44:18 +0100 Subject: [PATCH 17/54] save cache on failed tests --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d70b90f8..cb72bc0c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: key: | ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - # Build Vue frontend + # Build Vue frontend & Dependencies - name: Set up Node ${{ matrix.node-version }} if: steps.django_cache.outputs.cache-hit != 'true' uses: actions/setup-node@v3 @@ -61,13 +61,22 @@ jobs: working-directory: ./vue run: yarn build - # Build backend - name: Compile Django StatisFiles if: steps.django_cache.outputs.cache-hit != 'true' run: | python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse + - uses: actions/cache/save@v3 + if: steps.django_cache.outputs.cache-hit != 'true' + with: + path: | + ./cookbook/static + ./vue/webpack-stats.json + ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} + - name: Django Testing project run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml From fb65100b140adc42356da44d78964b13dfb8553d Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 20:30:54 +0100 Subject: [PATCH 18/54] add debug logging --- cookbook/connectors/homeassistant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index 4499c7e7b..e394e974d 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -31,6 +31,7 @@ class HomeAssistant(Connector): todo_domain = await self._client.async_get_domain('todo') self._domains_cache['todo'] = todo_domain + logging.debug(f"pushing {item} to {self._config.name}") await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") @@ -52,6 +53,7 @@ class HomeAssistant(Connector): todo_domain = await self._client.async_get_domain('todo') self._domains_cache['todo'] = todo_domain + logging.debug(f"deleting {item} from {self._config.name}") await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") From 245787b89e5bc743503dbeaad63045a050c85bb5 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sun, 14 Jan 2024 16:59:54 +0100 Subject: [PATCH 19/54] make the connectors form be able to display all types for connectors --- cookbook/admin.py | 10 ++- cookbook/connectors/connector_manager.py | 10 ++- cookbook/connectors/example.py | 27 +++++++ cookbook/forms.py | 57 +++++++++++---- ...0208_homeassistantconfig_exampleconfig.py} | 20 ++++- cookbook/models.py | 5 ++ cookbook/tables.py | 41 ++++++++++- cookbook/templates/base.html | 2 +- cookbook/templates/list_connectors.html | 55 ++++++++++++++ cookbook/urls.py | 7 +- cookbook/views/delete.py | 27 +++---- cookbook/views/edit.py | 73 ++++++++++--------- cookbook/views/lists.py | 17 +---- cookbook/views/new.py | 44 +++++++++-- 14 files changed, 302 insertions(+), 93 deletions(-) create mode 100644 cookbook/connectors/example.py rename cookbook/migrations/{0208_homeassistantconfig.py => 0208_homeassistantconfig_exampleconfig.py} (58%) create mode 100644 cookbook/templates/list_connectors.html diff --git a/cookbook/admin.py b/cookbook/admin.py index 3429e63d3..64a031734 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig) + ViewLog, HomeAssistantConfig, ExampleConfig) class CustomUserAdmin(UserAdmin): @@ -103,6 +103,14 @@ class HomeAssistantConfigAdmin(admin.ModelAdmin): admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) +class ExampleConfigAdmin(admin.ModelAdmin): + list_display = ('id', 'name', 'enabled', 'feed_url') + search_fields = ('name',) + + +admin.site.register(ExampleConfig, ExampleConfigAdmin) + + class SyncAdmin(admin.ModelAdmin): list_display = ('storage', 'path', 'active', 'last_checked') search_fields = ('storage__name', 'path') diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 39919597e..178c8efbc 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -12,14 +12,15 @@ from typing import List, Any, Dict, Optional from django_scopes import scope from cookbook.connectors.connector import Connector +from cookbook.connectors.example import Example from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ConnectorConfig +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ExampleConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan -CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ConnectorConfig +CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ExampleConfig class ActionType(Enum): @@ -97,7 +98,10 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] + connectors: List[Connector] = [ + *(HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled), + *(Example(config) for config in space.exampleconfig_set.all() if config.enabled) + ] _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: diff --git a/cookbook/connectors/example.py b/cookbook/connectors/example.py new file mode 100644 index 000000000..ce55f476d --- /dev/null +++ b/cookbook/connectors/example.py @@ -0,0 +1,27 @@ +from cookbook.connectors.connector import Connector +from cookbook.models import ExampleConfig, Space, ShoppingListEntry + + +class Example(Connector): + _config: ExampleConfig + + def __init__(self, config: ExampleConfig): + self._config = config + + async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_created_enabled: + return + pass + + async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_updated_enabled: + return + pass + + async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_deleted_enabled: + return + pass + + async def close(self) -> None: + pass diff --git a/cookbook/forms.py b/cookbook/forms.py index dce4da57d..18867958d 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig) + SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig, ExampleConfig) class SelectWidget(widgets.Select): @@ -188,12 +188,35 @@ class StorageForm(forms.ModelForm): } -class HomeAssistantConfigForm(forms.ModelForm): - token = forms.CharField( - widget=forms.TextInput( - attrs={'autocomplete': 'new-password', 'type': 'password'} - ), - required=True, +class ConnectorConfigForm(forms.ModelForm): + enabled = forms.BooleanField( + help_text="Is the connector enabled", + required=False, + ) + + on_shopping_list_entry_created_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry created events", + required=False, + ) + + on_shopping_list_entry_updated_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry updated events", + required=False, + ) + + on_shopping_list_entry_deleted_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry deleted events", + required=False, + ) + + class Meta: + fields = ('name', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + + +class HomeAssistantConfigForm(ConnectorConfigForm): + update_token = forms.CharField( + widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}), + required=False, help_text=_('Long Lived Access Token for your HomeAssistant instance') ) @@ -202,11 +225,6 @@ class HomeAssistantConfigForm(forms.ModelForm): help_text=_('Something like http://homeassistant.local:8123/api'), ) - enabled = forms.BooleanField( - help_text="Is the HomeAssistantConnector enabled", - required=False, - ) - on_shopping_list_entry_created_enabled = forms.BooleanField( help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", required=False, @@ -224,15 +242,24 @@ class HomeAssistantConfigForm(forms.ModelForm): class Meta: model = HomeAssistantConfig - fields = ( - 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', - 'on_shopping_list_entry_deleted_enabled') + + fields = ConnectorConfigForm.Meta.fields + ('url', 'todo_entity') help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), } +class ExampleConfigForm(ConnectorConfigForm): + feed_url = forms.URLField( + required=False, + ) + + class Meta: + model = ExampleConfig + fields = ConnectorConfigForm.Meta.fields + ('feed_url',) + + # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py similarity index 58% rename from cookbook/migrations/0208_homeassistantconfig.py rename to cookbook/migrations/0208_homeassistantconfig_exampleconfig.py index f21ff06ae..789094c12 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-12 21:01 +# Generated by Django 4.2.7 on 2024-01-14 16:00 import cookbook.models from django.conf import settings @@ -35,4 +35,22 @@ class Migration(migrations.Migration): }, bases=(models.Model, cookbook.models.PermissionModelMixin), ), + migrations.CreateModel( + name='ExampleConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), + ('feed_url', models.URLField(blank=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + options={ + 'abstract': False, + }, + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), ] diff --git a/cookbook/models.py b/cookbook/models.py index 9d6b9c5ae..a41dc37f2 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -340,6 +340,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() HomeAssistantConfig.objects.filter(space=self).delete() + ExampleConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -387,6 +388,10 @@ class HomeAssistantConfig(ConnectorConfig): todo_entity = models.CharField(max_length=128, default='todo.shopping_list') +class ExampleConfig(ConnectorConfig): + feed_url = models.URLField(blank=True) + + class UserPreference(models.Model, PermissionModelMixin): # Themes BOOTSTRAP = 'BOOTSTRAP' diff --git a/cookbook/tables.py b/cookbook/tables.py index 4f49690cb..ba14fbbeb 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -1,9 +1,14 @@ +from typing import Any, Dict + import django_tables2 as tables from django.utils.html import format_html from django.utils.translation import gettext as _ +from django.views.generic import TemplateView +from django_tables2 import MultiTableMixin from django_tables2.utils import A -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig +from .helper.permission_helper import GroupRequiredMixin +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig, ExampleConfig class StorageTable(tables.Table): @@ -15,6 +20,16 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') +class ExampleConfigTable(tables.Table): + id = tables.LinkColumn('edit_example_config', args=[A('id')]) + + class Meta: + model = ExampleConfig + template_name = 'generic/table_template.html' + fields = ('id', 'name', 'enabled', 'feed_url') + attrs = {'table_name': "Example Configs", 'create_url': 'new_example_config'} + + class HomeAssistantConfigTable(tables.Table): id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) @@ -22,6 +37,30 @@ class HomeAssistantConfigTable(tables.Table): model = HomeAssistantConfig template_name = 'generic/table_template.html' fields = ('id', 'name', 'enabled', 'url') + attrs = {'table_name': "HomeAssistant Configs", 'create_url': 'new_home_assistant_config'} + + +class ConnectorConfigTable(GroupRequiredMixin, MultiTableMixin, TemplateView): + groups_required = ['admin'] + template_name = "list_connectors.html" + + table_pagination = { + "per_page": 25 + } + + def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: + kwargs = super().get_context_data(**kwargs) + kwargs['title'] = _("Connectors") + return kwargs + + def get_tables(self): + example_configs = ExampleConfig.objects.filter(space=self.request.space).all() + home_assistant_configs = HomeAssistantConfig.objects.filter(space=self.request.space).all() + + return [ + ExampleConfigTable(example_configs), + HomeAssistantConfigTable(home_assistant_configs) + ] class ImportLogTable(tables.Table): diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index d9e2eda15..7fbb14dbb 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -336,7 +336,7 @@ class="fas fa-server fa-fw"> {% trans 'Space Settings' %} {% endif %} {% if request.user == request.space.created_by or user.is_superuser %} - {% trans 'External Connectors' %} {% endif %} {% if user.is_superuser %} diff --git a/cookbook/templates/list_connectors.html b/cookbook/templates/list_connectors.html new file mode 100644 index 000000000..283f7c314 --- /dev/null +++ b/cookbook/templates/list_connectors.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} +{% load crispy_forms_tags %} +{% load i18n %} +{% load django_tables2 %} + +{% block title %}{% trans 'List' %}{% endblock %} + + +{% block content %} + +{% if request.resolver_match.url_name in 'list_storage,list_recipe_import,list_sync_log' %} + +{% endif %} + +
+ +

{{ title }} {% trans 'List' %}

+ + + {% if filter %} +
+
+
+ {% csrf_token %} + {{ filter.form|crispy }} + +
+ {% endif %} + + {% if import_btn %} + {% trans 'Import all' %} +
+
+ {% endif %} + + {% for table in tables %} + +

{{ table.attrs.table_name }} {% trans 'List' %} + {% if table.attrs.create_url %} + + {% endif %} +

+ + + {% render_table table %} + {% endfor %} + +
+ +{% endblock content %} \ No newline at end of file diff --git a/cookbook/urls.py b/cookbook/urls.py index 96735037e..e85cfbaf9 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,7 +12,8 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name, HomeAssistantConfig) + UserFile, UserSpace, get_model_name, HomeAssistantConfig, ExampleConfig) +from .tables import ConnectorConfigTable from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -115,7 +116,7 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), - path('edit/home-assistant-config//', edit.edit_home_assistant_config, name='edit_home_assistant_config'), + path('list/connectors', ConnectorConfigTable.as_view(), name='list_connectors'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -168,7 +169,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, HomeAssistantConfig, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, HomeAssistantConfig, ExampleConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index b6cf857b0..bbdc98ee0 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig) + RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig, ExampleConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -126,23 +126,24 @@ class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): groups_required = ['admin'] template_name = "generic/delete_template.html" model = HomeAssistantConfig - success_url = reverse_lazy('list_storage') + success_url = reverse_lazy('list_connectors') def get_context_data(self, **kwargs): - context = super(HomeAssistantConfigDelete, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context['title'] = _("HomeAssistant Config Backend") return context - def post(self, request, *args, **kwargs): - try: - return self.delete(request, *args, **kwargs) - except ProtectedError: - messages.add_message( - request, - messages.WARNING, - _('Could not delete this storage backend as it is used in at least one monitor.') # noqa: E501 - ) - return HttpResponseRedirect(reverse('list_storage')) + +class ExampleConfigDelete(GroupRequiredMixin, DeleteView): + groups_required = ['admin'] + template_name = "generic/delete_template.html" + model = ExampleConfig + success_url = reverse_lazy('list_connectors') + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("Example Config Backend") + return context class CommentDelete(OwnerRequiredMixin, DeleteView): diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index deb9654bd..2f99216b3 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -9,10 +9,10 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm, ExampleConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig, ExampleConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -128,46 +128,49 @@ def edit_storage(request, pk): ) -@group_required('admin') -def edit_home_assistant_config(request, pk): - instance: HomeAssistantConfig = get_object_or_404(HomeAssistantConfig, pk=pk, space=request.space) +class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): + groups_required = ['admin'] + template_name = "generic/edit_template.html" + model = HomeAssistantConfig + form_class = HomeAssistantConfigForm - if not (instance.created_by == request.user or request.user.is_superuser): - messages.add_message(request, messages.ERROR, _('You cannot edit this homeassistant config!')) - return HttpResponseRedirect(reverse('edit_home_assistant_config')) + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs['initial']['update_token'] = VALUE_NOT_CHANGED + return kwargs - if request.space.demo or settings.HOSTED: - messages.add_message(request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) - return redirect('index') + def form_valid(self, form): + if form.cleaned_data['update_token'] != VALUE_NOT_CHANGED and form.cleaned_data['update_token'] != "": + form.instance.token = form.cleaned_data['update_token'] + messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) + return super(HomeAssistantConfigUpdate, self).form_valid(form) - if request.method == "POST": - form = HomeAssistantConfigForm(request.POST, instance=copy.deepcopy(instance)) - if form.is_valid(): - instance.name = form.cleaned_data['name'] - instance.url = form.cleaned_data['url'] - instance.todo_entity = form.cleaned_data['todo_entity'] - instance.enabled = form.cleaned_data['enabled'] - instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] - instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] - instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] + def get_success_url(self): + return reverse('edit_home_assistant_config', kwargs={'pk': self.object.pk}) - if form.cleaned_data['token'] != VALUE_NOT_CHANGED: - instance.token = form.cleaned_data['token'] + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("HomeAssistantConfig") + return context - instance.save() - messages.add_message(request, messages.SUCCESS, _('HomeAssistant config saved!')) - else: - messages.add_message(request, messages.ERROR, _('There was an error updating this config!')) - else: - instance.token = VALUE_NOT_CHANGED - form = HomeAssistantConfigForm(instance=instance) +class ExampleConfigUpdate(GroupRequiredMixin, UpdateView): + groups_required = ['admin'] + template_name = "generic/edit_template.html" + model = ExampleConfig + form_class = ExampleConfigForm - return render( - request, - 'generic/edit_template.html', - {'form': form, 'title': _('HomeAssistantConfig')} - ) + def form_valid(self, form): + messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) + return super().form_valid(form) + + def get_success_url(self): + return reverse('edit_example_config', kwargs={'pk': self.object.pk}) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("ExampleConfig") + return context class CommentUpdate(OwnerRequiredMixin, UpdateView): diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index 20dbb73ca..ed43c143a 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, HomeAssistantConfig -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, HomeAssistantConfigTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable @group_required('admin') @@ -65,19 +65,6 @@ def storage(request): ) -@group_required('admin') -def home_assistant_config(request): - table = HomeAssistantConfigTable(HomeAssistantConfig.objects.filter(space=request.space).all()) - RequestConfig(request, paginate={'per_page': 25}).configure(table) - - return render( - request, 'generic/list_template.html', { - 'title': _("HomeAssistant Config Backend"), - 'table': table, - 'create_url': 'new_home_assistant_config' - }) - - @group_required('admin') def invite_link(request): table = InviteLinkTable( diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 263dca092..22bfbc84c 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -5,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm, ExampleConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig, ExampleConfig from recipes import settings @@ -77,6 +77,40 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): form_class = HomeAssistantConfigForm success_url = reverse_lazy('list_home_assistant_config') + def get_form_class(self): + form_class = super().get_form_class() + + if self.request.method == 'GET': + update_token_field = form_class.base_fields['update_token'] + update_token_field.required = True + + return form_class + + def form_valid(self, form): + if self.request.space.demo or settings.HOSTED: + messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + obj = form.save(commit=False) + obj.token = form.cleaned_data['update_token'] + obj.created_by = self.request.user + obj.space = self.request.space + obj.save() + return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + +class ExampleConfigCreate(GroupRequiredMixin, CreateView): + groups_required = ['admin'] + template_name = "generic/new_template.html" + model = ExampleConfig + form_class = ExampleConfigForm + success_url = reverse_lazy('list_connectors') + def form_valid(self, form): if self.request.space.demo or settings.HOSTED: messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) @@ -86,11 +120,11 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): obj.created_by = self.request.user obj.space = self.request.space obj.save() - return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + return HttpResponseRedirect(reverse('edit_example_config', kwargs={'pk': obj.pk})) def get_context_data(self, **kwargs): - context = super(HomeAssistantConfigCreate, self).get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") + context = super().get_context_data(**kwargs) + context['title'] = _("Example Config Backend") return context From 409c0295ec7e2b2c51812773f7e1c0f5a3ca9bbf Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 17 Jan 2024 22:25:02 +0100 Subject: [PATCH 20/54] convert example & homeassistant specific configs to a generic with all optional fields --- cookbook/admin.py | 16 ++---- cookbook/connectors/connector.py | 7 ++- cookbook/connectors/connector_manager.py | 25 ++++++--- cookbook/connectors/example.py | 27 --------- cookbook/connectors/homeassistant.py | 9 ++- cookbook/forms.py | 41 +++----------- cookbook/migrations/0208_connectorconfig.py | 36 ++++++++++++ .../0208_homeassistantconfig_exampleconfig.py | 56 ------------------- cookbook/models.py | 25 ++++----- cookbook/serializer.py | 6 +- cookbook/tables.py | 49 ++-------------- cookbook/templates/base.html | 2 +- .../api/test_api_home_assistant_config.py | 12 ++-- ...nfig.py => test_edits_connector_config.py} | 19 ++++--- cookbook/urls.py | 8 +-- cookbook/views/api.py | 11 ++-- cookbook/views/delete.py | 22 ++------ cookbook/views/edit.py | 35 +++--------- cookbook/views/lists.py | 20 ++++++- cookbook/views/new.py | 49 +++------------- 20 files changed, 159 insertions(+), 316 deletions(-) delete mode 100644 cookbook/connectors/example.py create mode 100644 cookbook/migrations/0208_connectorconfig.py delete mode 100644 cookbook/migrations/0208_homeassistantconfig_exampleconfig.py rename cookbook/tests/edits/{test_edits_home_assistant_config.py => test_edits_connector_config.py} (77%) diff --git a/cookbook/admin.py b/cookbook/admin.py index 64a031734..fcc007bc1 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig, ExampleConfig) + ViewLog, ConnectorConfig) class CustomUserAdmin(UserAdmin): @@ -95,20 +95,12 @@ class StorageAdmin(admin.ModelAdmin): admin.site.register(Storage, StorageAdmin) -class HomeAssistantConfigAdmin(admin.ModelAdmin): - list_display = ('id', 'name', 'enabled', 'url') +class ConnectorConfigAdmin(admin.ModelAdmin): + list_display = ('id', 'name', 'type', 'enabled', 'url') search_fields = ('name', 'url') -admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) - - -class ExampleConfigAdmin(admin.ModelAdmin): - list_display = ('id', 'name', 'enabled', 'feed_url') - search_fields = ('name',) - - -admin.site.register(ExampleConfig, ExampleConfigAdmin) +admin.site.register(ConnectorConfig, ConnectorConfigAdmin) class SyncAdmin(admin.ModelAdmin): diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 4ed613500..3647dc71e 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod -from cookbook.models import ShoppingListEntry, Space +from cookbook.models import ShoppingListEntry, Space, ConnectorConfig class Connector(ABC): + @abstractmethod + def __init__(self, config: ConnectorConfig): + pass + @abstractmethod async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: pass @@ -20,5 +24,4 @@ class Connector(ABC): async def close(self) -> None: pass - # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 178c8efbc..86a60b048 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -12,15 +12,13 @@ from typing import List, Any, Dict, Optional from django_scopes import scope from cookbook.connectors.connector import Connector -from cookbook.connectors.example import Example from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ExampleConfig +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan -CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ExampleConfig class ActionType(Enum): @@ -37,7 +35,7 @@ class Work: class ConnectorManager: _queue: JoinableQueue - _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES + _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) @@ -88,7 +86,7 @@ class ConnectorManager: break # If a Connector was changed/updated, refresh connector from the database for said space - refresh_connector_cache = isinstance(item.instance, CONNECTOR_UPDATE_CLASSES) + refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space connectors: Optional[List[Connector]] = _connectors.get(space.name) @@ -98,10 +96,11 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = [ - *(HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled), - *(Example(config) for config in space.exampleconfig_set.all() if config.enabled) - ] + connectors: List[Connector] = list( + filter( + lambda x: x is not None, + [ConnectorManager.get_connected_for_config(config) for config in space.connectorconfig_set.all() if config.enabled], + )) _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: @@ -113,6 +112,14 @@ class ConnectorManager: loop.close() + @staticmethod + def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: + match config.type: + case ConnectorConfig.HOMEASSISTANT: + return HomeAssistant(config) + case _: + return None + async def close_connectors(connectors: List[Connector]): tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] diff --git a/cookbook/connectors/example.py b/cookbook/connectors/example.py deleted file mode 100644 index ce55f476d..000000000 --- a/cookbook/connectors/example.py +++ /dev/null @@ -1,27 +0,0 @@ -from cookbook.connectors.connector import Connector -from cookbook.models import ExampleConfig, Space, ShoppingListEntry - - -class Example(Connector): - _config: ExampleConfig - - def __init__(self, config: ExampleConfig): - self._config = config - - async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_created_enabled: - return - pass - - async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_updated_enabled: - return - pass - - async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_deleted_enabled: - return - pass - - async def close(self) -> None: - pass diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index e394e974d..e653c3f3d 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -4,16 +4,19 @@ from logging import Logger from homeassistant_api import Client, HomeassistantAPIError, Domain from cookbook.connectors.connector import Connector -from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space +from cookbook.models import ShoppingListEntry, ConnectorConfig, Space class HomeAssistant(Connector): _domains_cache: dict[str, Domain] - _config: HomeAssistantConfig + _config: ConnectorConfig _logger: Logger _client: Client - def __init__(self, config: HomeAssistantConfig): + def __init__(self, config: ConnectorConfig): + if not config.token or not config.url or not config.todo_entity: + raise ValueError("config for HomeAssistantConnector in incomplete") + self._domains_cache = dict() self._config = config self._logger = logging.getLogger("connector.HomeAssistant") diff --git a/cookbook/forms.py b/cookbook/forms.py index 18867958d..2244acada 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig, ExampleConfig) + SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig) class SelectWidget(widgets.Select): @@ -209,11 +209,6 @@ class ConnectorConfigForm(forms.ModelForm): required=False, ) - class Meta: - fields = ('name', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') - - -class HomeAssistantConfigForm(ConnectorConfigForm): update_token = forms.CharField( widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}), required=False, @@ -221,45 +216,23 @@ class HomeAssistantConfigForm(ConnectorConfigForm): ) url = forms.URLField( - required=True, + required=False, help_text=_('Something like http://homeassistant.local:8123/api'), ) - on_shopping_list_entry_created_enabled = forms.BooleanField( - help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", - required=False, - ) - - on_shopping_list_entry_updated_enabled = forms.BooleanField( - help_text="PLACEHOLDER", - required=False, - ) - - on_shopping_list_entry_deleted_enabled = forms.BooleanField( - help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List -- Warning: Might have negative performance impact", - required=False, - ) - class Meta: - model = HomeAssistantConfig + model = ConnectorConfig - fields = ConnectorConfigForm.Meta.fields + ('url', 'todo_entity') + fields = ( + 'name', 'type', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled', 'url', 'todo_entity', + ) help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), } -class ExampleConfigForm(ConnectorConfigForm): - feed_url = forms.URLField( - required=False, - ) - - class Meta: - model = ExampleConfig - fields = ConnectorConfigForm.Meta.fields + ('feed_url',) - - # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0208_connectorconfig.py b/cookbook/migrations/0208_connectorconfig.py new file mode 100644 index 000000000..c29c02538 --- /dev/null +++ b/cookbook/migrations/0208_connectorconfig.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-01-17 21:12 + +import cookbook.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ConnectorConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('type', models.CharField(choices=[('HomeAssistant', 'HomeAssistant')], default='HomeAssistant', max_length=128)), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), + ('url', models.URLField(blank=True, null=True)), + ('token', models.CharField(blank=True, max_length=512, null=True)), + ('todo_entity', models.CharField(blank=True, max_length=128, null=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), + ] diff --git a/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py deleted file mode 100644 index 789094c12..000000000 --- a/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py +++ /dev/null @@ -1,56 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-14 16:00 - -import cookbook.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='HomeAssistantConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('url', models.URLField(blank=True)), - ('token', models.CharField(blank=True, max_length=512)), - ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - migrations.CreateModel( - name='ExampleConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('feed_url', models.URLField(blank=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - ] diff --git a/cookbook/models.py b/cookbook/models.py index a41dc37f2..f0bc1af6c 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -339,8 +339,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): SyncLog.objects.filter(sync__space=self).delete() Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() - HomeAssistantConfig.objects.filter(space=self).delete() - ExampleConfig.objects.filter(space=self).delete() + ConnectorConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -366,30 +365,28 @@ class Space(ExportModelOperationsMixin('space'), models.Model): class ConnectorConfig(models.Model, PermissionModelMixin): + HOMEASSISTANT = 'HomeAssistant' + CONNECTER_TYPE = ((HOMEASSISTANT, 'HomeAssistant'),) + name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) + type = models.CharField( + choices=CONNECTER_TYPE, max_length=128, default=HOMEASSISTANT + ) enabled = models.BooleanField(default=True, help_text="Is Connector Enabled") on_shopping_list_entry_created_enabled = models.BooleanField(default=False) on_shopping_list_entry_updated_enabled = models.BooleanField(default=False) on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False) + url = models.URLField(blank=True, null=True) + token = models.CharField(max_length=512, blank=True, null=True) + todo_entity = models.CharField(max_length=128, blank=True, null=True) + created_by = models.ForeignKey(User, on_delete=models.PROTECT) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') - class Meta: - abstract = True - - -class HomeAssistantConfig(ConnectorConfig): - url = models.URLField(blank=True) - token = models.CharField(max_length=512, blank=True) - todo_entity = models.CharField(max_length=128, default='todo.shopping_list') - - -class ExampleConfig(ConnectorConfig): - feed_url = models.URLField(blank=True) class UserPreference(models.Model, PermissionModelMixin): diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 53fc8eebc..e102c46f1 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -34,7 +34,7 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Cu ShareLink, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserPreference, UserSpace, ViewLog, HomeAssistantConfig) + UserFile, UserPreference, UserSpace, ViewLog, ConnectorConfig) from cookbook.templatetags.custom_tags import markdown from recipes.settings import AWS_ENABLED, MEDIA_URL @@ -413,14 +413,14 @@ class StorageSerializer(SpacedModelSerializer): } -class HomeAssistantConfigSerializer(SpacedModelSerializer): +class ConnectorConfigConfigSerializer(SpacedModelSerializer): def create(self, validated_data): validated_data['created_by'] = self.context['request'].user return super().create(validated_data) class Meta: - model = HomeAssistantConfig + model = ConnectorConfig fields = ( 'id', 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', diff --git a/cookbook/tables.py b/cookbook/tables.py index ba14fbbeb..3fcec9c32 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -1,14 +1,9 @@ -from typing import Any, Dict - import django_tables2 as tables from django.utils.html import format_html from django.utils.translation import gettext as _ -from django.views.generic import TemplateView -from django_tables2 import MultiTableMixin from django_tables2.utils import A -from .helper.permission_helper import GroupRequiredMixin -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig, ExampleConfig +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, ConnectorConfig class StorageTable(tables.Table): @@ -20,47 +15,13 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') -class ExampleConfigTable(tables.Table): - id = tables.LinkColumn('edit_example_config', args=[A('id')]) +class ConnectorConfigTable(tables.Table): + id = tables.LinkColumn('edit_connector_config', args=[A('id')]) class Meta: - model = ExampleConfig + model = ConnectorConfig template_name = 'generic/table_template.html' - fields = ('id', 'name', 'enabled', 'feed_url') - attrs = {'table_name': "Example Configs", 'create_url': 'new_example_config'} - - -class HomeAssistantConfigTable(tables.Table): - id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) - - class Meta: - model = HomeAssistantConfig - template_name = 'generic/table_template.html' - fields = ('id', 'name', 'enabled', 'url') - attrs = {'table_name': "HomeAssistant Configs", 'create_url': 'new_home_assistant_config'} - - -class ConnectorConfigTable(GroupRequiredMixin, MultiTableMixin, TemplateView): - groups_required = ['admin'] - template_name = "list_connectors.html" - - table_pagination = { - "per_page": 25 - } - - def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: - kwargs = super().get_context_data(**kwargs) - kwargs['title'] = _("Connectors") - return kwargs - - def get_tables(self): - example_configs = ExampleConfig.objects.filter(space=self.request.space).all() - home_assistant_configs = HomeAssistantConfig.objects.filter(space=self.request.space).all() - - return [ - ExampleConfigTable(example_configs), - HomeAssistantConfigTable(home_assistant_configs) - ] + fields = ('id', 'name', 'type', 'enabled') class ImportLogTable(tables.Table): diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index 7fbb14dbb..6505b405a 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -336,7 +336,7 @@ class="fas fa-server fa-fw"> {% trans 'Space Settings' %} {% endif %} {% if request.user == request.space.created_by or user.is_superuser %} - {% trans 'External Connectors' %} {% endif %} {% if user.is_superuser %} diff --git a/cookbook/tests/api/test_api_home_assistant_config.py b/cookbook/tests/api/test_api_home_assistant_config.py index a3de0f453..79f496bf6 100644 --- a/cookbook/tests/api/test_api_home_assistant_config.py +++ b/cookbook/tests/api/test_api_home_assistant_config.py @@ -5,21 +5,21 @@ from django.contrib import auth from django.urls import reverse from django_scopes import scopes_disabled -from cookbook.models import HomeAssistantConfig +from cookbook.models import ConnectorConfig -LIST_URL = 'api:homeassistantconfig-list' -DETAIL_URL = 'api:homeassistantconfig-detail' +LIST_URL = 'api:connectorconfig-list' +DETAIL_URL = 'api:connectorconfig-detail' @pytest.fixture() def obj_1(space_1, u1_s1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 1', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) @pytest.fixture def obj_2(space_1, u1_s1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 2', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) @@ -123,4 +123,4 @@ def test_delete(a1_s1, a1_s2, obj_1): assert r.status_code == 204 with scopes_disabled(): - assert HomeAssistantConfig.objects.count() == 0 + assert ConnectorConfig.objects.count() == 0 diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_connector_config.py similarity index 77% rename from cookbook/tests/edits/test_edits_home_assistant_config.py rename to cookbook/tests/edits/test_edits_connector_config.py index 0df507de8..aad00f8d2 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_connector_config.py @@ -3,16 +3,18 @@ from django.contrib import auth from django.contrib import messages from django.contrib.messages import get_messages from django.urls import reverse +from pytest_django.asserts import assertTemplateUsed -from cookbook.models import HomeAssistantConfig +from cookbook.models import ConnectorConfig -EDIT_VIEW_NAME = 'edit_home_assistant_config' +EDIT_VIEW_NAME = 'edit_connector_config' @pytest.fixture def home_assistant_config_obj(a1_s1, space_1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 1', + type=ConnectorConfig.HOMEASSISTANT, token='token', url='http://localhost:8123/api', todo_entity='todo.shopping_list', @@ -22,20 +24,22 @@ def home_assistant_config_obj(a1_s1, space_1): ) -def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConfig, a1_s1, a1_s2): +def test_edit_connector_config_homeassistant(home_assistant_config_obj: ConnectorConfig, a1_s1, a1_s2): new_token = '1234_token' r = a1_s1.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { 'name': home_assistant_config_obj.name, + 'type': home_assistant_config_obj.type, 'url': home_assistant_config_obj.url, - 'token': new_token, + 'update_token': new_token, 'todo_entity': home_assistant_config_obj.todo_entity, 'enabled': home_assistant_config_obj.enabled, } ) - assert r.status_code == 200 + assert r.status_code == 302 + r_messages = [m for m in get_messages(r.wsgi_request)] assert not any(m.level > messages.SUCCESS for m in r_messages) @@ -46,9 +50,10 @@ def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConf reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { 'name': home_assistant_config_obj.name, + 'type': home_assistant_config_obj.type, 'url': home_assistant_config_obj.url, 'todo_entity': home_assistant_config_obj.todo_entity, - 'token': new_token, + 'update_token': new_token, 'enabled': home_assistant_config_obj.enabled, } ) diff --git a/cookbook/urls.py b/cookbook/urls.py index e85cfbaf9..54d29ae4e 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,8 +12,7 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name, HomeAssistantConfig, ExampleConfig) -from .tables import ConnectorConfigTable + UserFile, UserSpace, get_model_name, ConnectorConfig) from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -52,7 +51,7 @@ router.register(r'shopping-list-recipe', api.ShoppingListRecipeViewSet) router.register(r'space', api.SpaceViewSet) router.register(r'step', api.StepViewSet) router.register(r'storage', api.StorageViewSet) -router.register(r'home-assistant-config', api.HomeAssistantConfigViewSet) +router.register(r'home-assistant-config', api.ConnectorConfigConfigViewSet) router.register(r'supermarket', api.SupermarketViewSet) router.register(r'supermarket-category', api.SupermarketCategoryViewSet) router.register(r'supermarket-category-relation', api.SupermarketCategoryRelationViewSet) @@ -116,7 +115,6 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), - path('list/connectors', ConnectorConfigTable.as_view(), name='list_connectors'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -169,7 +167,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, HomeAssistantConfig, ExampleConfig, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, ConnectorConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index d2cf65f86..bffc3ee4d 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -76,7 +76,7 @@ from cookbook.models import (Automation, BookmarkletImport, CookLog, CustomFilte ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig) + ViewLog, ConnectorConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -102,7 +102,7 @@ from cookbook.serializer import (AccessTokenSerializer, AutomationSerializer, SupermarketCategorySerializer, SupermarketSerializer, SyncLogSerializer, SyncSerializer, UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, - UserSerializer, UserSpaceSerializer, ViewLogSerializer, HomeAssistantConfigSerializer) + UserSerializer, UserSpaceSerializer, ViewLogSerializer, ConnectorConfigConfigSerializer) from cookbook.views.import_export import get_integration from recipes import settings from recipes.settings import FDC_API_KEY, DRF_THROTTLE_RECIPE_URL_IMPORT @@ -464,10 +464,9 @@ class StorageViewSet(viewsets.ModelViewSet): return self.queryset.filter(space=self.request.space) -class HomeAssistantConfigViewSet(viewsets.ModelViewSet): - # TODO handle delete protect error and adjust test - queryset = HomeAssistantConfig.objects - serializer_class = HomeAssistantConfigSerializer +class ConnectorConfigConfigViewSet(viewsets.ModelViewSet): + queryset = ConnectorConfig.objects + serializer_class = ConnectorConfigConfigSerializer permission_classes = [CustomIsAdmin & CustomTokenHasReadWriteScope] def get_queryset(self): diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index bbdc98ee0..f37d8a558 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig, ExampleConfig) + RecipeImport, Space, Storage, Sync, UserSpace, ConnectorConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -122,27 +122,15 @@ class StorageDelete(GroupRequiredMixin, DeleteView): return HttpResponseRedirect(reverse('list_storage')) -class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): +class ConnectorConfigDelete(GroupRequiredMixin, DeleteView): groups_required = ['admin'] template_name = "generic/delete_template.html" - model = HomeAssistantConfig - success_url = reverse_lazy('list_connectors') + model = ConnectorConfig + success_url = reverse_lazy('list_connector_config') def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") - return context - - -class ExampleConfigDelete(GroupRequiredMixin, DeleteView): - groups_required = ['admin'] - template_name = "generic/delete_template.html" - model = ExampleConfig - success_url = reverse_lazy('list_connectors') - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("Example Config Backend") + context['title'] = _("Connectors Config Backend") return context diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index 2f99216b3..356191ece 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -9,10 +9,10 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm, ExampleConfigForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, ConnectorConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig, ExampleConfig +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, ConnectorConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -128,11 +128,11 @@ def edit_storage(request, pk): ) -class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): +class ConnectorConfigUpdate(GroupRequiredMixin, UpdateView): groups_required = ['admin'] template_name = "generic/edit_template.html" - model = HomeAssistantConfig - form_class = HomeAssistantConfigForm + model = ConnectorConfig + form_class = ConnectorConfigForm def get_form_kwargs(self): kwargs = super().get_form_kwargs() @@ -143,33 +143,14 @@ class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): if form.cleaned_data['update_token'] != VALUE_NOT_CHANGED and form.cleaned_data['update_token'] != "": form.instance.token = form.cleaned_data['update_token'] messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) - return super(HomeAssistantConfigUpdate, self).form_valid(form) + return super(ConnectorConfigUpdate, self).form_valid(form) def get_success_url(self): - return reverse('edit_home_assistant_config', kwargs={'pk': self.object.pk}) + return reverse('edit_connector_config', kwargs={'pk': self.object.pk}) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistantConfig") - return context - - -class ExampleConfigUpdate(GroupRequiredMixin, UpdateView): - groups_required = ['admin'] - template_name = "generic/edit_template.html" - model = ExampleConfig - form_class = ExampleConfigForm - - def form_valid(self, form): - messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) - return super().form_valid(form) - - def get_success_url(self): - return reverse('edit_example_config', kwargs={'pk': self.object.pk}) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("ExampleConfig") + context['title'] = _("ConnectorConfig") return context diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index ed43c143a..53b1ae799 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, ConnectorConfig +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, ConnectorConfigTable @group_required('admin') @@ -65,6 +65,22 @@ def storage(request): ) +@group_required('admin') +def connector_config(request): + table = ConnectorConfigTable(ConnectorConfig.objects.filter(space=request.space).all()) + RequestConfig(request, paginate={'per_page': 25}).configure(table) + + return render( + request, + 'generic/list_template.html', + { + 'title': _("Connector Config Backend"), + 'table': table, + 'create_url': 'new_connector_config' + } + ) + + @group_required('admin') def invite_link(request): table = InviteLinkTable( diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 22bfbc84c..93ad99966 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -5,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm, ExampleConfigForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, ConnectorConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig, ExampleConfig +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, ConnectorConfig from recipes import settings @@ -70,21 +70,12 @@ class StorageCreate(GroupRequiredMixin, CreateView): return context -class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): +class ConnectorConfigCreate(GroupRequiredMixin, CreateView): groups_required = ['admin'] template_name = "generic/new_template.html" - model = HomeAssistantConfig - form_class = HomeAssistantConfigForm - success_url = reverse_lazy('list_home_assistant_config') - - def get_form_class(self): - form_class = super().get_form_class() - - if self.request.method == 'GET': - update_token_field = form_class.base_fields['update_token'] - update_token_field.required = True - - return form_class + model = ConnectorConfig + form_class = ConnectorConfigForm + success_url = reverse_lazy('list_connector_config') def form_valid(self, form): if self.request.space.demo or settings.HOSTED: @@ -96,35 +87,11 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): obj.created_by = self.request.user obj.space = self.request.space obj.save() - return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + return HttpResponseRedirect(reverse('edit_connector_config', kwargs={'pk': obj.pk})) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") - return context - - -class ExampleConfigCreate(GroupRequiredMixin, CreateView): - groups_required = ['admin'] - template_name = "generic/new_template.html" - model = ExampleConfig - form_class = ExampleConfigForm - success_url = reverse_lazy('list_connectors') - - def form_valid(self, form): - if self.request.space.demo or settings.HOSTED: - messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) - return redirect('index') - - obj = form.save(commit=False) - obj.created_by = self.request.user - obj.space = self.request.space - obj.save() - return HttpResponseRedirect(reverse('edit_example_config', kwargs={'pk': obj.pk})) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("Example Config Backend") + context['title'] = _("Connector Config Backend") return context From 578bb2af252fc3dfd63172fc00644b95e975cfdf Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 24 Jan 2024 08:57:24 +0100 Subject: [PATCH 21/54] better error handling during connector initilization --- cookbook/connectors/connector_manager.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 86a60b048..bb74aaf05 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -82,6 +82,7 @@ class ConnectorManager: item: Optional[Work] = worker_queue.get() except KeyboardInterrupt: break + if item is None: break @@ -96,11 +97,20 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = list( - filter( - lambda x: x is not None, - [ConnectorManager.get_connected_for_config(config) for config in space.connectorconfig_set.all() if config.enabled], - )) + connectors: List[Connector] = list() + for config in space.connectorconfig_set.all(): + config: ConnectorConfig = config + if not config.enabled: + continue + + try: + connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config) + except BaseException: + logging.exception(f"failed to initialize {config.name}") + continue + + connectors.append(connector) + _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: From ba169ba38db0ebef641f29df1f2696a31c00719f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 24 Jan 2024 08:59:31 +0100 Subject: [PATCH 22/54] better logging on skipped action --- cookbook/connectors/connector_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index bb74aaf05..64c552653 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -59,7 +59,7 @@ class ConnectorManager: try: self._queue.put_nowait(Work(instance, action_type)) except queue.Full: - logging.info("queue was full, so skipping %s", instance) + logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}") return def stop(self): From 502a606534fe27000524eaa223f4a3bff940fb0f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sun, 28 Jan 2024 22:59:51 +0100 Subject: [PATCH 23/54] Update the code based on feedback. set Default to enabled, add to documentation how to disable it. Add extra documentation --- .gitignore | 1 - cookbook/connectors/connector.py | 2 ++ cookbook/connectors/connector_manager.py | 28 +++++++++++++++++------- cookbook/signals.py | 3 +-- cookbook/views/api.py | 13 +++++------ cookbook/views/new.py | 6 ++++- docs/system/configuration.md | 10 +++++++++ recipes/settings.py | 3 ++- 8 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 4c8df7a7c..1c5e40a45 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,6 @@ docs/_build/ target/ \.idea/dataSources/ -.idea \.idea/dataSources\.xml \.idea/dataSources\.local\.xml diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 3647dc71e..27e9408db 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from cookbook.models import ShoppingListEntry, Space, ConnectorConfig +# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes. class Connector(ABC): @abstractmethod def __init__(self, config: ConnectorConfig): @@ -12,6 +13,7 @@ class Connector(ABC): async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: pass + # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823 @abstractmethod async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None: pass diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 64c552653..fe42c1a7a 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -7,18 +7,18 @@ from dataclasses import dataclass from enum import Enum from multiprocessing import JoinableQueue from types import UnionType -from typing import List, Any, Dict, Optional +from typing import List, Any, Dict, Optional, Type from django_scopes import scope +from django.conf import settings from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, ConnectorConfig +from cookbook.models import ShoppingListEntry, Space, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 -QUEUE_MAX_SIZE = 25 -REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan +REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry class ActionType(Enum): @@ -33,12 +33,20 @@ class Work: actionType: ActionType +# The way ConnectionManager works is as follows: +# 1. On init, it starts a worker & creates a queue for 'Work' +# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non blocking) to the queue. +# 3. The worker consumes said work from the queue. +# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id) +# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector) +# 4. Work is marked as consumed, and next entry of the queue is consumed. +# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector] class ConnectorManager: _queue: JoinableQueue _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): - self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) + self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() @@ -75,7 +83,7 @@ class ConnectorManager: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - _connectors: Dict[str, List[Connector]] = dict() + _connectors: Dict[int, List[Connector]] = dict() while True: try: @@ -90,7 +98,7 @@ class ConnectorManager: refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.name) + connectors: Optional[List[Connector]] = _connectors.get(space.id) if connectors is None or refresh_connector_cache: if connectors is not None: @@ -111,7 +119,7 @@ class ConnectorManager: connectors.append(connector) - _connectors[space.name] = connectors + _connectors[space.id] = connectors if len(connectors) == 0 or refresh_connector_cache: worker_queue.task_done() @@ -134,6 +142,9 @@ class ConnectorManager: async def close_connectors(connectors: List[Connector]): tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] + if len(tasks) == 0: + return + try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: @@ -161,6 +172,7 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE return try: + # Wait for all async tasks to finish, if one fails, the others still continue. await asyncio.gather(*tasks, return_exceptions=False) except BaseException: logging.exception("received an exception from one of the connectors") diff --git a/cookbook/signals.py b/cookbook/signals.py index ce957bcea..cd183c585 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -15,7 +15,6 @@ from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY from cookbook.models import (Food, MealPlan, PropertyType, Recipe, SearchFields, SearchPreference, Step, Unit, UserPreference) -from recipes.settings import ENABLE_EXTERNAL_CONNECTORS SQLITE = True if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql': @@ -165,7 +164,7 @@ def clear_property_type_cache(sender, instance=None, created=False, **kwargs): caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) -if ENABLE_EXTERNAL_CONNECTORS: +if not settings.DISABLE_EXTERNAL_CONNECTORS: handler = ConnectorManager() post_save.connect(handler, dispatch_uid="connector_manager") post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/views/api.py b/cookbook/views/api.py index bffc3ee4d..446f0b7f3 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -643,13 +643,12 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if pt.fdc_id: for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: - food_property_list.append( - Property( - property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, - space=self.request.space, - )) + food_property_list.append(Property( + property_type_id=pt.id, + property_amount=round(fn['amount'], 2), + import_food_id=food.id, + space=self.request.space, + )) Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 93ad99966..7e9732511 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -78,10 +78,14 @@ class ConnectorConfigCreate(GroupRequiredMixin, CreateView): success_url = reverse_lazy('list_connector_config') def form_valid(self, form): - if self.request.space.demo or settings.HOSTED: + if self.request.space.demo: messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') + if settings.DISABLE_EXTERNAL_CONNECTORS: + messages.add_message(self.request, messages.ERROR, _('This feature is not enabled by the server admin!')) + return redirect('index') + obj = form.save(commit=False) obj.token = form.cleaned_data['update_token'] obj.created_by = self.request.user diff --git a/docs/system/configuration.md b/docs/system/configuration.md index f6e64d242..dc64d41da 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -437,6 +437,16 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` +#### External Connectors + +`DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely (e.g. HomeAssistant). +`EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +```env +DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled +EXTERNAL_CONNECTORS_QUEUE_SIZE=25 +``` + ### Debugging/Development settings !!! warning diff --git a/recipes/settings.py b/recipes/settings.py index 319271eaf..6439f7274 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -555,6 +555,7 @@ DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', 'webmaster@localhost') ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv( 'ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix -ENABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('ENABLE_EXTERNAL_CONNECTORS', False))) +DISABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('DISABLE_EXTERNAL_CONNECTORS', False))) +EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 25)) mimetypes.add_type("text/javascript", ".js", True) From a8983a4b8a302a39d034a88fec6060bd28abb592 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 29 Jan 2024 09:56:40 +0100 Subject: [PATCH 24/54] undo workflow changes --- .github/workflows/ci.yml | 75 +++++++--------------------------------- 1 file changed, 13 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb72bc0c5..d5da9d824 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,80 +10,31 @@ jobs: max-parallel: 4 matrix: python-version: ['3.10'] - node-version: ['18'] steps: - uses: actions/checkout@v3 - - uses: awalsh128/cache-apt-pkgs-action@v1.3.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - packages: libsasl2-dev python3-dev libldap2-dev libssl-dev - version: 1.0 - - # Setup python & dependencies - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + python-version: '3.10' + # Build Vue frontend + - uses: actions/setup-node@v4 with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Cache StaticFiles - uses: actions/cache@v3 - id: django_cache - with: - path: | - ./cookbook/static - ./vue/webpack-stats.json - ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - - # Build Vue frontend & Dependencies - - name: Set up Node ${{ matrix.node-version }} - if: steps.django_cache.outputs.cache-hit != 'true' - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'yarn' - cache-dependency-path: ./vue/yarn.lock - + node-version: '18' - name: Install Vue dependencies - if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn install - - name: Build Vue dependencies - if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn build - - - name: Compile Django StatisFiles - if: steps.django_cache.outputs.cache-hit != 'true' + - name: Install Django dependencies run: | + sudo apt-get -y update + sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev + python -m pip install --upgrade pip + pip install -r requirements.txt python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse - - - uses: actions/cache/save@v3 - if: steps.django_cache.outputs.cache-hit != 'true' - with: - path: | - ./cookbook/static - ./vue/webpack-stats.json - ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - - name: Django Testing project - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - comment_mode: off - files: | - junit/test-results-${{ matrix.python-version }}.xml + run: | + pytest From 75c0ca8a9e7d0d86ff06fbddd1406a91abcf779a Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 2 Feb 2024 20:52:05 +0100 Subject: [PATCH 25/54] bunp migration --- .../{0209_connectorconfig.py => 0210_connectorconfig.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename cookbook/migrations/{0209_connectorconfig.py => 0210_connectorconfig.py} (93%) diff --git a/cookbook/migrations/0209_connectorconfig.py b/cookbook/migrations/0210_connectorconfig.py similarity index 93% rename from cookbook/migrations/0209_connectorconfig.py rename to cookbook/migrations/0210_connectorconfig.py index 4f672e81f..3d88816f2 100644 --- a/cookbook/migrations/0209_connectorconfig.py +++ b/cookbook/migrations/0210_connectorconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-17 21:49 +# Generated by Django 4.2.7 on 2024-02-02 19:51 import cookbook.models from django.conf import settings @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0208_space_app_name_userpreference_max_owned_spaces'), + ('cookbook', '0209_remove_space_use_plural'), ] operations = [ From 247907ef254b3ae4f0b5f9d4e139fcc40cb79918 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:26:33 +0100 Subject: [PATCH 26/54] move from signals to apps, add dedicated feature docs, add config toggle to menu item, undo unnecessary changes --- .github/workflows/ci.yml | 1 + cookbook/apps.py | 7 + cookbook/connectors/connector_manager.py | 16 +- cookbook/helper/context_processors.py | 1 + cookbook/signals.py | 9 +- cookbook/templates/base.html | 2 +- cookbook/templatetags/custom_tags.py | 3 + .../tests/other/test_connector_manager.py | 2 - cookbook/views/api.py | 199 ++++++++---------- docs/features/connectors.md | 43 ++++ docs/system/configuration.md | 10 - recipes/settings.py | 2 +- 12 files changed, 151 insertions(+), 144 deletions(-) create mode 100644 docs/features/connectors.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 202de7b2c..56148b3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,7 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - name: Django Testing project + timeout-minutes: 15 run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - name: Publish Test Results diff --git a/cookbook/apps.py b/cookbook/apps.py index e551319db..3bd8d8470 100644 --- a/cookbook/apps.py +++ b/cookbook/apps.py @@ -3,6 +3,7 @@ import traceback from django.apps import AppConfig from django.conf import settings from django.db import OperationalError, ProgrammingError +from django.db.models.signals import post_save, post_delete from django_scopes import scopes_disabled from recipes.settings import DEBUG @@ -14,6 +15,12 @@ class CookbookConfig(AppConfig): def ready(self): import cookbook.signals # noqa + if not settings.DISABLE_EXTERNAL_CONNECTORS: + from cookbook.connectors.connector_manager import ConnectorManager # Needs to be here to prevent loading race condition of oauth2 modules in models.py + handler = ConnectorManager() + post_save.connect(handler, dispatch_uid="connector_manager") + post_delete.connect(handler, dispatch_uid="connector_manager") + # if not settings.DISABLE_TREE_FIX_STARTUP: # # when starting up run fix_tree to: # # a) make sure that nodes are sorted when switching between sort modes diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index fe42c1a7a..90ad1bfd3 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -29,13 +29,13 @@ class ActionType(Enum): @dataclass class Work: - instance: REGISTERED_CLASSES + instance: REGISTERED_CLASSES | ConnectorConfig actionType: ActionType # The way ConnectionManager works is as follows: # 1. On init, it starts a worker & creates a queue for 'Work' -# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non blocking) to the queue. +# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue. # 3. The worker consumes said work from the queue. # 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id) # 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector) @@ -50,6 +50,7 @@ class ConnectorManager: self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() + # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"): return @@ -77,13 +78,15 @@ class ConnectorManager: @staticmethod def worker(worker_queue: JoinableQueue): + # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections from django.db import connections connections.close_all() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - _connectors: Dict[int, List[Connector]] = dict() + # + _connectors_cache: Dict[int, List[Connector]] = dict() while True: try: @@ -98,7 +101,7 @@ class ConnectorManager: refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.id) + connectors: Optional[List[Connector]] = _connectors_cache.get(space.id) if connectors is None or refresh_connector_cache: if connectors is not None: @@ -117,9 +120,10 @@ class ConnectorManager: logging.exception(f"failed to initialize {config.name}") continue - connectors.append(connector) + if connector is not None: + connectors.append(connector) - _connectors[space.id] = connectors + _connectors_cache[space.id] = connectors if len(connectors) == 0 or refresh_connector_cache: worker_queue.task_done() diff --git a/cookbook/helper/context_processors.py b/cookbook/helper/context_processors.py index 30e704c13..9beb8b9bd 100644 --- a/cookbook/helper/context_processors.py +++ b/cookbook/helper/context_processors.py @@ -11,4 +11,5 @@ def context_settings(request): 'PRIVACY_URL': settings.PRIVACY_URL, 'IMPRINT_URL': settings.IMPRINT_URL, 'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL, + 'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS, } diff --git a/cookbook/signals.py b/cookbook/signals.py index cd183c585..a93ffba1e 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -4,12 +4,11 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.search import SearchVector from django.core.cache import caches -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save from django.dispatch import receiver from django.utils import translation from django_scopes import scope, scopes_disabled -from cookbook.connectors.connector_manager import ConnectorManager from cookbook.helper.cache_helper import CacheHelper from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY @@ -162,9 +161,3 @@ def clear_unit_cache(sender, instance=None, created=False, **kwargs): def clear_property_type_cache(sender, instance=None, created=False, **kwargs): if instance: caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) - - -if not settings.DISABLE_EXTERNAL_CONNECTORS: - handler = ConnectorManager() - post_save.connect(handler, dispatch_uid="connector_manager") - post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index 4cbc1544b..f1d6c2f7b 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -335,7 +335,7 @@ {% trans 'Space Settings' %} {% endif %} - {% if request.user == request.space.created_by or user.is_superuser %} + {% if not DISABLE_EXTERNAL_CONNECTORS and request.user == request.space.created_by or not DISABLE_EXTERNAL_CONNECTORS and user.is_superuser %} {% trans 'External Connectors' %} {% endif %} diff --git a/cookbook/templatetags/custom_tags.py b/cookbook/templatetags/custom_tags.py index cdacd8e7e..459855108 100644 --- a/cookbook/templatetags/custom_tags.py +++ b/cookbook/templatetags/custom_tags.py @@ -112,6 +112,9 @@ def recipe_last(recipe, user): def page_help(page_name): help_pages = { 'edit_storage': 'https://docs.tandoor.dev/features/external_recipes/', + 'list_connector_config': 'https://docs.tandoor.dev/features/connectors/', + 'new_connector_config': 'https://docs.tandoor.dev/features/connectors/', + 'edit_connector_config': 'https://docs.tandoor.dev/features/connectors/', 'view_shopping': 'https://docs.tandoor.dev/features/shopping/', 'view_import': 'https://docs.tandoor.dev/features/import_export/', 'view_export': 'https://docs.tandoor.dev/features/import_export/', diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py index c6df778fb..b86100824 100644 --- a/cookbook/tests/other/test_connector_manager.py +++ b/cookbook/tests/other/test_connector_manager.py @@ -1,5 +1,3 @@ -from dataclasses import dataclass - import pytest from django.contrib import auth from mock.mock import Mock diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 5b44c2f81..1a91a33cc 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -181,10 +181,9 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) query = self.request.query_params.get('query', None) if self.request.user.is_authenticated: - fuzzy = self.request.user.searchpreference.lookup or any( - [self.model.__name__.lower() in x for x in - self.request.user.searchpreference.trigram.values_list( - 'field', flat=True)]) + fuzzy = self.request.user.searchpreference.lookup or any([self.model.__name__.lower() in x for x in + self.request.user.searchpreference.trigram.values_list( + 'field', flat=True)]) else: fuzzy = True @@ -204,10 +203,8 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): filter |= Q(name__unaccent__icontains=query) self.queryset = ( - self.queryset.annotate( - starts=Case( - When(name__istartswith=query, then=(Value(100))), - default=Value(0))) # put exact matches at the top of the result set + self.queryset.annotate(starts=Case(When(name__istartswith=query, then=(Value(100))), + default=Value(0))) # put exact matches at the top of the result set .filter(filter).order_by('-starts', Lower('name').asc()) ) @@ -329,9 +326,8 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True) self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) - return self.annotate_recipe( - queryset=self.queryset, request=self.request, serializer=self.serializer_class, - tree=True) + return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, + tree=True) @decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) @@ -575,9 +571,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): pass self.queryset = super().get_queryset() - shopping_status = ShoppingListEntry.objects.filter( - space=self.request.space, food=OuterRef('id'), - checked=False).values('id') + shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), + checked=False).values('id') # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users])) return self.queryset \ .annotate(shopping_status=Exists(shopping_status)) \ @@ -598,9 +593,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): shared_users = list(self.request.user.get_shopping_share()) shared_users.append(request.user) if request.data.get('_delete', False) == 'true': - ShoppingListEntry.objects.filter( - food=obj, checked=False, space=request.space, - created_by__in=shared_users).delete() + ShoppingListEntry.objects.filter(food=obj, checked=False, space=request.space, + created_by__in=shared_users).delete() content = {'msg': _(f'{obj.name} was removed from the shopping list.')} return Response(content, status=status.HTTP_204_NO_CONTENT) @@ -608,9 +602,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): unit = request.data.get('unit', None) content = {'msg': _(f'{obj.name} was added to the shopping list.')} - ShoppingListEntry.objects.create( - food=obj, amount=amount, unit=unit, space=request.space, - created_by=request.user) + ShoppingListEntry.objects.create(food=obj, amount=amount, unit=unit, space=request.space, + created_by=request.user) return Response(content, status=status.HTTP_204_NO_CONTENT) @decorators.action(detail=True, methods=['POST'], ) @@ -623,11 +616,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}') if response.status_code == 429: - return JsonResponse( - {'msg', - 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, - status=429, - json_dumps_params={'indent': 4}) + return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, + json_dumps_params={'indent': 4}) try: data = json.loads(response.content) @@ -883,14 +873,12 @@ class RecipePagination(PageNumberPagination): return super().paginate_queryset(queryset, request, view) def get_paginated_response(self, data): - return Response( - OrderedDict( - [ - ('count', self.page.paginator.count), - ('next', self.get_next_link()), - ('previous', self.get_previous_link()), - ('results', data), - ])) + return Response(OrderedDict([ + ('count', self.page.paginator.count), + ('next', self.get_next_link()), + ('previous', self.get_previous_link()), + ('results', data), + ])) class RecipeViewSet(viewsets.ModelViewSet): @@ -976,10 +964,9 @@ class RecipeViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if self.request.GET.get('debug', False): - return JsonResponse( - { - 'new': str(self.get_queryset().query), - }) + return JsonResponse({ + 'new': str(self.get_queryset().query), + }) return super().list(request, *args, **kwargs) def get_serializer_class(self): @@ -1149,10 +1136,8 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet): permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope] query_params = [ QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='int'), - QueryParam( - name='checked', description=_( - 'Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') - ), + QueryParam(name='checked', description=_('Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') + ), QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'), ] schema = QueryParamAutoSchema() @@ -1343,28 +1328,25 @@ class CustomAuthToken(ObtainAuthToken): throttle_classes = [AuthTokenThrottle] def post(self, request, *args, **kwargs): - serializer = self.serializer_class( - data=request.data, - context={'request': request}) + serializer = self.serializer_class(data=request.data, + context={'request': request}) serializer.is_valid(raise_exception=True) user = serializer.validated_data['user'] if token := AccessToken.objects.filter(user=user, expires__gt=timezone.now(), scope__contains='read').filter( scope__contains='write').first(): access_token = token else: - access_token = AccessToken.objects.create( - user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', - expires=(timezone.now() + timezone.timedelta(days=365 * 5)), - scope='read write app') - return Response( - { - 'id': access_token.id, - 'token': access_token.token, - 'scope': access_token.scope, - 'expires': access_token.expires, - 'user_id': access_token.user.pk, - 'test': user.pk - }) + access_token = AccessToken.objects.create(user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', + expires=(timezone.now() + timezone.timedelta(days=365 * 5)), + scope='read write app') + return Response({ + 'id': access_token.id, + 'token': access_token.token, + 'scope': access_token.scope, + 'expires': access_token.expires, + 'user_id': access_token.user.pk, + 'test': user.pk + }) class RecipeUrlImportView(APIView): @@ -1393,71 +1375,61 @@ class RecipeUrlImportView(APIView): url = serializer.validated_data.get('url', None) data = unquote(serializer.validated_data.get('data', None)) if not url and not data: - return Response( - { - 'error': True, - 'msg': _('Nothing to do.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Nothing to do.') + }, status=status.HTTP_400_BAD_REQUEST) elif url and not data: if re.match('^(https?://)?(www\\.youtube\\.com|youtu\\.be)/.+$', url): if validators.url(url, public=True): - return Response( - { - 'recipe_json': get_from_youtube_scraper(url, request), - 'recipe_images': [], - }, status=status.HTTP_200_OK) + return Response({ + 'recipe_json': get_from_youtube_scraper(url, request), + 'recipe_images': [], + }, status=status.HTTP_200_OK) if re.match( '^(.)*/view/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url): recipe_json = requests.get( - url.replace('/view/recipe/', '/api/recipe/').replace( - re.split('/view/recipe/[0-9]+', url)[1], - '') + '?share=' + + url.replace('/view/recipe/', '/api/recipe/').replace(re.split('/view/recipe/[0-9]+', url)[1], + '') + '?share=' + re.split('/view/recipe/[0-9]+', url)[1].replace('/', '')).json() recipe_json = clean_dict(recipe_json, 'id') serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request}) if serialized_recipe.is_valid(): recipe = serialized_recipe.save() if validators.url(recipe_json['image'], public=True): - recipe.image = File( - handle_image( - request, - File( - io.BytesIO(requests.get(recipe_json['image']).content), - name='image'), - filetype=pathlib.Path(recipe_json['image']).suffix), - name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') + recipe.image = File(handle_image(request, + File(io.BytesIO(requests.get(recipe_json['image']).content), + name='image'), + filetype=pathlib.Path(recipe_json['image']).suffix), + name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') recipe.save() - return Response( - { - 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) - }, status=status.HTTP_201_CREATED) + return Response({ + 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) + }, status=status.HTTP_201_CREATED) else: try: if validators.url(url, public=True): scrape = scrape_me(url_path=url, wild_mode=True) else: - return Response( - { - 'error': True, - 'msg': _('Invalid Url') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Invalid Url') + }, status=status.HTTP_400_BAD_REQUEST) except NoSchemaFoundInWildMode: pass except requests.exceptions.ConnectionError: - return Response( - { - 'error': True, - 'msg': _('Connection Refused.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Connection Refused.') + }, status=status.HTTP_400_BAD_REQUEST) except requests.exceptions.MissingSchema: - return Response( - { - 'error': True, - 'msg': _('Bad URL Schema.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Bad URL Schema.') + }, status=status.HTTP_400_BAD_REQUEST) else: try: data_json = json.loads(data) @@ -1473,18 +1445,16 @@ class RecipeUrlImportView(APIView): scrape = text_scraper(text=data, url=found_url) if scrape: - return Response( - { - 'recipe_json': helper.get_from_scraper(scrape, request), - 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), - }, status=status.HTTP_200_OK) + return Response({ + 'recipe_json': helper.get_from_scraper(scrape, request), + 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), + }, status=status.HTTP_200_OK) else: - return Response( - { - 'error': True, - 'msg': _('No usable data could be found.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('No usable data could be found.') + }, status=status.HTTP_400_BAD_REQUEST) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -1576,9 +1546,8 @@ def import_files(request): return Response({'import_id': il.pk}, status=status.HTTP_200_OK) except NotImplementedError: - return Response( - {'error': True, 'msg': _('Importing is not implemented for this provider')}, - status=status.HTTP_400_BAD_REQUEST) + return Response({'error': True, 'msg': _('Importing is not implemented for this provider')}, + status=status.HTTP_400_BAD_REQUEST) else: return Response({'error': True, 'msg': form.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1654,9 +1623,8 @@ def get_recipe_file(request, recipe_id): @group_required('user') def sync_all(request): if request.space.demo or settings.HOSTED: - messages.add_message( - request, messages.ERROR, - _('This feature is not yet available in the hosted version of tandoor!')) + messages.add_message(request, messages.ERROR, + _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') monitors = Sync.objects.filter(active=True).filter(space=request.user.userspace_set.filter(active=1).first().space) @@ -1695,9 +1663,8 @@ def share_link(request, pk): if request.space.allow_sharing and has_group_permission(request.user, ('user',)): recipe = get_object_or_404(Recipe, pk=pk, space=request.space) link = ShareLink.objects.create(recipe=recipe, created_by=request.user, space=request.space) - return JsonResponse( - {'pk': pk, 'share': link.uuid, - 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) + return JsonResponse({'pk': pk, 'share': link.uuid, + 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) else: return JsonResponse({'error': 'sharing_disabled'}, status=403) diff --git a/docs/features/connectors.md b/docs/features/connectors.md new file mode 100644 index 000000000..2687f7afb --- /dev/null +++ b/docs/features/connectors.md @@ -0,0 +1,43 @@ +!!! warning + Connectors are currently in a beta stage. + +## Connectors + +Connectors are a powerful add-on component to TandoorRecipes. +They allow for certain actions to be translated to api calls to external services. + +### General Config + +!!! danger + In order for this application to push data to external providers it needs to store authentication information. + Please use read only/separate accounts or app passwords wherever possible. + +- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. +- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +Example Config +```env +DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled +EXTERNAL_CONNECTORS_QUEUE_SIZE=100 +``` + +## Current Connectors + +### HomeAssistant + +The current HomeAssistant connector supports the following features: +1. Push newly created shopping list items. +2. Pushes all shopping list items if a recipe is added to the shopping list. +3. Removed todo's from HomeAssistant IF they are unchanged and are removed through TandoorRecipes. + +#### How to configure: + +Step 1: +1. Generate a HomeAssistant Long-Lived Access Tokens +![Profile Page](https://github.com/TandoorRecipes/recipes/assets/7824786/15ebeec5-5be3-48db-97d1-c698405db533) +2. Get/create a todo list entry you want to sync too. +![Todos Viewer](https://github.com/TandoorRecipes/recipes/assets/7824786/506c4c34-3d40-48ae-a80c-e50374c5c618) +3. Create a connector +![New Connector Config](https://github.com/TandoorRecipes/recipes/assets/7824786/7f14f181-1341-4cab-959b-a6bef79e2159) +4. ??? +5. Profit diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 0eb1b7839..8d93f9859 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -437,16 +437,6 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` -#### External Connectors - -`DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely (e.g. HomeAssistant). -`EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. - -```env -DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled -EXTERNAL_CONNECTORS_QUEUE_SIZE=25 -``` - ### Debugging/Development settings !!! warning diff --git a/recipes/settings.py b/recipes/settings.py index 009c6a5aa..f14d27012 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -557,6 +557,6 @@ ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv( 'ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix DISABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('DISABLE_EXTERNAL_CONNECTORS', False))) -EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 25)) +EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 100)) mimetypes.add_type("text/javascript", ".js", True) From 074244ee127e99a246469ae736b3f0e9e98c5daa Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:35:39 +0100 Subject: [PATCH 27/54] add timeout to async test --- cookbook/tests/other/test_connector_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py index b86100824..e6009f6b4 100644 --- a/cookbook/tests/other/test_connector_manager.py +++ b/cookbook/tests/other/test_connector_manager.py @@ -15,6 +15,7 @@ def obj_1(space_1, u1_s1): return e +@pytest.mark.timeout(10) @pytest.mark.asyncio async def test_run_connectors(space_1, u1_s1, obj_1) -> None: connector_mock = Mock(spec=Connector) From 0279013f72493b2e3b364dfdcf3992ae4af0b7e7 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:37:18 +0100 Subject: [PATCH 28/54] remove loop closing --- cookbook/connectors/connector_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 90ad1bfd3..8a35ba741 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -132,7 +132,6 @@ class ConnectorManager: loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) worker_queue.task_done() - loop.close() @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: From 0e945f4bd77c99de95291b27954493496c2ebe60 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:40:50 +0100 Subject: [PATCH 29/54] add startup & termination log to worker --- cookbook/connectors/connector_manager.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 8a35ba741..bc3293ffd 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -9,8 +9,8 @@ from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional, Type -from django_scopes import scope from django.conf import settings +from django_scopes import scope from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant @@ -47,7 +47,7 @@ class ConnectorManager: def __init__(self): self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) - self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) + self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() # Called by post save & post delete signals @@ -77,7 +77,7 @@ class ConnectorManager: self._worker.join() @staticmethod - def worker(worker_queue: JoinableQueue): + def worker(worker_id: int, worker_queue: JoinableQueue): # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections from django.db import connections connections.close_all() @@ -85,7 +85,9 @@ class ConnectorManager: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - # + logging.info(f"started ConnectionManager worker {worker_id}") + + # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior. _connectors_cache: Dict[int, List[Connector]] = dict() while True: @@ -132,6 +134,7 @@ class ConnectorManager: loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) worker_queue.task_done() + logging.info(f"terminating ConnectionManager worker {worker_id}") @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: From 408c2271a696a4a07850310b46d5b31246c51c94 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:43:13 +0100 Subject: [PATCH 30/54] reduce timeout, remove report generation --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56148b3b5..ca7ea7352 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,8 +78,8 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - name: Django Testing project - timeout-minutes: 15 - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml + timeout-minutes: 6 + run: pytest - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 From 2a6c13fc5ca0cde8a84b12fe9af0dd32790c9b4a Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:50:57 +0100 Subject: [PATCH 31/54] add finalizer to stop worker on terminate --- .github/workflows/ci.yml | 2 +- cookbook/connectors/connector_manager.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca7ea7352..6eb6195d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - name: Django Testing project timeout-minutes: 6 - run: pytest + run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index bc3293ffd..132bc56e4 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,6 +2,7 @@ import asyncio import logging import multiprocessing import queue +import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -49,6 +50,7 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() + self._finalizer = weakref.finalize(self, self.stop) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: From 16e8c1e8e337050a02180067060ee5476bca6f60 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:59:40 +0100 Subject: [PATCH 32/54] disable connector in tests --- cookbook/connectors/connector_manager.py | 5 +++-- pytest.ini | 4 +++- requirements.txt | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 132bc56e4..60a0d5e2b 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,7 +2,6 @@ import asyncio import logging import multiprocessing import queue -import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -50,7 +49,6 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() - self._finalizer = weakref.finalize(self, self.stop) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: @@ -138,6 +136,9 @@ class ConnectorManager: logging.info(f"terminating ConnectionManager worker {worker_id}") + asyncio.set_event_loop(None) + loop.close() + @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: match config.type: diff --git a/pytest.ini b/pytest.ini index d766a8425..3738a6c87 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] DJANGO_SETTINGS_MODULE = recipes.settings -python_files = tests.py test_*.py *_tests.py \ No newline at end of file +python_files = tests.py test_*.py *_tests.py +env = + DISABLE_EXTERNAL_CONNECTORS = 1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7f08008af..a9d1029b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ django-scopes==2.0.0 pytest==7.4.3 pytest-asyncio==0.23.3 pytest-django==4.6.0 +pytest-env==1.1.3 django-treebeard==4.7 django-cors-headers==4.2.0 django-storages==1.14.2 From 2bfc8b07171a83f696971b989b165de43bff9d38 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:11:46 +0100 Subject: [PATCH 33/54] format --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3738a6c87..bdfb66bcd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ DJANGO_SETTINGS_MODULE = recipes.settings python_files = tests.py test_*.py *_tests.py env = - DISABLE_EXTERNAL_CONNECTORS = 1 \ No newline at end of file + DISABLE_EXTERNAL_CONNECTORS=1 \ No newline at end of file From 65a7c82af9e733d622f1e86142de042528f3124b Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:17:23 +0100 Subject: [PATCH 34/54] terminate worker on finalize --- cookbook/connectors/connector_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 60a0d5e2b..f32e49c84 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,6 +2,7 @@ import asyncio import logging import multiprocessing import queue +import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -49,6 +50,7 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() + weakref.finalize(self, self._worker.terminate) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: From 962d61783991cea382855b794c8fe1779e966da7 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:37:37 +0100 Subject: [PATCH 35/54] switch to threading, f multiprocessing in python --- cookbook/connectors/connector_manager.py | 20 +++++--------------- pytest.ini | 2 -- requirements.txt | 1 - 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index f32e49c84..e247db305 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,12 +1,10 @@ import asyncio import logging -import multiprocessing import queue -import weakref +import threading from asyncio import Task from dataclasses import dataclass from enum import Enum -from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional, Type @@ -17,8 +15,6 @@ from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant from cookbook.models import ShoppingListEntry, Space, ConnectorConfig -multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 - REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry @@ -43,14 +39,13 @@ class Work: # 4. Work is marked as consumed, and next entry of the queue is consumed. # Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector] class ConnectorManager: - _queue: JoinableQueue + _queue: queue.Queue _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): - self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) - self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) + self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) + self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() - weakref.finalize(self, self._worker.terminate) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: @@ -75,15 +70,10 @@ class ConnectorManager: def stop(self): self._queue.join() - self._queue.close() self._worker.join() @staticmethod - def worker(worker_id: int, worker_queue: JoinableQueue): - # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections - from django.db import connections - connections.close_all() - + def worker(worker_id: int, worker_queue: queue.Queue): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) diff --git a/pytest.ini b/pytest.ini index bdfb66bcd..79e7a4e2a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,3 @@ [pytest] DJANGO_SETTINGS_MODULE = recipes.settings python_files = tests.py test_*.py *_tests.py -env = - DISABLE_EXTERNAL_CONNECTORS=1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a9d1029b2..7f08008af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,6 @@ django-scopes==2.0.0 pytest==7.4.3 pytest-asyncio==0.23.3 pytest-django==4.6.0 -pytest-env==1.1.3 django-treebeard==4.7 django-cors-headers==4.2.0 django-storages==1.14.2 From 1dc9244ac2a5b3ca0e8452d0f507a78a19722dfd Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:47:46 +0100 Subject: [PATCH 36/54] dont use timezone in test --- cookbook/tests/other/test_recipe_full_text_search.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cookbook/tests/other/test_recipe_full_text_search.py b/cookbook/tests/other/test_recipe_full_text_search.py index eed338875..b6b7a858f 100644 --- a/cookbook/tests/other/test_recipe_full_text_search.py +++ b/cookbook/tests/other/test_recipe_full_text_search.py @@ -1,8 +1,9 @@ import itertools import json -from datetime import timedelta +from datetime import timedelta, datetime import pytest +import pytz from django.conf import settings from django.contrib import auth from django.urls import reverse @@ -343,7 +344,7 @@ def test_search_date(found_recipe, recipes, param_type, result, u1_s1, u2_s1, sp Recipe.objects.filter(id=recipe.id).update( updated_at=recipe.created_at) - date = (timezone.now() - timedelta(days=15)).strftime("%Y-%m-%d") + date = (datetime.now() - timedelta(days=15)).strftime("%Y-%m-%d") param1 = f"?{param_type}={date}" param2 = f"?{param_type}=-{date}" r = json.loads(u1_s1.get(reverse(LIST_URL) + f'{param1}').content) From 20e1435abf71716d1500b8587e389b76ce3dcf11 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 8 Feb 2024 17:28:33 +0100 Subject: [PATCH 37/54] remove migration --- cookbook/migrations/0210_connectorconfig.py | 36 --------------------- 1 file changed, 36 deletions(-) delete mode 100644 cookbook/migrations/0210_connectorconfig.py diff --git a/cookbook/migrations/0210_connectorconfig.py b/cookbook/migrations/0210_connectorconfig.py deleted file mode 100644 index 3d88816f2..000000000 --- a/cookbook/migrations/0210_connectorconfig.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-02 19:51 - -import cookbook.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0209_remove_space_use_plural'), - ] - - operations = [ - migrations.CreateModel( - name='ConnectorConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('type', models.CharField(choices=[('HomeAssistant', 'HomeAssistant')], default='HomeAssistant', max_length=128)), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('url', models.URLField(blank=True, null=True)), - ('token', models.CharField(blank=True, max_length=512, null=True)), - ('todo_entity', models.CharField(blank=True, max_length=128, null=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - ] From 6ce95fb393ab75b30539e1459f656f38fcafe487 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 09:25:04 +0100 Subject: [PATCH 38/54] add reference to the feature configuration in configuration.md --- docs/system/configuration.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 8d93f9859..2d7b288c6 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -218,7 +218,18 @@ SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified ### Features Some features can be enabled/disabled on a server level because they might change the user experience significantly, -they might be unstable/beta or they have performance/security implications. +they might be unstable/beta, or they have performance/security implications. + +For more see configurations: +- [Authentication](https://docs.tandoor.dev/features/authentication/) +- [Automation](https://docs.tandoor.dev/features/automation/) +- [Connectors](https://docs.tandoor.dev/features/connectors/) +- [External Recipes](https://docs.tandoor.dev/features/external_recipes/) +- [Import/Export](https://docs.tandoor.dev/features/import_export/) +- [Shopping](https://docs.tandoor.dev/features/shopping/) +- [Telegram Bot](https://docs.tandoor.dev/features/telegram_bot/) +- [Templating](https://docs.tandoor.dev/features/templating/) + #### Captcha From 5e508944a32c889009a4e67c65f5b6649c9fda64 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:48:35 +0100 Subject: [PATCH 39/54] move env settings to configuration with backlink from connectors page --- docs/features/connectors.md | 11 +---------- docs/system/configuration.md | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/docs/features/connectors.md b/docs/features/connectors.md index 2687f7afb..1f9a1b9c2 100644 --- a/docs/features/connectors.md +++ b/docs/features/connectors.md @@ -6,20 +6,11 @@ Connectors are a powerful add-on component to TandoorRecipes. They allow for certain actions to be translated to api calls to external services. -### General Config - !!! danger In order for this application to push data to external providers it needs to store authentication information. Please use read only/separate accounts or app passwords wherever possible. -- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. -- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. - -Example Config -```env -DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled -EXTERNAL_CONNECTORS_QUEUE_SIZE=100 -``` +for the configuration please see [Configuration](https://docs.tandoor.dev/system/configuration/#connectors) ## Current Connectors diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 2d7b288c6..6d5f1ce67 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -218,18 +218,7 @@ SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified ### Features Some features can be enabled/disabled on a server level because they might change the user experience significantly, -they might be unstable/beta, or they have performance/security implications. - -For more see configurations: -- [Authentication](https://docs.tandoor.dev/features/authentication/) -- [Automation](https://docs.tandoor.dev/features/automation/) -- [Connectors](https://docs.tandoor.dev/features/connectors/) -- [External Recipes](https://docs.tandoor.dev/features/external_recipes/) -- [Import/Export](https://docs.tandoor.dev/features/import_export/) -- [Shopping](https://docs.tandoor.dev/features/shopping/) -- [Telegram Bot](https://docs.tandoor.dev/features/telegram_bot/) -- [Templating](https://docs.tandoor.dev/features/templating/) - +they might be unstable/beta or they have performance/security implications. #### Captcha @@ -448,6 +437,18 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` +#### Connectors + +- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. +- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +(External) Connectors are used to sync the status from Tandoor to other services. More info can be found [here](https://docs.tandoor.dev/features/connectors/). + +```env +DISABLE_EXTERNAL_CONNECTORS=0 # Default 0 (false), set to 1 (true) to disable connectors +EXTERNAL_CONNECTORS_QUEUE_SIZE=100 # Defaults to 100, set to any number >1 +``` + ### Debugging/Development settings !!! warning From 8f3effe194c6c58668e30f9c7962fec64e433a09 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:53:26 +0100 Subject: [PATCH 40/54] bump pytest-asyncio for pytest 8.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ae4ded832..5d3dbab04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ django-allauth==0.61.1 recipe-scrapers==14.52.0 django-scopes==2.0.0 pytest==8.0.0 -pytest-asyncio==0.23.3 +pytest-asyncio==0.23.5 pytest-django==4.8.0 django-treebeard==4.7 django-cors-headers==4.2.0 From 4e43a7a325fa6d2e47a314dcfa492143bdcad168 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:54:58 +0100 Subject: [PATCH 41/54] add connectors to mkdocs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index b7f50de22..644396fd5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,6 +42,7 @@ nav: - Shopping: features/shopping.md - Authentication: features/authentication.md - Automation: features/automation.md + - Connectors: features/connectors.md - Storages and Sync: features/external_recipes.md - Import/Export: features/import_export.md - System: From b1f418622f1bd37a20ddf4e2ecb12e6a1fde882b Mon Sep 17 00:00:00 2001 From: smilerz Date: Wed, 21 Feb 2024 08:25:06 -0600 Subject: [PATCH 42/54] change Recipe API to make keywords an optional field --- cookbook/serializer.py | 2 +- vue/src/utils/openapi/api.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 8446e3fb5..0d2a55659 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -901,7 +901,7 @@ class RecipeSerializer(RecipeBaseSerializer): nutrition = NutritionInformationSerializer(allow_null=True, required=False) properties = PropertySerializer(many=True, required=False) steps = StepSerializer(many=True) - keywords = KeywordSerializer(many=True) + keywords = KeywordSerializer(many=True, required=False) shared = UserSerializer(many=True, required=False) rating = CustomDecimalField(required=False, allow_null=True, read_only=True) last_cooked = serializers.DateTimeField(required=False, allow_null=True, read_only=True) diff --git a/vue/src/utils/openapi/api.ts b/vue/src/utils/openapi/api.ts index 45749f4d2..115892605 100644 --- a/vue/src/utils/openapi/api.ts +++ b/vue/src/utils/openapi/api.ts @@ -2200,7 +2200,7 @@ export interface Recipe { * @type {Array} * @memberof Recipe */ - keywords: Array; + keywords?: Array; /** * * @type {Array} From dccdb7cc2f674fd81ebee5ff3c4abca607b55d96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:50:58 +0000 Subject: [PATCH 43/54] Bump cryptography from 42.0.2 to 42.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e799af7ab..cbec64fb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ Django==4.2.10 -cryptography===42.0.2 +cryptography===42.0.4 django-annoying==0.10.6 django-autocomplete-light==3.9.7 django-cleanup==8.0.0 From 64ab768add86487cf87ab723a26d685ca6484f7b Mon Sep 17 00:00:00 2001 From: smilerz Date: Fri, 23 Feb 2024 07:23:10 -0600 Subject: [PATCH 44/54] paginate cooklog and viewlog tables on history view --- cookbook/views/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cookbook/views/views.py b/cookbook/views/views.py index 55474494d..22c957f14 100644 --- a/cookbook/views/views.py +++ b/cookbook/views/views.py @@ -289,8 +289,11 @@ def shopping_settings(request): @group_required('guest') def history(request): - view_log = ViewLogTable(ViewLog.objects.filter(created_by=request.user, space=request.space).order_by('-created_at').all()) - cook_log = CookLogTable(CookLog.objects.filter(created_by=request.user).order_by('-created_at').all()) + view_log = ViewLogTable(ViewLog.objects.filter(created_by=request.user, space=request.space).order_by('-created_at').all(), prefix="viewlog-") + view_log.paginate(page=request.GET.get("viewlog-page", 1), per_page=25) + + cook_log = CookLogTable(CookLog.objects.filter(created_by=request.user).order_by('-created_at').all(), prefix="cooklog-") + cook_log.paginate(page=request.GET.get("cooklog-page", 1), per_page=25) return render(request, 'history.html', {'view_log': view_log, 'cook_log': cook_log}) From 6de68707edc3c9a05f8e380a2c7eee3aa1050b68 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sat, 24 Feb 2024 11:16:09 +0100 Subject: [PATCH 45/54] fixed missing servings get parameter breaking view in some cases --- cookbook/views/views.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cookbook/views/views.py b/cookbook/views/views.py index 55474494d..6254e83e0 100644 --- a/cookbook/views/views.py +++ b/cookbook/views/views.py @@ -50,7 +50,7 @@ def index(request): # TODO need to deprecate def search(request): - if has_group_permission(request.user, ('guest', )): + if has_group_permission(request.user, ('guest',)): return render(request, 'search.html', {}) else: if request.user.is_authenticated: @@ -130,7 +130,7 @@ def recipe_view(request, pk, share=None): messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!')) return HttpResponseRedirect(reverse('account_login') + '?next=' + request.path) - if not (has_group_permission(request.user, ('guest', )) and recipe.space == request.space) and not share_link_valid(recipe, share): + if not (has_group_permission(request.user, ('guest',)) and recipe.space == request.space) and not share_link_valid(recipe, share): messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!')) return HttpResponseRedirect(reverse('index')) @@ -157,11 +157,11 @@ def recipe_view(request, pk, share=None): if not ViewLog.objects.filter(recipe=recipe, created_by=request.user, created_at__gt=(timezone.now() - timezone.timedelta(minutes=5)), space=request.space).exists(): ViewLog.objects.create(recipe=recipe, created_by=request.user, space=request.space) - if request.method == "GET": + servings = recipe.servings + if request.method == "GET" and 'servings' in request.GET: servings = request.GET.get("servings") return render(request, 'recipe_view.html', - {'recipe': recipe, 'comments': comments, 'comment_form': comment_form, 'share': share, 'servings': servings }) - + {'recipe': recipe, 'comments': comments, 'comment_form': comment_form, 'share': share, 'servings': servings}) @group_required('user') @@ -451,19 +451,19 @@ def web_manifest(request): manifest_info = { "name": - theme_values['app_name'], "short_name": - theme_values['app_name'], "description": - _("Manage recipes, shopping list, meal plans and more."), "icons": - icons, "start_url": - "./search", "background_color": - theme_values['nav_bg_color'], "display": - "standalone", "scope": - ".", "theme_color": - theme_values['nav_bg_color'], "shortcuts": - [{"name": _("Plan"), "short_name": _("Plan"), "description": _("View your meal Plan"), "url": - "./plan"}, {"name": _("Books"), "short_name": _("Books"), "description": _("View your cookbooks"), "url": "./books"}, - {"name": _("Shopping"), "short_name": _("Shopping"), "description": _("View your shopping lists"), "url": - "./list/shopping-list/"}], "share_target": {"action": "/data/import/url", "method": "GET", "params": {"title": "title", "url": "url", "text": "text"}} + theme_values['app_name'], "short_name": + theme_values['app_name'], "description": + _("Manage recipes, shopping list, meal plans and more."), "icons": + icons, "start_url": + "./search", "background_color": + theme_values['nav_bg_color'], "display": + "standalone", "scope": + ".", "theme_color": + theme_values['nav_bg_color'], "shortcuts": + [{"name": _("Plan"), "short_name": _("Plan"), "description": _("View your meal Plan"), "url": + "./plan"}, {"name": _("Books"), "short_name": _("Books"), "description": _("View your cookbooks"), "url": "./books"}, + {"name": _("Shopping"), "short_name": _("Shopping"), "description": _("View your shopping lists"), "url": + "./list/shopping-list/"}], "share_target": {"action": "/data/import/url", "method": "GET", "params": {"title": "title", "url": "url", "text": "text"}} } return JsonResponse(manifest_info, json_dumps_params={'indent': 4}) From ae70064c068b75eaa6eb7efd9fc46194b71f4371 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sat, 24 Feb 2024 13:05:23 +0100 Subject: [PATCH 46/54] renamed ingredients_markdown and removed ingredients_vue --- .gitignore | 2 ++ cookbook/serializer.py | 31 +++++++++++++--------------- vue/src/components/StepComponent.vue | 2 +- vue/src/utils/openapi/api.ts | 4 ++-- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 553403a5f..1657d4ca1 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ vue/webpack-stats.json cookbook/templates/sw.js .prettierignore vue/.yarn +vue3/.vite +vue3/node_modules diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 8446e3fb5..e72c776d2 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -337,7 +337,7 @@ class UserSpaceSerializer(WritableNestedModelSerializer): class Meta: model = UserSpace fields = ( - 'id', 'user', 'space', 'groups', 'active', 'internal_note', 'invite_link', 'created_at', 'updated_at',) + 'id', 'user', 'space', 'groups', 'active', 'internal_note', 'invite_link', 'created_at', 'updated_at',) read_only_fields = ('id', 'invite_link', 'created_at', 'updated_at', 'space') @@ -772,8 +772,7 @@ class IngredientSerializer(IngredientSimpleSerializer): class StepSerializer(WritableNestedModelSerializer, ExtendedRecipeMixin): ingredients = IngredientSerializer(many=True) - ingredients_markdown = serializers.SerializerMethodField('get_ingredients_markdown') - ingredients_vue = serializers.SerializerMethodField('get_ingredients_vue') + instructions_markdown = serializers.SerializerMethodField('get_instructions_markdown') file = UserFileViewSerializer(allow_null=True, required=False) step_recipe_data = serializers.SerializerMethodField('get_step_recipe_data') recipe_filter = 'steps' @@ -782,10 +781,7 @@ class StepSerializer(WritableNestedModelSerializer, ExtendedRecipeMixin): validated_data['space'] = self.context['request'].space return super().create(validated_data) - def get_ingredients_vue(self, obj): - return obj.get_instruction_render() - - def get_ingredients_markdown(self, obj): + def get_instructions_markdown(self, obj): return obj.get_instruction_render() def get_step_recipes(self, obj): @@ -800,8 +796,8 @@ class StepSerializer(WritableNestedModelSerializer, ExtendedRecipeMixin): class Meta: model = Step fields = ( - 'id', 'name', 'instruction', 'ingredients', 'ingredients_markdown', - 'ingredients_vue', 'time', 'order', 'show_as_header', 'file', 'step_recipe', + 'id', 'name', 'instruction', 'ingredients', 'instructions_markdown', + 'time', 'order', 'show_as_header', 'file', 'step_recipe', 'step_recipe_data', 'numrecipe', 'show_ingredients_table' ) @@ -846,7 +842,7 @@ class UnitConversionSerializer(WritableNestedModelSerializer, OpenDataModelMixin class Meta: model = UnitConversion fields = ( - 'id', 'name', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug') + 'id', 'name', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug') class NutritionInformationSerializer(serializers.ModelSerializer): @@ -873,6 +869,13 @@ class RecipeBaseSerializer(WritableNestedModelSerializer): return False +class CommentSerializer(serializers.ModelSerializer): + class Meta: + model = Comment + fields = '__all__' + read_only_fields = ['id', 'created_at', 'created_by', 'updated_at',] + + class RecipeOverviewSerializer(RecipeBaseSerializer): keywords = KeywordLabelSerializer(many=True) new = serializers.SerializerMethodField('is_recipe_new') @@ -950,12 +953,6 @@ class RecipeImportSerializer(SpacedModelSerializer): fields = '__all__' -class CommentSerializer(serializers.ModelSerializer): - class Meta: - model = Comment - fields = '__all__' - - class CustomFilterSerializer(SpacedModelSerializer, WritableNestedModelSerializer): shared = UserSerializer(many=True, required=False) @@ -1150,7 +1147,7 @@ class ShoppingListEntrySerializer(WritableNestedModelSerializer): 'recipe_mealplan', 'created_by', 'created_at', 'updated_at', 'completed_at', 'delay_until' ) - read_only_fields = ('id', 'created_by', 'created_at','updated_at',) + read_only_fields = ('id', 'created_by', 'created_at', 'updated_at',) class ShoppingListEntryBulkSerializer(serializers.Serializer): diff --git a/vue/src/components/StepComponent.vue b/vue/src/components/StepComponent.vue index 9585a3a76..3d25335e1 100644 --- a/vue/src/components/StepComponent.vue +++ b/vue/src/components/StepComponent.vue @@ -44,7 +44,7 @@
-
diff --git a/vue/src/utils/openapi/api.ts b/vue/src/utils/openapi/api.ts index 45749f4d2..5e642c7a6 100644 --- a/vue/src/utils/openapi/api.ts +++ b/vue/src/utils/openapi/api.ts @@ -2878,7 +2878,7 @@ export interface RecipeSteps { * @type {string} * @memberof RecipeSteps */ - ingredients_markdown?: string; + instructions_markdown?: string; /** * * @type {string} @@ -3731,7 +3731,7 @@ export interface Step { * @type {string} * @memberof Step */ - ingredients_markdown?: string; + instructions_markdown?: string; /** * * @type {string} From 59ecc40dc634e36b493611a900eeb974383b34b6 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sat, 24 Feb 2024 13:18:08 +0100 Subject: [PATCH 47/54] added comment field and a recipe filter to cook log --- ...log_comment_cooklog_updated_at_and_more.py | 33 +++++++++++++++++++ cookbook/models.py | 7 ++-- cookbook/serializer.py | 6 ++-- cookbook/views/api.py | 5 +++ 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 cookbook/migrations/0214_cooklog_comment_cooklog_updated_at_and_more.py diff --git a/cookbook/migrations/0214_cooklog_comment_cooklog_updated_at_and_more.py b/cookbook/migrations/0214_cooklog_comment_cooklog_updated_at_and_more.py new file mode 100644 index 000000000..fe28d646f --- /dev/null +++ b/cookbook/migrations/0214_cooklog_comment_cooklog_updated_at_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.7 on 2024-02-24 12:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0213_remove_property_property_unique_import_food_per_space_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='cooklog', + name='comment', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='cooklog', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='cooklog', + name='rating', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cooklog', + name='servings', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index fc1a62ad1..f9c16607d 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -1293,10 +1293,13 @@ class TelegramBot(models.Model, PermissionModelMixin): class CookLog(ExportModelOperationsMixin('cook_log'), models.Model, PermissionModelMixin): recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) + rating = models.IntegerField(null=True, blank=True) + servings = models.IntegerField(null=True, blank=True) + comment = models.TextField(null=True, blank=True) + created_by = models.ForeignKey(User, on_delete=models.CASCADE) created_at = models.DateTimeField(default=timezone.now) - rating = models.IntegerField(null=True) - servings = models.IntegerField(default=0) + updated_at = models.DateTimeField(auto_now=True) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') diff --git a/cookbook/serializer.py b/cookbook/serializer.py index e72c776d2..cf8154905 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -873,7 +873,7 @@ class CommentSerializer(serializers.ModelSerializer): class Meta: model = Comment fields = '__all__' - read_only_fields = ['id', 'created_at', 'created_by', 'updated_at',] + read_only_fields = ['id', 'created_at', 'created_by', 'updated_at', ] class RecipeOverviewSerializer(RecipeBaseSerializer): @@ -1200,6 +1200,8 @@ class ShareLinkSerializer(SpacedModelSerializer): class CookLogSerializer(serializers.ModelSerializer): + created_by = UserSerializer(read_only=True) + def create(self, validated_data): validated_data['created_by'] = self.context['request'].user validated_data['space'] = self.context['request'].space @@ -1207,7 +1209,7 @@ class CookLogSerializer(serializers.ModelSerializer): class Meta: model = CookLog - fields = ('id', 'recipe', 'servings', 'rating', 'created_by', 'created_at') + fields = ('id', 'recipe', 'servings', 'rating', 'comment', 'created_by', 'created_at', 'updated_at') read_only_fields = ('id', 'created_by') diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 710a8e375..50e1b77fb 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -1270,8 +1270,13 @@ class CookLogViewSet(viewsets.ModelViewSet): serializer_class = CookLogSerializer permission_classes = [CustomIsOwner & CustomTokenHasReadWriteScope] pagination_class = DefaultPagination + query_params = [ + QueryParam(name='recipe', description=_('Filter for entries with the given recipe'), qtype='integer'), + ] def get_queryset(self): + if self.request.query_params.get('recipe', None): + self.queryset = self.queryset.filter(recipe=self.request.query_params.get('recipe')) return self.queryset.filter(space=self.request.space) From 4a7eb91e674f5fcb895a67c3fdc064445a9acd38 Mon Sep 17 00:00:00 2001 From: Jinna Kiisuo Date: Mon, 26 Feb 2024 13:55:53 +0200 Subject: [PATCH 48/54] docs(authentication): Improve auth docs to match current allauth best practices and syntax --- docs/features/authentication.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/features/authentication.md b/docs/features/authentication.md index 84df9fca7..34a7970fb 100644 --- a/docs/features/authentication.md +++ b/docs/features/authentication.md @@ -3,7 +3,7 @@ methods of central account management and authentication. ## Allauth [Django Allauth](https://django-allauth.readthedocs.io/en/latest/index.html) is an awesome project that -allows you to use a [huge number](https://django-allauth.readthedocs.io/en/latest/providers.html) of different +allows you to use a [huge number](https://docs.allauth.org/en/latest/socialaccount/providers/index.html) of different authentication providers. They basically explain everything in their documentation, but the following is a short overview on how to get started. @@ -17,42 +17,50 @@ They basically explain everything in their documentation, but the following is a Choose a provider from the [list](https://docs.allauth.org/en/latest/socialaccount/providers/index.html) and install it using the environment variable `SOCIAL_PROVIDERS` as shown in the example below. -When at least one social provider is set up, the social login sign in buttons should appear on the login page. +When at least one social provider is set up, the social login sign in buttons should appear on the login page. The example below enables Nextcloud and the generic OpenID Connect providers. ```ini -SOCIAL_PROVIDERS=allauth.socialaccount.providers.github,allauth.socialaccount.providers.nextcloud +SOCIAL_PROVIDERS=allauth.socialaccount.providers.openid_connect,allauth.socialaccount.providers.nextcloud ``` !!! warning "Formatting" The exact formatting is important so make sure to follow the steps explained here! +### Configuration, via environment + Depending on your authentication provider you **might need** to configure it. This needs to be done through the settings system. To make the system flexible (allow multiple providers) and to not require another file to be mounted into the container the configuration ins done through a single environment variable. The downside of this approach is that the configuration needs to be put into a single line as environment files loaded by docker compose don't support multiple lines for a single variable. +The line data needs to either be in json or as Python dictionary syntax. + Take the example configuration from the allauth docs, fill in your settings and then inline the whole object (you can use a service like [www.freeformatter.com](https://www.freeformatter.com/json-formatter.html) for formatting). Assign it to the additional `SOCIALACCOUNT_PROVIDERS` variable. + +The example below is for a generic OIDC provider with PKCE enabled. Most values need to be customized for your specifics! ```ini -SOCIALACCOUNT_PROVIDERS={"nextcloud":{"SERVER":"https://nextcloud.example.org"}} +SOCIALACCOUNT_PROVIDERS = "{ 'openid_connect': { 'OAUTH_PKCE_ENABLED': True, 'APPS': [ { 'provider_id': 'oidc', 'name': 'My-IDM', 'client_id': 'my_client_id', 'secret': 'my_client_secret', 'settings': { 'server_url': 'https://idm.example.com/oidc/recipes' } } ] } }" ``` !!! success "Improvements ?" There are most likely ways to achieve the same goal but with a cleaner or simpler system. If you know such a way feel free to let me know. -After that, use your superuser account to configure your authentication backend. -Open the admin page and do the following +### Configuration, via Django Admin + +Instead of defining `SOCIALACCOUNT_PROVIDERS` in your environment, most configuration options can be done via the Admin interface. PKCE for `openid_connect` cannot currently be enabled this way. +Use your superuser account to configure your authentication backend by opening the admin page and do the following 1. Select `Sites` and edit the default site with the URL of your installation (or create a new). 2. Create a new `Social Application` with the required information as stated in the provider documentation of allauth. 3. Make sure to add your site to the list of available sites Now the provider is configured and you should be able to sign up and sign in using the provider. -Use the superuser account to grant permissions to the newly created users. +Use the superuser account to grant permissions to the newly created users, or enable default access via `SOCIAL_DEFAULT_ACCESS` & `SOCIAL_DEFAULT_GROUP`. !!! info "WIP" I do not have a ton of experience with using various single signon providers and also cannot test all of them. @@ -70,13 +78,7 @@ SOCIALACCOUNT_PROVIDERS='{"openid_connect":{"APPS":[{"provider_id":"keycloak","n ' ``` -1. Restart the service, login as superuser and open the `Admin` page. -2. Make sure that the correct `Domain Name` is defined at `Sites`. -3. Select `Social Application` and chose `Keycloak` from the provider list. -4. Provide an arbitrary name for your authentication provider, and enter the `Client-ID` and `Secret Key` values obtained from Keycloak earlier. -5. Make sure to add your `Site` to the list of available sites and save the new `Social Application`. - -You are now able to sign in using Keycloak. +You are now able to sign in using Keycloak after a restart of the service. ### Linking accounts To link an account to an already existing normal user go to the settings page of the user and link it. From dc5de6f0a29b4e9a808fc134a5b3a49efe8b6e76 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Mon, 26 Feb 2024 16:13:19 +0100 Subject: [PATCH 49/54] added migration and try/catch to apps.py --- cookbook/apps.py | 14 +++++--- cookbook/migrations/0215_connectorconfig.py | 36 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 cookbook/migrations/0215_connectorconfig.py diff --git a/cookbook/apps.py b/cookbook/apps.py index 3bd8d8470..63b8296e8 100644 --- a/cookbook/apps.py +++ b/cookbook/apps.py @@ -16,11 +16,15 @@ class CookbookConfig(AppConfig): import cookbook.signals # noqa if not settings.DISABLE_EXTERNAL_CONNECTORS: - from cookbook.connectors.connector_manager import ConnectorManager # Needs to be here to prevent loading race condition of oauth2 modules in models.py - handler = ConnectorManager() - post_save.connect(handler, dispatch_uid="connector_manager") - post_delete.connect(handler, dispatch_uid="connector_manager") - + try: + from cookbook.connectors.connector_manager import ConnectorManager # Needs to be here to prevent loading race condition of oauth2 modules in models.py + handler = ConnectorManager() + post_save.connect(handler, dispatch_uid="connector_manager") + post_delete.connect(handler, dispatch_uid="connector_manager") + except Exception as e: + traceback.print_exc() + print('Failed to initialize connectors') + pass # if not settings.DISABLE_TREE_FIX_STARTUP: # # when starting up run fix_tree to: # # a) make sure that nodes are sorted when switching between sort modes diff --git a/cookbook/migrations/0215_connectorconfig.py b/cookbook/migrations/0215_connectorconfig.py new file mode 100644 index 000000000..eaee40b2c --- /dev/null +++ b/cookbook/migrations/0215_connectorconfig.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.10 on 2024-02-26 14:41 + +import cookbook.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cookbook', '0214_cooklog_comment_cooklog_updated_at_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ConnectorConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('type', models.CharField(choices=[('HomeAssistant', 'HomeAssistant')], default='HomeAssistant', max_length=128)), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), + ('url', models.URLField(blank=True, null=True)), + ('token', models.CharField(blank=True, max_length=512, null=True)), + ('todo_entity', models.CharField(blank=True, max_length=128, null=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), + ] From c9e0f40e883e4b467eb12b9a4b5f792c2635d97b Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Mon, 26 Feb 2024 16:29:04 +0100 Subject: [PATCH 50/54] fixed mealmaster regex --- cookbook/integration/mealmaster.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbook/integration/mealmaster.py b/cookbook/integration/mealmaster.py index 9dce4fdd9..786103b48 100644 --- a/cookbook/integration/mealmaster.py +++ b/cookbook/integration/mealmaster.py @@ -22,7 +22,7 @@ class MealMaster(Integration): if 'Yield:' in line: servings_text = line.replace('Yield:', '').strip() else: - if re.match('\s{2,}([0-9])+', line): + if re.match(r'\s{2,}([0-9])+', line): ingredients.append(line.strip()) else: directions.append(line.strip()) diff --git a/requirements.txt b/requirements.txt index f27f16c91..c416671b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ gunicorn==20.1.0 lxml==4.9.3 Markdown==3.5.1 Pillow==10.2.0 -psycopg2-binary==2.9.5 +psycopg2-binary==2.9.9 python-dotenv==1.0.0 requests==2.31.0 six==1.16.0 From d71557dcec83d9f8e76d56042b5685f1449159c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20=C3=85teg?= Date: Mon, 26 Feb 2024 12:07:39 +0000 Subject: [PATCH 51/54] Translated using Weblate (Swedish) Currently translated at 100.0% (371 of 371 strings) Translation: Tandoor/Recipes Backend Translate-URL: http://translate.tandoor.dev/projects/tandoor/recipes-backend/sv/ --- cookbook/locale/sv/LC_MESSAGES/django.po | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cookbook/locale/sv/LC_MESSAGES/django.po b/cookbook/locale/sv/LC_MESSAGES/django.po index 5137fb7eb..6fec1dc0e 100644 --- a/cookbook/locale/sv/LC_MESSAGES/django.po +++ b/cookbook/locale/sv/LC_MESSAGES/django.po @@ -8,8 +8,8 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2021-04-11 15:09+0200\n" -"PO-Revision-Date: 2022-04-17 00:31+0000\n" -"Last-Translator: Oskar Stenberg <01ste02@gmail.com>\n" +"PO-Revision-Date: 2024-02-27 12:19+0000\n" +"Last-Translator: Lukas Åteg \n" "Language-Team: Swedish \n" "Language: sv\n" @@ -17,7 +17,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 4.10.1\n" +"X-Generator: Weblate 4.15\n" #: .\cookbook\filters.py:23 .\cookbook\templates\base.html:91 #: .\cookbook\templates\forms\edit_internal_recipe.html:219 @@ -1812,7 +1812,7 @@ msgstr "Google ld+json info" #: .\cookbook\templates\url_import.html:268 msgid "GitHub Issues" -msgstr "GitHub Issues" +msgstr "GitHub Problem" #: .\cookbook\templates\url_import.html:270 msgid "Recipe Markup Specification" @@ -1852,7 +1852,7 @@ msgstr "Kunde inte tolka korrekt..." msgid "Batch edit done. %(count)d recipe was updated." msgid_plural "Batch edit done. %(count)d Recipes where updated." msgstr[0] "Batchredigering klar. %(count)d recept uppdaterades." -msgstr[1] "Batchredigering klar. %(count)d recept uppdaterades." +msgstr[1] "Batchredigering klar. %(count)d recepten uppdaterades." #: .\cookbook\views\delete.py:72 msgid "Monitor" From 67b294c14138bcc173dd1df81454fde057106565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20=C3=85teg?= Date: Mon, 26 Feb 2024 10:40:44 +0000 Subject: [PATCH 52/54] Translated using Weblate (Swedish) Currently translated at 95.3% (536 of 562 strings) Translation: Tandoor/Recipes Frontend Translate-URL: http://translate.tandoor.dev/projects/tandoor/recipes-frontend/sv/ --- vue/src/locales/sv.json | 65 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/vue/src/locales/sv.json b/vue/src/locales/sv.json index e352a1e3b..131aad603 100644 --- a/vue/src/locales/sv.json +++ b/vue/src/locales/sv.json @@ -200,7 +200,7 @@ "Keyword_Alias": "Nyckelord alias", "Recipe_Book": "Receptbok", "Search Settings": "Sökinställningar", - "warning_feature_beta": "Den här funktionen är för närvarande i ett BETA-läge (testning). Vänligen förvänta dig buggar och eventuellt brytande ändringar i framtiden (möjligen att förlora funktionsrelaterad data) när du använder den här funktionen.", + "warning_feature_beta": "Den här funktionen är för närvarande i ett BETA-läge (testning). Förvänta dig buggar och eventuellt större ändringar i framtiden (möjligtvis framtida data kan gå förlorad) när du använder den här funktionen.", "success_deleting_resource": "En resurs har raderats!", "file_upload_disabled": "Filuppladdning är inte aktiverat för ditt utrymme.", "show_only_internal": "Visa endast interna recept", @@ -246,7 +246,7 @@ "CountMore": "...+{count} fler", "IgnoreThis": "Lägg aldrig till {mat} automatiskt i inköpslista", "DelayFor": "Fördröjning på {hours} timmar", - "ShowDelayed": "Visa fördröjda artiklar", + "ShowDelayed": "Visa fördröjda föremål", "Completed": "Avslutad", "OfflineAlert": "Du är offline, inköpslistan kanske inte synkroniseras.", "shopping_share": "Dela inköpslista", @@ -477,5 +477,64 @@ "Warning_Delete_Supermarket_Category": "Om du tar bort en mataffärskategori raderas också alla relationer till livsmedel. Är du säker?", "Disabled": "Inaktiverad", "Social_Authentication": "Social autentisering", - "Single": "Enstaka" + "Single": "Enstaka", + "Properties": "Egenskaper", + "err_importing_recipe": "Ett fel uppstod vid import av receptet!", + "recipe_property_info": "Du kan också lägga till egenskaper till maträtter för att beräkna dessa automatiskt baserat på ditt recept!", + "total": "totalt", + "CustomLogos": "Anpassade logotyper", + "Welcome": "Välkommen", + "Input": "Inmatning", + "Undo": "Ångra", + "NoMoreUndo": "Inga ändringar att ångra.", + "Delete_All": "Radera alla", + "Property": "Egendom", + "Property_Editor": "Egendom redigerare", + "Conversion": "Omvandling", + "created_by": "Skapad av", + "ShowRecentlyCompleted": "Visa nyligen genomförda föremål", + "ShoppingBackgroundSyncWarning": "Dålig uppkoppling, inväntar synkronisering...", + "show_step_ingredients": "Visa ingredienser för steget", + "hide_step_ingredients": "Dölj ingredienser för steget", + "Logo": "Logga", + "Show_Logo": "Visa logga", + "Show_Logo_Help": "Visa Tandoor eller hushålls-logga i navigationen.", + "Nav_Text_Mode": "Navigation Textläge", + "Nav_Text_Mode_Help": "Beter sig annorlunda för varje tema.", + "g": "gram [g] (metriskt, vikt)", + "kg": "kilogram [kg] (metriskt, vikt)", + "ounce": "ounce [oz] (vikt)", + "FDC_Search": "FDC Sök", + "property_type_fdc_hint": "Bara egendomstyper med ett FDC ID kan automatiskt hämta data från FDC databasen", + "Alignment": "Orientering", + "base_amount": "Basmängd", + "Datatype": "Datatyp", + "Number of Objects": "Antal objekt", + "StartDate": "Startdatum", + "EndDate": "Slutdatum", + "FDC_ID_help": "FDC databas ID", + "Data_Import_Info": "Förbättra din samling genom att importera en framtagen lista av livsmedel, enheter och mer för att förbättra din recept-samling.", + "Update_Existing_Data": "Uppdatera existerande data", + "Use_Metric": "Använd metriska enheter", + "Learn_More": "Läs mer", + "converted_unit": "Konverterad enhet", + "converted_amount": "Konverterad mängd", + "base_unit": "Basenhet", + "FDC_ID": "FDC ID", + "per_serving": "per servering", + "Properties_Food_Amount": "Egenskaper Livsmedel Mängd", + "Open_Data_Slug": "Öppen Data Slug", + "Open_Data_Import": "Öppen Data Import", + "Properties_Food_Unit": "Egenskaper Livsmedel Enhet", + "OrderInformation": "Objekt är sorterade från små till stora siffror.", + "show_step_ingredients_setting": "Visa ingredienser bredvid recept-steg", + "show_step_ingredients_setting_help": "Lägg till tabell med ingredienser bredvid recept-steg. Verkställs vid skapande. Kan skrivas över i redigering av receptvyn.", + "Space_Cosmetic_Settings": "Vissa kosmetiska inställningar kan ändras av hushålls-administratörer och skriver över klientinställningar för det hushållet.", + "show_ingredients_table": "Visa en tabell över ingredienserna bredvid stegets text", + "Enable": "Aktivera", + "CustomTheme": "Anpassat tema", + "CustomThemeHelp": "Skriv över nuvarande tema genom att ladda upp en anpassad CSS-fil.", + "CustomNavLogoHelp": "Ladda upp en bild att använda som meny-logga.", + "CustomImageHelp": "Ladda upp en bild som visas i överblicken.", + "CustomLogoHelp": "Ladda upp kvadratiska bilder i olika storlekar för att ändra logga i webbläsare." } From 961201412ea3ec65b7e03fcd297e1b97018cd740 Mon Sep 17 00:00:00 2001 From: Jerome Date: Mon, 26 Feb 2024 16:09:27 -0500 Subject: [PATCH 53/54] doc: Add installation instructions for ArchLinux --- docs/install/archlinux.md | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/install/archlinux.md diff --git a/docs/install/archlinux.md b/docs/install/archlinux.md new file mode 100644 index 000000000..9a6286b27 --- /dev/null +++ b/docs/install/archlinux.md @@ -0,0 +1,48 @@ +These are instructions for pacman based distributions, like ArchLinux. The package is available from the [AUR](https://aur.archlinux.org/packages/tandoor-recipes-git) or from [GitHub](https://github.com/jdecourval/tandoor-recipes-pkgbuild). + +## Features +- systemd integration. +- Provide configuration for Nginx. +- Use socket activation. +- Use a non-root user. +- Apply migrations automatically. + +## Installation +1. Clone the package, build and install with makepkg: +```shell +git clone https://aur.archlinux.org/tandoor-recipes-git.git +cd tandoor-recipes-git +makepkg -si +``` +or use your favourite AUR helper. + +2. Setup a PostgreSQL database and user, as explained here: https://docs.tandoor.dev/install/manual/#setup-postgresql + +3. Configure the service in `/etc/tandoor/tandoor.conf`. + +4. Reinstall the package, or follow [the official instructions](https://docs.tandoor.dev/install/manual/#initialize-the-application) to have tandoor creates its DB tables. + +5. Optionally configure a reverse proxy. A configuration for Nginx is provided, but you can Traefik, Apache, etc.. +Edit `/etc/nginx/sites-available/tandoor.conf`. You may want to use another `server_name`, or configure TLS. Then: +```shell +cd /etc/nginx/sites-enabled +ln -s ../sites-available/tandoor.conf +systemctl restart nginx +``` + +6. Enable the service +```shell +systemctl enable --now tandoor +``` + +## Upgrade +```shell +cd tandoor-recipes-git +git pull +makepkg -sif +``` +Or use your favourite AUR helper. +You shouldn't need to do anything else. This package applies migration automatically. If PostgreSQL has been updated to a new major version, you may need to [run pg_upgrade](https://wiki.archlinux.org/title/PostgreSQL#pg_upgrade). + +## Help +This package is non-official. Issues should be posted to https://github.com/jdecourval/tandoor-recipes-pkgbuild or https://aur.archlinux.org/packages/tandoor-recipes-git. \ No newline at end of file From d4ba2b9dd2d0f061679afcaed88a12b82f9f41c5 Mon Sep 17 00:00:00 2001 From: Jerome Date: Wed, 28 Feb 2024 10:39:48 -0500 Subject: [PATCH 54/54] Add archlinux doc to mkdocs config + add warning --- docs/install/archlinux.md | 5 ++++- mkdocs.yml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/install/archlinux.md b/docs/install/archlinux.md index 9a6286b27..2f17147f6 100644 --- a/docs/install/archlinux.md +++ b/docs/install/archlinux.md @@ -1,3 +1,6 @@ +!!! info "Community Contributed" + This guide was contributed by the community and is neither officially supported, nor updated or tested. + These are instructions for pacman based distributions, like ArchLinux. The package is available from the [AUR](https://aur.archlinux.org/packages/tandoor-recipes-git) or from [GitHub](https://github.com/jdecourval/tandoor-recipes-pkgbuild). ## Features @@ -45,4 +48,4 @@ Or use your favourite AUR helper. You shouldn't need to do anything else. This package applies migration automatically. If PostgreSQL has been updated to a new major version, you may need to [run pg_upgrade](https://wiki.archlinux.org/title/PostgreSQL#pg_upgrade). ## Help -This package is non-official. Issues should be posted to https://github.com/jdecourval/tandoor-recipes-pkgbuild or https://aur.archlinux.org/packages/tandoor-recipes-git. \ No newline at end of file +This package is non-official. Issues should be posted to https://github.com/jdecourval/tandoor-recipes-pkgbuild or https://aur.archlinux.org/packages/tandoor-recipes-git. diff --git a/mkdocs.yml b/mkdocs.yml index 644396fd5..68d2b70a9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,6 +35,7 @@ nav: - KubeSail or PiBox: install/kubesail.md - TrueNAS Portainer: install/truenas_portainer.md - WSL: install/wsl.md + - ArchLinux: install/archlinux.md - Manual: install/manual.md - Other setups: install/other.md - Features: