playing with AI image recognition

cookbook/serializer.py

@@ -63,7 +63,7 @@ class WritableNestedModelSerializer(WNMS):
                     pk_data = [x for x in data[f] if isinstance(x, int)]
                     # merge non-pk values with retrieved values
                     data[f] = [x for x in data[f] if not isinstance(x, int)] \
-                        + list(self.fields[f].child.Meta.model.objects.filter(id__in=pk_data).values(*required_fields))
+                              + list(self.fields[f].child.Meta.model.objects.filter(id__in=pk_data).values(*required_fields))
         return super().to_internal_value(data)
 
 
@@ -1526,3 +1526,7 @@ class RecipeFromSourceSerializer(serializers.Serializer):
     url = serializers.CharField(max_length=4096, required=False, allow_null=True, allow_blank=True)
     data = serializers.CharField(required=False, allow_null=True, allow_blank=True)
     bookmarklet = serializers.IntegerField(required=False, allow_null=True, )
+
+
+class ImportImageSerializer(serializers.Serializer):
+    image = serializers.ImageField()

cookbook/urls.py

@@ -125,6 +125,7 @@ urlpatterns = [
     path('api/reset-food-inheritance/', api.reset_food_inheritance, name='api_reset_food_inheritance'),
     path('api/switch-active-space/<int:space_id>/', api.switch_active_space, name='api_switch_active_space'),
     path('api/download-file/<int:file_id>/', api.download_file, name='api_download_file'),
+    path('api/image-to-recipe', api.ImageToRecipeView.as_view(), name='api_image_to_recipe'),
     path('telegram/setup/<int:pk>', telegram.setup_bot, name='telegram_setup'),
     path('telegram/remove/<int:pk>', telegram.remove_bot, name='telegram_remove'),
     path('telegram/hook/<slug:token>/', telegram.hook, name='telegram_hook'),

cookbook/views/api.py

@@ -12,6 +12,7 @@ from json import JSONDecodeError
 from urllib.parse import unquote
 from zipfile import ZipFile
 
+import PIL.Image
 import requests
 import validators
 from PIL import UnidentifiedImageError
@@ -33,6 +34,7 @@ from django.utils.translation import gettext as _
 from django_scopes import scopes_disabled
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view, OpenApiExample, inline_serializer
+from google import generativeai
 from icalendar import Calendar, Event
 from oauth2_provider.models import AccessToken
 from recipe_scrapers import scrape_html
@@ -83,12 +85,11 @@ from cookbook.serializer import (AccessTokenSerializer, AutomationSerializer, Au
                                  RecipeOverviewSerializer, RecipeSerializer, RecipeShoppingUpdateSerializer, RecipeSimpleSerializer, ShoppingListEntryBulkSerializer,
                                  ShoppingListEntrySerializer, ShoppingListRecipeSerializer, SpaceSerializer, StepSerializer, StorageSerializer,
                                  SupermarketCategoryRelationSerializer, SupermarketCategorySerializer, SupermarketSerializer, SyncLogSerializer, SyncSerializer,
-                                 UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, UserSerializer, UserSpaceSerializer, ViewLogSerializer
+                                 UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, UserSerializer, UserSpaceSerializer, ViewLogSerializer, ImportImageSerializer
                                  )
 from cookbook.views.import_export import get_integration
 from recipes import settings
-from recipes.settings import DRF_THROTTLE_RECIPE_URL_IMPORT, FDC_API_KEY
-
+from recipes.settings import DRF_THROTTLE_RECIPE_URL_IMPORT, FDC_API_KEY, GOOGLE_AI_API_KEY
 
 DateExample = OpenApiExample('Date Format', value='1972-12-05', request_only=True)
 BeforeDateExample = OpenApiExample('Before Date Format', value='-1972-12-05', request_only=True)
@@ -1141,14 +1142,14 @@ class UnitConversionViewSet(viewsets.ModelViewSet):
 
 
 @extend_schema_view(list=extend_schema(
-        parameters=[OpenApiParameter(
-            name='category',
-            description=_('Return the PropertyTypes matching the property category.  Repeat for multiple.'),
-            type=str,
-            many=True,
-            enum=[m[0] for m in PropertyType.CHOICES])
-        ]
-    ))
+    parameters=[OpenApiParameter(
+        name='category',
+        description=_('Return the PropertyTypes matching the property category.  Repeat for multiple.'),
+        type=str,
+        many=True,
+        enum=[m[0] for m in PropertyType.CHOICES])
+    ]
+))
 class PropertyTypeViewSet(viewsets.ModelViewSet):
     queryset = PropertyType.objects
     serializer_class = PropertyTypeSerializer
@@ -1361,7 +1362,7 @@ class AutomationViewSet(StandardFilterModelViewSet):
 
 # TODO explain what internal_note is for
 @extend_schema_view(list=extend_schema(parameters=[
-            OpenApiParameter(name='internal_note', description=_('I have no idea what internal_note is for.'), type=str)
+    OpenApiParameter(name='internal_note', description=_('I have no idea what internal_note is for.'), type=str)
 ]))
 class InviteLinkViewSet(StandardFilterModelViewSet):
     queryset = InviteLink.objects
@@ -1382,14 +1383,14 @@ class InviteLinkViewSet(StandardFilterModelViewSet):
 
 
 @extend_schema_view(list=extend_schema(
-        parameters=[OpenApiParameter(
-            name='type',
-            description=_('Return the CustomFilters matching the model type.  Repeat for multiple.'),
-            type=str,
-            many=True,
-            enum=[m[0] for m in CustomFilter.MODELS])
-        ]
-    ))
+    parameters=[OpenApiParameter(
+        name='type',
+        description=_('Return the CustomFilters matching the model type.  Repeat for multiple.'),
+        type=str,
+        many=True,
+        enum=[m[0] for m in CustomFilter.MODELS])
+    ]
+))
 class CustomFilterViewSet(StandardFilterModelViewSet):
     queryset = CustomFilter.objects
     serializer_class = CustomFilterSerializer
@@ -1537,6 +1538,31 @@ class RecipeUrlImportView(APIView):
             return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
 
+class ImageToRecipeView(APIView):
+    serializer_class = ImportImageSerializer
+    http_method_names = ['post', 'options']
+    #parser_classes = [MultiPartParser]
+    throttle_classes = [RecipeImportThrottle]
+    permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope]
+
+    def post(self, request, *args, **kwargs):
+        """
+
+        """
+        serializer = ImportImageSerializer(data=request.data, partial=True)
+        if serializer.is_valid():
+            generativeai.configure(api_key=GOOGLE_AI_API_KEY)
+
+            # model = generativeai.GenerativeModel('gemini-1.5-flash-latest')
+            # img = PIL.Image.open('')
+            # response = model.generate_content(["The image contains a recipe. Please return all data contained in the recipe formatted according to the schema.org specification for recipes", img], stream=True)
+            # response.resolve()
+            Response({'msg': 'SUCCESS'})
+        else:
+            Response({'msg': serializer.errors})
+        return Response({'test': 'test'})
+
+
 @extend_schema(
     request=None,
     responses=None,

recipes/settings.py

@@ -96,6 +96,7 @@ HCAPTCHA_SITEKEY = os.getenv('HCAPTCHA_SITEKEY', '')
 HCAPTCHA_SECRET = os.getenv('HCAPTCHA_SECRET', '')
 
 FDC_API_KEY = os.getenv('FDC_API_KEY', 'DEMO_KEY')
+GOOGLE_AI_API_KEY = os.getenv('GOOGLE_AI_API_KEY', '')
 
 SHARING_ABUSE = bool(int(os.getenv('SHARING_ABUSE', False)))
 SHARING_LIMIT = int(os.getenv('SHARING_LIMIT', 0))

requirements.txt

@@ -47,6 +47,7 @@ validators==0.20.0
 pytube==15.0.0
 aiohttp==3.9.4
 django-vite==3.0.3
+google-generativeai==0.5.3
 
 # Development
 pytest==8.0.0

vue3/src/apps/tandoor/Tandoor.vue

@@ -22,6 +22,7 @@
             <v-list-item prepend-icon="fas fa-calendar-alt" title="Mealplan" :to="{ name: 'view_mealplan', params: {} }"></v-list-item>
             <v-list-item prepend-icon="fas fa-shopping-cart" title="Shopping" :to="{ name: 'view_shopping', params: {} }"></v-list-item>
             <v-list-item prepend-icon="fas fa-bars" title="More" :to="{ name: 'view_books', params: {} }"></v-list-item>
+            <v-list-item prepend-icon="fas fa-bars" title="Test" :to="{ name: 'view_test', params: {} }"></v-list-item>
             <!-- TODO link -->
         </v-navigation-drawer>
 

vue3/src/apps/tandoor/main.ts

@@ -14,9 +14,11 @@ import luxonPlugin from "@/plugins/luxonPlugin";
 import RecipeEditPage from "@/pages/RecipeEditPage.vue";
 import MealPlanPage from "@/pages/MealPlanPage.vue";
 import SearchPage from "@/pages/SearchPage.vue";
+import TestPage from "@/pages/TestPage.vue";
 
 const routes = [
     {path: '/', component: StartPage, name: 'view_home'},
+    {path: '/test', component: TestPage, name: 'view_test'},
     {path: '/search', component: SearchPage, name: 'view_search'},
     {path: '/shopping', component: ShoppingListPage, name: 'view_shopping'},
     {path: '/mealplan', component: MealPlanPage, name: 'view_mealplan'},

vue3/src/openapi/.openapi-generator/FILES

@@ -22,6 +22,7 @@ models/FoodInheritField.ts
 models/FoodShoppingUpdate.ts
 models/FoodSimple.ts
 models/Group.ts
+models/ImportImage.ts
 models/ImportLog.ts
 models/Ingredient.ts
 models/IngredientString.ts

vue3/src/openapi/apis/ApiApi.ts

@@ -27,6 +27,7 @@ import type {
   FoodInheritField,
   FoodShoppingUpdate,
   Group,
+  ImportImage,
   ImportLog,
   Ingredient,
   IngredientString,
@@ -167,6 +168,8 @@ import {
     FoodShoppingUpdateToJSON,
     GroupFromJSON,
     GroupToJSON,
+    ImportImageFromJSON,
+    ImportImageToJSON,
     ImportLogFromJSON,
     ImportLogToJSON,
     IngredientFromJSON,
@@ -662,6 +665,10 @@ export interface ApiGroupRetrieveRequest {
     id: number;
 }
 
+export interface ApiImageToRecipeCreateRequest {
+    image: string;
+}
+
 export interface ApiImportLogCreateRequest {
     importLog: ImportLog;
 }
@@ -3962,6 +3969,60 @@ export class ApiApi extends runtime.BaseAPI {
         return await response.value();
     }
 
+    /**
+     */
+    async apiImageToRecipeCreateRaw(requestParameters: ApiImageToRecipeCreateRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<runtime.ApiResponse<ImportImage>> {
+        if (requestParameters['image'] == null) {
+            throw new runtime.RequiredError(
+                'image',
+                'Required parameter "image" was null or undefined when calling apiImageToRecipeCreate().'
+            );
+        }
+
+        const queryParameters: any = {};
+
+        const headerParameters: runtime.HTTPHeaders = {};
+
+        if (this.configuration && this.configuration.apiKey) {
+            headerParameters["Authorization"] = await this.configuration.apiKey("Authorization"); // ApiKeyAuth authentication
+        }
+
+        const consumes: runtime.Consume[] = [
+            { contentType: 'multipart/form-data' },
+        ];
+        // @ts-ignore: canConsumeForm may be unused
+        const canConsumeForm = runtime.canConsumeForm(consumes);
+
+        let formParams: { append(param: string, value: any): any };
+        let useForm = false;
+        if (useForm) {
+            formParams = new FormData();
+        } else {
+            formParams = new URLSearchParams();
+        }
+
+        if (requestParameters['image'] != null) {
+            formParams.append('image', requestParameters['image'] as any);
+        }
+
+        const response = await this.request({
+            path: `/api/image-to-recipe`,
+            method: 'POST',
+            headers: headerParameters,
+            query: queryParameters,
+            body: formParams,
+        }, initOverrides);
+
+        return new runtime.JSONApiResponse(response, (jsonValue) => ImportImageFromJSON(jsonValue));
+    }
+
+    /**
+     */
+    async apiImageToRecipeCreate(requestParameters: ApiImageToRecipeCreateRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise<ImportImage> {
+        const response = await this.apiImageToRecipeCreateRaw(requestParameters, initOverrides);
+        return await response.value();
+    }
+
     /**
      * function to handle files passed by application importer
      */

vue3/src/openapi/models/ImportImage.ts

@@ -0,0 +1,61 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * Tandoor
+ * Tandoor API Docs
+ *
+ * The version of the OpenAPI document: 0.0.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+import { mapValues } from '../runtime';
+/**
+ * 
+ * @export
+ * @interface ImportImage
+ */
+export interface ImportImage {
+    /**
+     * 
+     * @type {string}
+     * @memberof ImportImage
+     */
+    image: string;
+}
+
+/**
+ * Check if a given object implements the ImportImage interface.
+ */
+export function instanceOfImportImage(value: object): boolean {
+    if (!('image' in value)) return false;
+    return true;
+}
+
+export function ImportImageFromJSON(json: any): ImportImage {
+    return ImportImageFromJSONTyped(json, false);
+}
+
+export function ImportImageFromJSONTyped(json: any, ignoreDiscriminator: boolean): ImportImage {
+    if (json == null) {
+        return json;
+    }
+    return {
+        
+        'image': json['image'],
+    };
+}
+
+export function ImportImageToJSON(value?: ImportImage | null): any {
+    if (value == null) {
+        return value;
+    }
+    return {
+        
+        'image': value['image'],
+    };
+}
+

vue3/src/openapi/models/index.ts

@@ -19,6 +19,7 @@ export * from './FoodInheritField';
 export * from './FoodShoppingUpdate';
 export * from './FoodSimple';
 export * from './Group';
+export * from './ImportImage';
 export * from './ImportLog';
 export * from './Ingredient';
 export * from './IngredientString';

vue3/src/pages/TestPage.vue

@@ -0,0 +1,41 @@
+<template>
+    <v-file-input label="File input" v-model="image"></v-file-input>
+
+
+    <v-btn @click="imageToRecipe()">Upload</v-btn>
+
+
+    <v-textarea v-model="response"></v-textarea>
+</template>
+
+<script setup lang="ts">
+
+
+import {ApiApi} from "@/openapi";
+import {ref} from "vue";
+
+const image = ref(File)
+const response = ref('')
+
+function imageToRecipe() {
+    const api = new ApiApi()
+
+    const reader = new FileReader()
+    reader.readAsDataURL(image.value)
+
+    api.apiImageToRecipeCreate({image: image.value}).then(r => {
+        console.log(r)
+        response.value = r
+    }).catch(err => {
+        console.log(err)
+        response.value = err
+    })
+}
+
+
+</script>
+
+
+<style scoped>
+
+</style>