app.py

import gradio as gr
import openai
import requests
import os
import fileinput
from dotenv import load_dotenv
import io
import sys
import json
from PIL import Image
from stability_sdk import client
import stability_sdk.interfaces.gooseai.generation.generation_pb2 as generation

title="najimino AI recipe generator"
inputs_label="どんな料理か教えてくれれば，新しいレシピを考えます"
outputs_label="najimino AIが返信をします"
visual_outputs_label="料理のイメージ"
description="""
- ※入出力の文字数は最大1000文字程度までを目安に入力してください。回答に20秒くらいかかります．
"""

article = """
"""

load_dotenv()
openai.api_key = os.getenv('OPENAI_API_KEY')
os.environ['STABILITY_HOST'] = 'grpc.stability.ai:443'
stability_api = client.StabilityInference(
    key=os.getenv('STABILITY_KEY'), 
    verbose=True,
    engine="stable-diffusion-512-v2-1",
    # Available engines: stable-diffusion-v1 stable-diffusion-v1-5 stable-diffusion-512-v2-0 stable-diffusion-768-v2-0
    # stable-diffusion-512-v2-1 stable-diffusion-768-v2-1 stable-diffusion-xl-beta-v2-2-2 stable-inpainting-v1-0 stable-inpainting-512-v2-0
)
# MODEL = "gpt-4"
# MODEL = "gpt-3.5-turbo-16k"
MODEL = "gpt-3.5-turbo-0613"

def get_filetext(filename, cache={}):
    if filename in cache:
        # キャッシュに保存されている場合は、キャッシュからファイル内容を取得する
        return cache[filename]
    else:
        if not os.path.exists(filename):
            raise ValueError(f"ファイル '{filename}' が見つかりませんでした")
        with open(filename, "r") as f:
            text = f.read()
        # ファイル内容をキャッシュする
        cache[filename] = text
        return text

class StabilityAI:
    @classmethod
    def generate_image(cls, visualize_prompt):

        print("visualize_prompt:"+visualize_prompt)

        answers = stability_api.generate(
            prompt=visualize_prompt,
        )
        
        for resp in answers:
            for artifact in resp.artifacts:
                if artifact.finish_reason == generation.FILTER:
                    print("NSFW")
                if artifact.type == generation.ARTIFACT_IMAGE:
                    img = Image.open(io.BytesIO(artifact.binary))
                    return img

class OpenAI:
    
    @classmethod
    def chat_completion(cls, prompt, start_with=""):
        constraints = get_filetext(filename = "constraints.md")
        template = get_filetext(filename = "template.md")
        
        # ChatCompletion APIに渡すデータを定義する
        data = {
            "model": MODEL,
            "messages": [
            {"role": "system", "content": constraints}
                ,{"role": "system", "content": template}
                ,{"role": "assistant", "content": "Sure!"}
            ,{"role": "user", "content": prompt}
                ,{"role": "assistant", "content": start_with}
                    ],
                }
                
        # ChatCompletion APIを呼び出す
        response = requests.post(
            "https://api.openai.com/v1/chat/completions",
            headers={
                "Content-Type": "application/json",
                "Authorization": f"Bearer {openai.api_key}"
            },
            json=data
            )

        # ChatCompletion APIから返された結果を取得する
        result = response.json()
        print(result)
        
        content = result["choices"][0]["message"]["content"].strip()
        
        visualize_prompt = content.split("### Prompt for Visual Expression\n\n")[1]
        
        #print("split_content:"+split_content)

        #if len(split_content) > 1:
        #    visualize_prompt = split_content[1]
        #else:
        #    visualize_prompt = "vacant dish"

        #print("visualize_prompt:"+visualize_prompt)

        answers = stability_api.generate(
            prompt=visualize_prompt,
        )
        
    @classmethod
    def chat_completion_with_function(cls, prompt, messages, functions):
        print("prompt:"+prompt)
                
        response = openai.ChatCompletion.create(
                model=MODEL,
                messages=messages,
                functions=functions,
                function_call="auto"
            )

        # ChatCompletion APIから返された結果を取得する
        message = response.choices[0].message
        print(json.dumps(message, indent=2))
        
        return message
        
class NajiminoAI:

    def __init__(self, user_message):
        self.user_message = user_message

    def generate_recipe_prompt(self):
        template = get_filetext(filename="template.md")
        prompt = f"""
        {self.user_message}
        ---
        上記を元に、下記テンプレートを埋めてください。
        ---
        {template}
        """
        return prompt

    def format_recipe(self, lang, title, description, ingredients, instruction, comment_feelings_taste, explanation_to_blind_person, prompt_for_visual_expression_in_en):

        debug_message = f"""
            lang: {lang}
            title: {title}
            description: {description}
            ingredients: {ingredients}
            instruction: {instruction}
            comment_feelings_taste: {comment_feelings_taste}
            explanation_to_blind_person: {explanation_to_blind_person}
            prompt_for_visual_expression_in_en: {prompt_for_visual_expression_in_en}
        """
        
        print(debug_message)
        
        return debug_message
    
    @classmethod
    def generate(cls, user_message):
        
        najiminoai = NajiminoAI(user_message)
        
        return najiminoai.generate_recipe()
    
    def generate_recipe(self):
        
        user_message = self.user_message
        constraints = get_filetext(filename = "constraints.md")
        
        messages = [
            {"role": "system", "content": constraints}
            ,{"role": "user", "content": user_message}
        ]
        
        functions = [
            {
                "name": "format_recipe",
                "description": "どんな料理か教えてくれれば，新しいレシピを考えます",
                "parameters": {
                    "type": "object",
                    "default": {},
                    "title": "The Schema of new recipe",
                    "required": [
                        "lang",
                        "title",
                        "description",
                        "ingredients",
                        "instruction",
                        "comment_feelings_taste",
                        "explanation_to_blind_person",
                        "prompt_for_visual_expression_in_en"
                    ],
                    "properties": {
                        "lang": {
                            "type": "string",
                            "default": "ja",
                            "title": "Language Schema",
                            "examples": [
                                "ja"
                            ]
                        },
                        "title": {
                            "type": "string",
                            "default": "",
                            "title": "Title of New Recipe.",
                            "description": "Write your title of new recipe.",
                            "examples": [
                                "グルテンフリーサバのお好み焼き"
                            ]
                        },
                        "description": {
                            "type": "string",
                            "default": "",
                            "title": "Your New Recipe",
                            "description": "Write new recipe and brainstorm and work out the details of every aspect of the new recipe.",
                            "examples": [
                                "サバを使ったお好み焼きのレシピです。グルテンフリー仕様で作られているので、小麦粉を使わずに作ることができます。"
                            ]
                        },
                        "ingredients": {
                            "type": "string",
                            "default": "",
                            "title": "Your Ingredients",
                            "description": "Brainstorm the ingredients needed to cook a new recipe and come up with all the ingredients for the new recipe.",
                            "examples": [
                                "・サバのフィレ 200g\n・卵 2個\n・キャベツ 1/4個\n・もやし 50g\n・紅しょうが（刻んだもの） 2片\n・ネギ（小口切り） 適量\n・酒大さじ2\n・しょうゆ 大さじ2\n・だし汁 100ml\n・片栗粉 大さじ2\n・サラダ油 お好みで"
                            ]
                        },
                        "instruction": {
                            "type": "string",
                            "default": "",
                            "title": "Your Instruction",
                            "description": "Instruct them to cook a new recipe, brainstorming and working out every aspect and detail of the new recipe.",
                            "examples": [
                                "1. キャベツは粗みじん切りにし、もやしは石づきを取っておく。\n2. ボウルに卵を割り入れ、よく混ぜる。酒、しょうゆ、だし汁を加えてさらに混ぜる。\n3. キャベツ、もやし、紅しょうが、ネギを卵液に加え、さらに混ぜる。\n4. サバのフィレを食べやすい大きさに切り、片栗粉をまぶす。\n5. フライパンにサラダ油を熱し、サバを並べ入れる。両面をこんがり焼く。\n6. サバの上に卵液を流し入れ、蓋をして約5分ほど蒸し焼きにする。\n7. ひっくり返して、もう片面も蓋をして約5分ほど蒸し焼きにする。\n8. お好み焼きを取り出し、お好みでソースやマヨネーズをかけて完成です。"
                            ]
                        },
                        "comment_feelings_taste": {
                            "type": "string",
                            "default": "",
                            "title": "Your Comment and Feelings, taste of new recipe.",
                            "description": "Review commnet of new recipe and brainstorm every point of new recipe to fill the details.",
                            "examples": [
                                "このグルテンフリーサバのお好み焼きは、サバの旨味とキャベツの甘みが絶妙にマッチしています。表面はサクサク、中はもちもちとした食感で、一度食べたらやみつきになる美味しさです。"
                            ]
                        },
                        "explanation_to_blind_person": {
                            "type": "string",
                            "default": "",
                            "title": "Your Explanation to Blind Person",
                            "description": "Review commnet of new recipe here to explain to the blind people more concretely in detail. Please brainstorm every point of new recipe to fill the details.",
                            "examples": [
                                "このグルテンフリーサバのお好み焼きは、サバのフィレとキャベツを主な材料としています。まず、キャベツを細かく刻んでおきます。次に、卵をボウルに割り入れ、酒、しょうゆ、だし汁と一緒によく混ぜます。その後、刻んだキャベツ、もやし、紅しょうが、ネギを卵液に加えて混ぜます。サバのフィレは食べやすい大きさに切り、片栗粉をまぶしてから焼きます。フライパンにサラダ油を熱し、サバを焼きます。その上に卵液を流し入れ、蓋をして約5分蒸し焼きにします。ひっくり返して、もう片面も蓋をして約5分蒸し焼きにします。最後にお好み焼きを取り出し、ソースやマヨネーズをかけて完成です。"
                            ]
                        },
                        "prompt_for_visual_expression_in_en": {
                            "type": "string",
                            "default": "",
                            "title": "The Schema of prompt for visual expression in English",
                            "examples": [
                                "Imagine a delicious gluten-free okonomiyaki with mackerel. The okonomiyaki is crispy on the outside and chewy on the inside. It is topped with savory sauce and creamy mayonnaise, creating a mouthwatering visual. The dish is garnished with finely chopped green onions and red pickled ginger, adding a pop of color. The mackerel fillets are beautifully grilled and placed on top of the okonomiyaki, adding a touch of elegance. The dish is served on a traditional Japanese plate, completing the visual presentation."
                            ]
                        }
                    }
                }
            }
        ]
        
        message = OpenAI.chat_completion_with_function(prompt=user_message, messages=messages, functions=functions)
        
        # resultからfunction_callを取り出す
        # message = result["function_call"]
        print(json.dumps(message, indent=2))
        
        if message.get("function_call"):
            function_name = message["function_call"]["name"]
            
            args = json.loads(message["function_call"]["arguments"])
            
            lang=args.get("lang")
            title=args.get("title")
            description=args.get("description")
            ingredients=args.get("ingredients")
            instruction=args.get("instruction")
            comment_feelings_taste=args.get("comment_feelings_taste")
            explanation_to_blind_person=args.get("explanation_to_blind_person")
            prompt_for_visual_expression_in_en=args.get("prompt_for_visual_expression_in_en")
            
            function_response = self.format_recipe(
                lang=lang,
                title=title,
                description=description,
                ingredients=ingredients,
                instruction=instruction,
                comment_feelings_taste=comment_feelings_taste,
                explanation_to_blind_person=explanation_to_blind_person,
                prompt_for_visual_expression_in_en=prompt_for_visual_expression_in_en
            )
            
            answers = StabilityAI.generate_image(prompt_for_visual_expression_in_en)
            
        return [function_response, answers]

def main():
    iface = gr.Interface(fn=NajiminoAI.generate,
                        inputs=gr.Textbox(label=inputs_label),
                        outputs=[gr.Textbox(label=inputs_label), gr.Image(label=visual_outputs_label)],
                        title=title,
                        description=description,
                        article=article,
                        allow_flagging='never'
                        )

    iface.launch()

if __name__ == '__main__':
    function = ''
    if len(sys.argv) > 1:
        function = sys.argv[1]
    
    if function == 'generate':
        NajiminoAI.generate("グルテンフリーの香ばしいサバのお好み焼き")
        
    elif function == 'generate_image':
        answers = StabilityAI.generate_image("Imagine a delicious gluten-free okonomiyaki with mackerel. The okonomiyaki is crispy on the outside and chewy on the inside. It is topped with savory sauce and creamy mayonnaise, creating a mouthwatering visual. The dish is garnished with finely chopped green onions and red pickled ginger, adding a pop of color. The mackerel fillets are beautifully grilled and placed on top of the okonomiyaki, adding a touch of elegance. The dish is served on a traditional Japanese plate, completing the visual presentation.")
        print(answers)
        # <PIL.PngImagePlugin.PngImageFile image mode=RGB size=512x512 at 0x139900430>
        
        import PIL
        
        # answersが何のクラス確認する
        if type(answers) == PIL.PngImagePlugin.PngImageFile:
            #save image
            answers.save("image.png")

    else:
        main()