app.py

import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM

model = None


def greet(name):
    return "Hello " + name + "!!"


def main():

    print(f"Loading model...")
    model_path = "meta-llama/Meta-Llama-3-8B"
    tokenizer = AutoTokenizer.from_pretrained(model_path)
    model = AutoModelForCausalLM.from_pretrained(
        model_path,
        torch_dtype=torch.bfloat16,
        device_map="auto",
    )

    demo = gr.Interface(fn=greet, inputs="text", outputs="text")
    demo.launch()


if __name__ == "__main__":
    main()