Updated run.py to support Huggingface backend

ChatAtomicFlow.yaml

@@ -23,8 +23,7 @@ backend:
 
 
   top_p: 0.2
-  frequency_penalty: 0
-  presence_penalty: 0
+
   stream: True
 
 

demo.yaml

@@ -15,6 +15,7 @@ backend:
   model_name: 
     openai: "gpt-3.5-turbo"
     azure: "azure/gpt-4"
+    huggingface: "huggingface/meta-llama/Meta-Llama-3-70B-Instruct"
   
   # ~~~ generation_parameters ~~
   n: 1
@@ -24,6 +25,7 @@ backend:
   top_p: 0.2
   frequency_penalty: 0
   presence_penalty: 0
+  stream: False
 
 n_api_retries: 6
 wait_time_between_retries: 20

run.py

@@ -20,6 +20,11 @@ CACHING_PARAMETERS.do_caching = False  # Set to True in order to disable caching
 # clear_cache() # Uncomment this line to clear the cache
 
 logging.set_verbosity_debug()
+# Comment out if using huggingface backend
+# The set_verbose() method is used for debugging and logging the outputs and drop_params() method automatically drops the parameters which are not set my litellm (since the response format of OpenAI and huggingface is different we require this to translate Huggingface to OpenAI response format).
+# import litellm
+# litellm.set_verbose=True
+# litellm.drop_params=True
 
 dependencies = [
     {"url": "aiflows/ChatFlowModule", "revision": os.getcwd()},
@@ -44,6 +49,10 @@ if __name__ == "__main__":
     # OpenAI backend
     api_information = [ApiInfo(backend_used="openai",
                               api_key = os.getenv("OPENAI_API_KEY"))]
+    #Huggingface backend
+    # Here the API_BASE is the api link to your hosted server. By this way, you can run any flows on your own server.
+    # api_information = [ApiInfo(backend_used="huggingface",
+    #                           api_key = os.getenv("HUGGINGFACE_API_KEY"), api_base="http://0.0.0.0:5000/v1/completions")]
     # # Azure backend
     # api_information = ApiInfo(backend_used = "azure",
     #                           api_base = os.getenv("AZURE_API_BASE"),