Added user messaging when the endpoint is down.

src/architectures.py

@@ -279,13 +279,29 @@ class InputRequestScreener(ArchitectureComponent):
 class OutputResponseScreener(ArchitectureComponent):
     description = "Screens outputs for offensive responses."
 
+    def __init__(self):
+        self.api_token = hf_api_token()
+        self.endpoint_url = "https://yl89ru8gdr1wkbej.eu-west-1.aws.endpoints.huggingface.cloud"
+
     def process_request(self, request: ArchitectureRequest) -> None:
         system_prompt = "You are screening for offensive content. In a single word (yes or no), is the response offensive?"
-        llm = HFLlamaChatModel.for_model('meta-llama/Llama-2-7b-chat-hf')
-        if llm is None:
-            raise ValueError(f'Screener model "meta-llama/Llama-2-7b-chat-hf" not set up')
-        response = llm(request.response, system_prompt=system_prompt)
-        if response[0:2].lower() != 'no':  # Lean cautious even if the model fails to return yes/no
+        headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {self.api_token}",
+            "Content-Type": "application/json"
+        }
+        query_input = f"[INST] <<SYS>> {system_prompt} <<SYS>> {request.response} [/INST] "
+        payload = {
+            "inputs": query_input,
+            "parameters": {
+                "temperature": 0.1,
+                "max_new_tokens": 10
+            }
+        }
+        llm_response = requests.post(self.endpoint_url, headers=headers, json=payload)
+        generated_text = json.loads(llm_response.text)[0]['generated_text'].strip()
+        print(f"Response screener got LLM response: {generated_text}")
+        if generated_text[0:2].lower() != 'no':  # Lean cautious even if the model fails to return yes/no
             request.response = "Sorry - I cannot answer this question. Please try and rephrase it."
             request.early_exit = True
 
@@ -383,7 +399,7 @@ class HFInferenceEndpoint(ArchitectureComponent):
         """
         desc = f"Model: {self.model_name};  "
         desc += f"Endpoint: {self.endpoint_url};  "
-        desc += f"Max tokens: {self.max_tokens};  "
+        desc += f"Max tokens: {self.max_new_tokens};  "
         desc += f"Temperature: {self.temperature};  "
         desc += f"System prompt: {self.system_prompt}"
         return desc
@@ -407,9 +423,14 @@ class HFInferenceEndpoint(ArchitectureComponent):
                 "max_new_tokens": self.max_new_tokens
             }
         }
-        response = requests.post(self.endpoint_url, headers=headers, json=payload)
-        generated_text = json.loads(response.text)[0]['generated_text'].strip()
-        request.response = generated_text
+        llm_response = requests.post(self.endpoint_url, headers=headers, json=payload)
+        if llm_response.status_code == 200:
+            generated_text = llm_response.json()[0]['generated_text'].strip()
+            request.response = generated_text
+        elif llm_response.status_code == 502:
+            request.response = "Received 502 error from LLM service - service initialising, try again shortly"
+        else:
+            request.response = f"Received {llm_response.status_code} - {llm_response.text}"
 
 
 class ResponseTrimmer(ArchitectureComponent):
@@ -433,3 +454,5 @@ class ResponseTrimmer(ArchitectureComponent):
 
     def config_description(self) -> str:
         return f"Regexes: {self.regex_display}"
+
+