max context length limit

app/main.py

@@ -23,6 +23,7 @@ config = getconfig("params.cfg")
 RETRIEVER = config.get("retriever", "RETRIEVER")
 GENERATOR = config.get("generator", "GENERATOR")
 INGESTOR = config.get("ingestor", "INGESTOR")
+MAX_CONTEXT_CHARS = config.get("general", "MAX_CONTEXT_CHARS")
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -83,7 +84,7 @@ def ingest_node(state: GraphState) -> GraphState:
         try:
             # Call the ingestor's ingest endpoint - use gradio_client.file() for proper formatting
             ingestor_context = client.predict(
-                file(tmp_file_path),  # Use gradio_client.file() to properly format
+                file=tmp_file_path,
                 api_name="/ingest"
             )
             
@@ -168,13 +169,20 @@ def generate_node(state: GraphState) -> GraphState:
         retrieved_context = state.get("context", "")
         ingestor_context = state.get("ingestor_context", "")
         
+        # Limit context size to prevent token overflow
+        MAX_CONTEXT_CHARS = int(MAX_CONTEXT_CHARS)  # Adjust based on your model's limits
+
         combined_context = ""
         if ingestor_context and retrieved_context:
-            combined_context = f"=== UPLOADED DOCUMENT CONTEXT ===\n{ingestor_context}\n\n=== RETRIEVED CONTEXT ===\n{retrieved_context}"
+            # Prioritize ingestor context, truncate if needed
+            ingestor_truncated = ingestor_context[:MAX_CONTEXT_CHARS//2] if len(ingestor_context) > MAX_CONTEXT_CHARS//2 else ingestor_context
+            retrieved_truncated = retrieved_context[:MAX_CONTEXT_CHARS//2] if len(retrieved_context) > MAX_CONTEXT_CHARS//2 else retrieved_context
+            combined_context = f"=== UPLOADED DOCUMENT CONTEXT ===\n{ingestor_truncated}\n\n=== RETRIEVED CONTEXT ===\n{retrieved_truncated}"
         elif ingestor_context:
-            combined_context = f"=== UPLOADED DOCUMENT CONTEXT ===\n{ingestor_context}"
+            ingestor_truncated = ingestor_context[:MAX_CONTEXT_CHARS] if len(ingestor_context) > MAX_CONTEXT_CHARS else ingestor_context
+            combined_context = f"=== UPLOADED DOCUMENT CONTEXT ===\n{ingestor_truncated}"
         elif retrieved_context:
-            combined_context = retrieved_context
+            combined_context = retrieved_context[:MAX_CONTEXT_CHARS] if len(retrieved_context) > MAX_CONTEXT_CHARS else retrieved_context
         
         client = Client(GENERATOR)
         result = client.predict(

params.cfg

@@ -6,3 +6,6 @@ GENERATOR = giz/chatfed_generator
 
 [ingestor]
 INGESTOR = mtyrrell/chatfed_ingestor
+
+[general]
+MAX_CONTEXT_CHARS = 15000