Added LLM Response Evaluation frontend

error_output.txt

@@ -0,0 +1,10 @@
+```python
+["This is a sample LLM response."]
+```
+```python
+["{'text': 'This is a sample LLM response.'}"]
+```
+def process(input: str):
+    sentences = input.replace("{'text': '", "").replace("'}", "").split(". ")
+    processed_sentences = [sentences[0] + ".", sentences[1] + "."]
+    return processed_sentences

requirements.txt

@@ -1,174 +1,172 @@
-aiofiles
-aiohttp
-aiosignal
-alabaster
-altair
-annotated-types
-anyio
-arrow
-async-timeout
-asyncio
-attrs
-Babel
-backoff
-beautifulsoup4
-bibtexparser
-blinker
-blis
-bs4
-cachetools
-catalogue
-certifi
-charset-normalizer
-click
-cloudpathlib
-confection
-contourpy
-cycler
-cymem
-datasets
-Deprecated
-dill
-distro
-dnspython
-docutils
-email_validator
+aiohttp==3.8.4
+aiosignal==1.3.1
+alabaster==0.7.16
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+arrow==1.3.0
+async-timeout==4.0.3
+asyncio==3.4.3
+attrs==23.2.0
+Babel==2.15.0
+backoff==2.2.1
+beautifulsoup4==4.12.3
+bibtexparser==1.4.1
+blinker==1.8.2
+blis==0.7.11
+bs4==0.0.2
+cachetools==5.4.0
+catalogue==2.0.10
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.18.1
+confection==0.1.5
+contourpy==1.2.1
+cycler==0.12.1
+cymem==2.0.8
+datasets==2.20.0
+Deprecated==1.2.14
+dill==0.3.8
+distro==1.9.0
+docutils==0.20.1
 en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl#sha256=86cc141f63942d4b2c5fcee06630fd6f904788d2f0ab005cce45aadb8fb73889
-evaluate
-factool
-fake-useragent
-fastapi
-fastapi-cli
-ffmpy
-filelock
-Flask
-fonttools
-free-proxy
-frozenlist
-fsspec
-gitdb
-GitPython
-gradio
-gradio_client
-h11
-httpcore
-httptools
-httpx
-huggingface-hub
-idna
-imagesize
-importlib_resources
-itsdangerous
-Jinja2
-joblib
-jsonlines
-jsonschema
-jsonschema-specifications
-kiwisolver
-langcodes
-language_data
-lxml
-marisa-trie
-markdown-it-py
-MarkupSafe
-matplotlib
-mdurl
-mpmath
-multidict
-multiprocess
-murmurhash
-networkx
-nltk
-numpy
-openai
-orjson
-outcome
-packaging
-pandas
-pillow
-preshed
-protobuf
-pyarrow
-pyarrow-hotfix
-pydantic
-pydantic_core
-pydeck
-pydub
-Pygments
-pyparsing
-PySocks
-python-dateutil
-python-dotenv
-python-multipart
-pytz
-PyYAML
-referencing
-regex
-requests
-rich
-rpds-py
-ruff
-safetensors
-scholarly
-scikit-learn
-scipy
-selenium
-semantic-version
-sentence-transformers
-shellingham
-six
-smart-open
-smmap
-sniffio
-snowballstemmer
-sortedcontainers
-soupsieve
-spacy
-spacy-legacy
-spacy-loggers
-Sphinx
-sphinx-rtd-theme
-sphinxcontrib-applehelp
-sphinxcontrib-devhelp
-sphinxcontrib-htmlhelp
-sphinxcontrib-jquery
-sphinxcontrib-jsmath
-sphinxcontrib-qthelp
-sphinxcontrib-serializinghtml
-srsly
-starlette
-streamlit
-streamlit-option-menu
-sympy
-tenacity
-thinc
-threadpoolctl
-tokenizers
-toml
-tomlkit
-toolz
-torch
-torchaudio
-torchvision
-tornado
-tqdm
-transformers
-trio
-trio-websocket
-typer
-types-python-dateutil
-typing_extensions
-tzdata
-urllib3
-uvicorn
-uvloop
-wasabi
-watchfiles
-weasel
-websocket-client
-websockets
-Werkzeug
-wrapt
-wsproto
-xxhash
-yarl
+entrypoints==0.4
+evaluate==0.4.2
+factool==0.1.3
+fake-useragent==1.5.1
+Faker==26.0.0
+fastapi==0.96.0
+favicon==0.7.0
+filelock==3.15.4
+fonttools==4.53.1
+free-proxy==1.1.1
+frozenlist==1.4.1
+fsspec==2024.5.0
+gitdb==4.0.11
+GitPython==3.1.43
+h11==0.14.0
+htbuilder==0.6.2
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.2
+idna==3.7
+imagesize==1.4.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonlines==4.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+langcodes==3.4.0
+language_data==1.2.0
+lxml==5.2.2
+marisa-trie==1.2.0
+Markdown==3.6
+markdown-it-py==3.0.0
+markdownlit==0.0.7
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+mdurl==0.1.2
+more-itertools==10.3.0
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+murmurhash==1.0.10
+networkx==3.3
+nltk==3.8.1
+numpy==1.26.4
+openai==1.37.0
+outcome==1.3.0.post0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+preshed==3.0.9
+prometheus_client==0.20.0
+protobuf==5.27.2
+pyarrow==17.0.0
+pyarrow-hotfix==0.6
+pydantic==1.10.9
+pydantic_core==2.20.1
+pydeck==0.9.1
+Pygments==2.18.0
+pymdown-extensions==10.8.1
+pyparsing==3.1.2
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+PyYAML==6.0
+referencing==0.35.1
+regex==2024.7.24
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.1
+safetensors==0.4.3
+scholarly==1.7.11
+scikit-learn==1.5.1
+scipy==1.14.0
+selenium==4.23.1
+sentence-transformers==3.0.1
+shellingham==1.5.4
+six==1.16.0
+smart-open==7.0.4
+smmap==5.0.1
+sniffio==1.3.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+spacy==3.7.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+Sphinx==7.4.7
+sphinx-rtd-theme==2.0.0
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.6
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.8
+sphinxcontrib-serializinghtml==1.1.10
+srsly==2.4.8
+st-annotated-text==4.0.1
+st-theme==1.2.3
+starlette==0.27.0
+streamlit==1.36.0
+streamlit-camera-input-live==0.2.0
+streamlit-card==1.0.2
+streamlit-embedcode==0.1.2
+streamlit-extras==0.4.3
+streamlit-faker==0.0.3
+streamlit-image-coordinates==0.1.9
+streamlit-keyup==0.2.4
+streamlit-option-menu==0.3.13
+streamlit-toggle-switch==1.0.2
+streamlit-vertical-slider==2.5.5
+sympy==1.13.1
+tenacity==8.5.0
+thinc==8.2.5
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+toml==0.10.2
+toolz==0.12.1
+torch==2.4.0
+tornado==6.4.1
+tqdm==4.66.4
+transformers==4.43.2
+trio==0.26.0
+trio-websocket==0.11.1
+typer==0.12.3
+types-python-dateutil==2.9.0.20240316
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.22.0
+validators==0.33.0
+wasabi==1.1.3
+weasel==0.4.1
+websocket-client==1.8.0
+wrapt==1.16.0
+wsproto==1.2.0
+xxhash==3.4.1
+yarl==1.9.4

scripts/app.sh

@@ -11,4 +11,4 @@ source "${BASH_SOURCE%/*}/common.sh"
 
 # Executing Python script
 export PYTHONPATH="$PYTHONPATH:src/"
-gradio src/openfactcheck/app/app.py --demo-name=demo
+streamlit run src/openfactcheck/app/app.py -- "$@"

src/openfactcheck/app/app.py

@@ -1,11 +1,31 @@
+import argparse
 import streamlit as st
 from streamlit_option_menu import option_menu
 
+from openfactcheck.core.base import OpenFactCheck, OpenFactCheckConfig
+from openfactcheck.app.evaluate_response import evaluate_response
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Initialize OpenFactCheck with custom configuration.')
+    
+    # Add arguments here, example:
+    parser.add_argument("--config-path", 
+                        type=str, 
+                        help="Config File Path",
+                        default="config.json")
+    
+    # Parse arguments from command line
+    args = parser.parse_args()
+    return args
+
 class App:
     def __init__(self):
         pass
 
-    def run(self):
+    def run(self, config_path: str = "config.json"):
+        # Initialize OpenFactCheck
+        config = OpenFactCheckConfig(config_path)
+        ofc = OpenFactCheck(config)
 
         # Set up Dashboard
         st.set_page_config(page_title="OpenFactCheck Dashboard", 
@@ -24,6 +44,20 @@ class App:
             orientation="horizontal"
         )
 
+        # Load the selected page
+        if selected == "Evaluate LLM Response":
+            evaluate_response(ofc)
+        # elif selected == "Evaluate LLM":
+        #     evaluate_llm()
+        # elif selected == "Evaluate FactChecker":
+        #     evaluate_factchecker()
+        # elif selected == "Leaderboards":
+        #     leaderboards()
+        # else:
+        #     about()
+
 if __name__ == "__main__":
+    args = parse_args()
+
     app = App()
-    app.run()
+    app.run(args.config_path)

src/openfactcheck/app/evaluate_response.py

@@ -0,0 +1,154 @@
+import re
+import time
+import pandas as pd
+import streamlit as st
+
+from openfactcheck.core.base import OpenFactCheck
+from openfactcheck.app.utils import style_metric_cards
+
+# Create a function to check a LLM response
+def evaluate_response(ofc: OpenFactCheck):
+    """
+    This function creates a Streamlit app to evaluate the factuality of a LLM response.
+    """
+    if 'response' not in st.session_state:
+        st.session_state.response = None
+
+    # Initialize the solvers
+    claimprocessors = ofc.list_claimprocessors()
+    retrievers = ofc.list_retrievers()
+    verifiers = ofc.list_verifiers()
+
+    st.write("This is where you can check factuality of a LLM response.")
+
+    # Customize FactChecker
+    st.write("Customize FactChecker")
+
+    # Dropdown in three columns
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        claimprocessor = st.selectbox("Select Claim Processor", list(claimprocessors))
+    with col2:
+        retriever = st.selectbox("Select Retriever", list(retrievers))
+    with col3:
+        verifier = st.selectbox("Select Verifier", list(verifiers))
+
+    # Input
+    input_text = {"text": st.text_area("Enter LLM response here", "This is a sample LLM response.")}
+
+    # Button to check factuality
+    if st.button("Check Factuality"):
+        with st.status("Checking factuality...", expanded=True) as status:
+            # Configure the pipeline
+            st.write("Configuring pipeline...")
+            ofc.init_pipeline_manually([claimprocessor, retriever, verifier])
+            st.write("Pipeline configured...")
+
+            # Evaluate the response
+            st.write("Evaluating response...")
+
+            response = ofc(input_text, stream=True)
+            st.write("Response evaluated...")
+
+            status.update(label="Factuality checked...", state="complete", expanded=False)
+
+        # Display pipeline configuration
+        pipeline_str = "   ┈➤   ".join([claimprocessor, retriever, verifier])
+        st.info(f"""**Pipeline**:    \n{pipeline_str}""")
+
+        # Store the final response in the session state
+        st.session_state.final_response = None
+
+        col1, col2 = st.columns([3, 1])
+        with col1:
+            def process_stream(responses):
+                """
+                Process each response from the stream as a simulated chat output.
+                This function yields each word from the formatted text of the response,
+                adding a slight delay to simulate typing in a chat.
+                """
+
+                for response in responses:
+                    if "claimprocessor" in response["solver_name"]:
+                        # Extract response details
+                        output_text = response["output"]
+
+                        # Get the number of detected claims
+                        detected_claims = output_text.get("claims", [])
+
+                        def extract_text(claim):
+                            """
+                            Extracts text from a claim that might be a string formatted as a dictionary.
+                            """
+                            # Try to extract text using regular expression if claim is a string formatted as a dictionary
+                            match = re.search(r"'text': '([^']+)'", claim)
+                            if match:
+                                return match.group(1)
+                            return claim  # Return as is if no dictionary format detected
+
+                        # Generate formatted text with enumerated claims in Markdown format
+                        formatted_text = "#### Detected Claims\n" + "\n".join(f"{i}. {extract_text(claim)}" for i, claim in enumerate(detected_claims, start=1)) + "\n"
+
+                        with col2:
+                            st.metric(label="Detected Claims", value=len(detected_claims))
+                            style_metric_cards(background_color="#F0F0F0", border_color="#F0F0F0", border_radius_px=0)
+
+                        # Yield each word with a space and simulate typing by sleeping
+                        for word in formatted_text.split(" "):
+                            yield word + " "
+                            time.sleep(0.01)
+
+                        st.session_state.claimprocessor_flag = True
+
+                    elif "retriever" in response["solver_name"]:
+                        # Extract response details
+                        output_text = response["output"]
+
+                        evidences = []
+                        for _, claim_with_evidences in output_text.get("claims_with_evidences", {}).items():
+                            for evidence in claim_with_evidences:
+                                evidences.append(evidence[1])
+
+                        # Generate formatted text with enumerated evidences in Markdown format
+                        formatted_text = "#### Retrieved Evidences\n" + "\n".join(f"{i}. {evidence}" for i, evidence in enumerate(evidences, start=1))
+
+                        with col2:
+                            st.metric(label="Retrieved Evidences", value=len(evidences))
+                            style_metric_cards(background_color="#F0F0F0", border_color="#F0F0F0", border_radius_px=0)
+
+                        # Yield each word with a space and simulate typing by sleeping
+                        for word in formatted_text.split(" "):
+                            yield word + " "
+                            time.sleep(0.01)
+
+                    elif "verifier" in response["solver_name"]:
+                        # Extract response details
+                        output_text = response["output"]
+
+                        # Store the final response in the session state
+                        st.session_state.final_response = output_text
+
+                        # Yield each word with a space and simulate typing by sleeping
+                        for word in formatted_text.split(" "):
+                            yield word + " "
+                            time.sleep(0.01)
+
+            st.write_stream(process_stream(response))
+
+            # Process the final response
+            final_response = st.session_state.final_response
+            if final_response is not None:
+                overall_factuality = final_response.get("label", "Unknown")
+                with col2:
+                    if overall_factuality == True:
+                        st.metric(label="Overall Factuality", value="True")
+                        style_metric_cards(background_color="#D4EDDA", border_color="#D4EDDA", border_radius_px=0, border_left_color="#28A745")
+                    elif overall_factuality == False:
+                        st.metric(label="Overall Factuality", value="False")
+                        style_metric_cards(background_color="#F8D7DA", border_color="#F8D7DA", border_radius_px=0, border_left_color="#DC3545")
+
+    # Button to reset
+    if st.session_state.response is not None:
+        if st.button("Reset"):
+            st.session_state.response = None
+            st.rerun()

src/openfactcheck/app/utils.py

@@ -0,0 +1,35 @@
+import streamlit as st
+
+def style_metric_cards(
+    background_color: str = "#FFF",
+    border_size_px: int = 1,
+    border_color: str = "#CCC",
+    border_radius_px: int = 5,
+    border_left_color: str = "#9AD8E1",
+) -> None:
+    """
+    Applies a custom style to st.metrics in the page
+
+    Args:
+        background_color (str, optional): Background color. Defaults to "#FFF".
+        border_size_px (int, optional): Border size in pixels. Defaults to 1.
+        border_color (str, optional): Border color. Defaults to "#CCC".
+        border_radius_px (int, optional): Border radius in pixels. Defaults to 5.
+        border_left_color (str, optional): Borfer left color. Defaults to "#9AD8E1".
+        box_shadow (bool, optional): Whether a box shadow is applied. Defaults to True.
+    """
+    st.markdown(
+        f"""
+        <style>
+            div[data-testid="stMetric"],
+            div[data-testid="metric-container"] {{
+                background-color: {background_color};
+                border: {border_size_px}px solid {border_color};
+                padding: 5% 5% 5% 10%;
+                border-radius: {border_radius_px}px;
+                border-left: 0.5rem solid {border_left_color} !important;
+            }}
+        </style>
+        """,
+        unsafe_allow_html=True,
+    )

src/openfactcheck/core/base.py

@@ -1,10 +1,11 @@
 import os 
 import sys
+import uuid
 import tqdm
-import yaml
 import json
 import traceback
 from pathlib import Path
+from typing import Callable
 
 from openfactcheck.lib.logger import logger
 from openfactcheck.lib.config import OpenFactCheckConfig
@@ -12,6 +13,66 @@ from openfactcheck.core.solver import SOLVER_REGISTRY, Solver
 from openfactcheck.core.state import FactCheckerState
 
 class OpenFactCheck:
+    """
+    OpenFactCheck class to evaluate the factuality of a response using a pipeline of solvers.
+
+    Parameters
+    ----------
+    config : OpenFactCheckConfig
+        An instance of OpenFactCheckConfig containing the configuration
+        settings for OpenFactCheck.
+
+    Attributes
+    ----------
+    logger : Logger
+        An instance of the logger to log messages.
+    config : OpenFactCheckConfig
+        An instance of OpenFactCheckConfig containing the configuration
+        settings for OpenFactCheck.
+    solver_configs : dict
+        A dictionary containing the configuration settings for the solvers.
+    pipeline : list
+        A list of solvers to be included in the pipeline.
+    output_path : str
+        The path to the output directory where the results will be stored.
+
+    Methods
+    -------
+    load_solvers(solver_paths)
+        Load solvers from the given paths.
+    list_solvers()
+        List all registered solvers.
+    list_claimprocessors()
+        List all registered claim processors.
+    list_retrievers()
+        List all registered retrievers.
+    list_verifiers()
+        List all registered verifiers.
+    init_solver(solver_name, args)
+        Initialize a solver with the given configuration.
+    init_solvers()
+        Initialize all registered solvers.
+    init_pipeline()
+        Initialize the pipeline with the given configuration.
+    init_pipeline_manually(pipeline)
+        Initialize the pipeline with the given configuration.
+    persist_output(state, idx, solver_name, cont, sample_name=0)
+        Persist the output of the solver.
+    read_output(sample_name)
+        Read the output file for the given sample.
+    remove_output(sample_name)
+        Remove the output file for the given sample.
+    __call__(response, question, callback_fun, **kwargs)
+        Evaluate the response using the pipeline.
+
+    Examples
+    --------
+    >>> config = OpenFactCheckConfig("config.json")
+    >>> ofc = OpenFactCheck(config)
+    >>> response, sample_name = ofc("This is a sample response.")
+    >>> output = ofc.read_output(sample_name)
+    >>> ofc.remove_output(sample_name)
+    """
     def __init__(self, config: OpenFactCheckConfig):
         """
         Initialize OpenFactCheck with the given configuration.
@@ -61,6 +122,45 @@ class OpenFactCheck:
         """
         return SOLVER_REGISTRY
     
+    @staticmethod
+    def list_claimprocessors():
+        """
+        List all registered claim processors
+        """
+        # Get all claim processors
+        claimprocessors = {}
+        for solver, value in SOLVER_REGISTRY.items():
+            if "claimprocessor" in solver:
+                claimprocessors[solver] = value
+
+        return claimprocessors
+    
+    @staticmethod
+    def list_retrievers():
+        """
+        List all registered retrievers
+        """
+        # Get all retrievers
+        retrievers = {}
+        for solver, value in SOLVER_REGISTRY.items():
+            if "retriever" in solver:
+                retrievers[solver] = value
+
+        return retrievers
+    
+    @staticmethod
+    def list_verifiers():
+        """
+        List all registered verifiers
+        """
+        # Get all verifiers
+        verifiers = {}
+        for solver, value in SOLVER_REGISTRY.items():
+            if "verifier" in solver:
+                verifiers[solver] = value
+
+        return verifiers
+    
     def init_solver(self, solver_name, args):
         """
         Initialize a solver with the given configuration
@@ -130,36 +230,85 @@ class OpenFactCheck:
         with open(os.path.join(self.output_path, f'{sample_name}.jsonl'), 'a', encoding="utf-8") as f:
             f.write(json.dumps(result, ensure_ascii=False) + '\n')
 
-    def __call__(self, response: str, question: str = None, callback_fun=None, **kwargs):
-        sample_name = kwargs.get("sample_name", 0)
-        solver_output = FactCheckerState(question=question, response=response)
-        oname = "response"
-        for idx, (name, (solver, iname, oname)) in tqdm.tqdm(enumerate(self.pipeline.items()),
-                                                             total=len(self.pipeline)):
-            logger.info(f"Invoking solver: {idx}-{name}")
-            logger.debug(f"State content: {solver_output}")
-            try:
-                solver_input = solver_output
-                cont, solver_output = solver(solver_input, **kwargs)
-                logger.debug(f"Latest result: {solver_output}")
-                if callback_fun:
-                    callback_fun(
-                        index=idx,
-                        sample_name=sample_name,
-                        solver_name=name,
-                        input_name=iname,
-                        output_name=oname,
-                        input=solver_input.__dict__,
-                        output=solver_output.__dict__,
-                        continue_run=cont
-                    )
-                self.persist_output(solver_output, idx, name, cont, sample_name=sample_name)
-            except:
-                print(traceback.format_exc())
-                cont = False
-                oname = iname
-            if not cont:
-                logger.info(f"Break at {name}")
-                break
-
-        return solver_output.get(oname)
+    def read_output(self, sample_name):
+        """
+        Read the output file for the given sample
+        """
+        with open(os.path.join(self.output_path, f'{sample_name}.jsonl'), 'r', encoding="utf-8") as f:
+            return [json.loads(line) for line in f]
+        
+    def remove_output(self, sample_name):
+        """
+        Remove the output file for the given sample
+        """
+        os.remove(os.path.join(self.output_path, f'{sample_name}.jsonl'))
+
+    def __call__(self, response: str, question: str = None, stream: bool = False, callback: Callable = None, **kwargs):
+        """
+        Evaluate the response using the pipeline
+        """
+
+        def evaluate_response():
+            # Check if sample_name is provided in kwargs else generate a random one
+            sample_name = kwargs.get("sample_name", str(uuid.uuid4().hex[:6]))
+
+            # Initialize the state
+            solver_output = FactCheckerState(question=question, response=response)
+
+            # Initialize the output name
+            output_name = "response"
+            for idx, (name, (solver, input_name, output_name)) in tqdm.tqdm(enumerate(self.pipeline.items()),
+                                                                total=len(self.pipeline)):
+                logger.info(f"Invoking solver: {idx}-{name}")
+                logger.info(f"State content: {solver_output}")
+            
+                try:
+                    # Solver input is the output of the previous solver
+                    solver_input = solver_output
+
+                    # Run the solver
+                    cont, solver_output = solver(solver_input, **kwargs)
+
+                    # Persist the output
+                    logger.debug(f"Latest result: {solver_output}")
+                    if callback:
+                        callback(
+                            index=idx,
+                            sample_name=sample_name,
+                            solver_name=name,
+                            input_name=input_name,
+                            output_name=output_name,
+                            input=solver_input.__dict__,
+                            output=solver_output.__dict__,
+                            continue_run=cont
+                        )
+                    
+                    # Stream the output
+                    if stream:
+                        yield {
+                            "index": idx,
+                            "solver_name": name,
+                            "input_name": input_name,
+                            "output_name": output_name,
+                            "input": solver_input.__dict__,
+                            "output": solver_output.__dict__,
+                            "continue_run": cont
+                        }
+
+                    self.persist_output(solver_output, idx, name, cont, sample_name=sample_name)
+                    
+                except:
+                    logger.error(f"Error at {traceback.format_exc()}")
+                    cont = False
+                    output_name = input_name
+
+                # Break if the solver returns False
+                if not cont:
+                    logger.info(f"Break at {name}")
+                    break
+                
+            if not stream:
+                return solver_output.get(output_name)
+        
+        # Execute the generator if stream is True, otherwise process normally
+        return evaluate_response()

src/openfactcheck/solvers/rarr_solvers/rarr_agreement_gate.py

@@ -14,7 +14,7 @@ class RARRAgreementGate(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
-        self.model = self.global_config.get("model", "text-davinci-003")
+        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims = state.get(self.input_name)

src/openfactcheck/solvers/rarr_solvers/rarr_editor.py

@@ -14,8 +14,8 @@ import Levenshtein
 class RARREditor(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "text-davinci-003")
-        # self.model = args.get("model", "text-davinci-003")
+        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
+        # self.model = args.get("model", "gpt-3.5-turbo-instruct")
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
         self.max_edit_ratio = args.get("max_edit_ratio", 100)
         self.output_claim_only = args.get("output_claim_only", False)

src/openfactcheck/solvers/rarr_solvers/rarr_llm_retriever.py

@@ -13,7 +13,7 @@ from .prompts.hallucination_prompts import EVIDENCE_HALLUCINATION
 class LLMRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "text-davinci-003")
+        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims = state.get(self.input_name)

src/openfactcheck/solvers/rarr_solvers/rarr_question_generator.py

@@ -17,7 +17,7 @@ from .prompts import rarr_prompts
 class RARRQuestionGenerator(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "text-davinci-003")
+        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
         self.temperature_qgen = args.get("temperature_qgen", 0.7)
         self.num_rounds_qgen = args.get("num_rounds_qgen", 3)
 

src/openfactcheck/solvers/tutorial_solvers/utils/api.py

@@ -34,7 +34,7 @@ def chatgpt(user_input):
 
 def davinci(prompt):
     # Set up the model and prompt
-    model_engine = "text-davinci-003"
+    model_engine = "gpt-3.5-turbo-instruct"
 
     # Generate a response
     completion = client.completions.create(

src/openfactcheck/solvers/webservice/rarr_rtv.py

@@ -10,7 +10,7 @@ from .rarr_utils import search
 class RARRRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("rarr_model", "text-davinci-003")
+        self.model = self.global_config.get("rarr_model", "gpt-3.5-turbo-instruct")
         self.temperature_qgen = args.get("temperature_qgen", 0.7)
         self.num_rounds_qgen = args.get("num_rounds_qgen", 3)
         self.max_search_results_per_query = args.get("max_search_results_per_query", 5)

src/openfactcheck/solvers/webservice/rarr_vfr.py

@@ -10,7 +10,7 @@ class RARRAgreementGate(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
-        self.model = self.global_config.get("rarr_model", "text-davinci-003")
+        self.model = self.global_config.get("rarr_model", "gpt-3.5-turbo-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims_with_evidences = state.get(self.input_name)