Add application file

Smurfs/agents/answer_agent/answer.py

@@ -0,0 +1,303 @@
+from Smurfs.agents.base import BaseAgent
+from Smurfs.agents.answer_agent.prompt import answer_generation_direct_prompt, answer_generation_prompt, final_answer_generation_prompt, tool_check_prompt, hotpot_answer_parser_prompt
+from typing import Any
+
+class answer_agent(BaseAgent):
+    direct_prompt: Any
+    answer_prompt: Any
+    final_prompt: Any
+    tool_check_prompt: Any
+    HP_parser_prompt: Any
+    
+    def __init__(self, *args, **kwargs):
+        direct_prompt = answer_generation_direct_prompt
+        answer_prompt = answer_generation_prompt
+        final_prompt = final_answer_generation_prompt
+        check_prompt = tool_check_prompt
+        name = "Answer Agent"
+        kwargs.update({"direct_prompt": direct_prompt})
+        kwargs.update({"answer_prompt": answer_prompt})
+        kwargs.update({"final_prompt": final_prompt})
+        kwargs.update({"tool_check_prompt": check_prompt})
+        kwargs.update({"name": name})
+        kwargs.update({"HP_parser_prompt": hotpot_answer_parser_prompt})
+        super().__init__(
+            *args,
+            **kwargs,
+        )
+
+    def run(self, query_id, task, **kwargs):
+        """agent run one step"""
+        if task == "direct":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            result = self.llm.prediction(message)
+            self.log(query_id, result)
+            self.colorful_print(result, "Answer Directly")
+            return result
+        
+        elif task == "answer":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Answer Generation")
+                    break
+                except Exception as e:
+                    print(f"answer generation fails: {e}")
+                    self.log(query_id, f"answer generation fails: {e}")
+                    if ind > 2:
+                        return -1
+                    ind += 1
+                    continue
+            return result
+
+        elif task == "final":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Final Answer Generation")
+                    break
+                except Exception as e:
+                    print(f"answer generation fails: {e}")
+                    self.log(query_id, f"answer generation fails: {e}")
+                    if ind > 2:
+                        return -1
+                    ind += 1
+                    continue
+            return result
+        
+        elif task == "tool_check":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    result = eval(result)
+                    a = result["Reason"]
+                    b = result["Choice"]
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Tool Check")
+                    if 'yes' in b.lower():
+                        return -1, a
+                    else:
+                        return 1, a
+                except Exception as e:
+                    print(f"tool check fails: {e}")
+                    self.log(query_id, f"tool check fails: {e}")
+                    if ind > self.max_retry:
+                        return -1, 'fail'
+                    ind += 1
+                    continue
+
+        elif task == "parse":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    # result = eval(result)
+                    # a = result["Reason"]
+                    # b = result["Choice"]
+                    self.colorful_print(result, "Parse Answer Hotpot QA")
+                    self.log(query_id, result)
+                    # if 'yes' in b.lower():
+                    #     return result, -1
+                    # else:
+                    #     return result, 1
+                    return result
+                except Exception as e:
+                    print(f"answer parse fails: {e}")
+                    self.log(query_id, f"answer parse fails: {e}")
+                    if ind > self.max_retry:
+                        return "answer parse fails"
+                    ind += 1
+                    continue
+
+
+    def get_memory(self, **kwargs):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = kwargs
+    
+    def get_prompt(self, task):
+        """get the prompt for the agent"""
+        if task == "direct":
+            agent_prompt = self.direct_prompt.format(**self.memory)
+        elif task == "answer":
+            agent_prompt = self.answer_prompt.format(**self.memory)
+        elif task == "final":
+            agent_prompt = self.final_prompt.format(**self.memory)
+        elif task == "tool_check":
+            agent_prompt = self.tool_check_prompt.format(**self.memory)
+        elif task == "parse":
+            agent_prompt = self.HP_parser_prompt.format(**self.memory)            
+        return agent_prompt
+
+class stream_answer_agent(BaseAgent):
+    direct_prompt: Any
+    answer_prompt: Any
+    final_prompt: Any
+    tool_check_prompt: Any
+    HP_parser_prompt: Any
+    
+    def __init__(self, *args, **kwargs):
+        direct_prompt = answer_generation_direct_prompt
+        answer_prompt = answer_generation_prompt
+        final_prompt = final_answer_generation_prompt
+        check_prompt = tool_check_prompt
+        name = "Answer Agent"
+        kwargs.update({"direct_prompt": direct_prompt})
+        kwargs.update({"answer_prompt": answer_prompt})
+        kwargs.update({"final_prompt": final_prompt})
+        kwargs.update({"tool_check_prompt": check_prompt})
+        kwargs.update({"name": name})
+        kwargs.update({"HP_parser_prompt": hotpot_answer_parser_prompt})
+        super().__init__(
+            *args,
+            **kwargs,
+        )
+
+    def run(self, query_id, task, **kwargs):
+        """agent run one step"""
+        if task == "direct":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            result = self.llm.prediction(message)
+            self.log(query_id, result)
+            self.colorful_print(result, "Answer Directly")
+            return result, "Answer Directly", self.name, result
+        
+        elif task == "answer":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Answer Generation")
+                    break
+                except Exception as e:
+                    print(f"answer generation fails: {e}")
+                    self.log(query_id, f"answer generation fails: {e}")
+                    if ind > 2:
+                        return -1, "Answer Generation", self.name, str(e)
+                    ind += 1
+                    continue
+            return result, "Answer Generation", self.name, result
+
+        elif task == "final":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Final Answer Generation")
+                    break
+                except Exception as e:
+                    print(f"answer generation fails: {e}")
+                    self.log(query_id, f"answer generation fails: {e}")
+                    if ind > 2:
+                        return -1, "Final Answer Generation", self.name, str(e)
+                    ind += 1
+                    continue
+            return result, "Final Answer Generation", self.name, result
+        
+        elif task == "tool_check":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    result = eval(result)
+                    a = result["Reason"]
+                    b = result["Choice"]
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Tool Check")
+                    if 'yes' in b.lower():
+                        return -1, a
+                    else:
+                        return 1, a
+                except Exception as e:
+                    print(f"tool check fails: {e}")
+                    self.log(query_id, f"tool check fails: {e}")
+                    if ind > self.max_retry:
+                        return -1, 'fail'
+                    ind += 1
+                    continue
+
+        elif task == "parse":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    # result = eval(result)
+                    # a = result["Reason"]
+                    # b = result["Choice"]
+                    self.colorful_print(result, "Parse Answer Hotpot QA")
+                    self.log(query_id, result)
+                    # if 'yes' in b.lower():
+                    #     return result, -1
+                    # else:
+                    #     return result, 1
+                    return result
+                except Exception as e:
+                    print(f"answer parse fails: {e}")
+                    self.log(query_id, f"answer parse fails: {e}")
+                    if ind > self.max_retry:
+                        return "answer parse fails"
+                    ind += 1
+                    continue
+
+
+    def get_memory(self, **kwargs):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = kwargs
+    
+    def get_prompt(self, task):
+        """get the prompt for the agent"""
+        if task == "direct":
+            agent_prompt = self.direct_prompt.format(**self.memory)
+        elif task == "answer":
+            agent_prompt = self.answer_prompt.format(**self.memory)
+        elif task == "final":
+            agent_prompt = self.final_prompt.format(**self.memory)
+        elif task == "tool_check":
+            agent_prompt = self.tool_check_prompt.format(**self.memory)
+        elif task == "parse":
+            agent_prompt = self.HP_parser_prompt.format(**self.memory)            
+        return agent_prompt

Smurfs/agents/answer_agent/prompt.py

@@ -0,0 +1,73 @@
+from langchain.prompts import PromptTemplate
+
+tool_check_prompt = """As a powerful language model, you're equipped to answer user's question with accumulated knowledge.
+However, in some cases, you need to use external APIs to answer accurately.
+Thus, you need to check whether the user's question requires you to call an external API to solve it.
+Here are some tips to help you check: 
+1. If the user's question requires real-time information, since your knowledge base isn't updated in real-time, any such question will demand an API call.
+2. If you need to obtain information (e.g., ID, name, phone number, geographical location, rank, etc.), you need to call the database APIs if you are not sure.
+3. If the question demand a database search or internet research to generate an answer, this is another situation where an API call is necessary.
+4. If the question demand coding and math calculation to generate an answer (e.g., algebraic operation, coding problem), you must call external APIs no matter how simple you think it is.
+If need, please output 'YES'; If not, please output 'NO'
+You need to give reasons first and then decide whether to keep it or not. You must only output in a parsible JSON format. Two example outputs look like:
+Example 1: {{\"Reason\": \"The reason why you think you do not need to call an external API to solve the user's question\", \"Choice\": \"No\"}}
+Example 2: {{\"Reason\": \"The reason why you think you need to call an external API to solve the user's question\", \"Choice\": \"Yes\"}}
+This is the user's question: {question}
+Output: """
+tool_check_prompt = PromptTemplate.from_template(tool_check_prompt)
+
+answer_generation_prompt = """
+You should answer the question based on the response output by the API tool.
+Please note that:
+1. Answer the question in natural language based on the API response reasonably and effectively.
+2. The user cannot directly get API response, so you need to make full use of the response and give the information in the response that can satisfy the user's question in as much detail as possible.
+3. Do not output answer that is too long. Output in 3-6 sentences is OK.
+
+This is the user's question:
+{question}
+This is the API response:
+{call_result}
+Output:"""
+answer_generation_prompt = PromptTemplate.from_template(answer_generation_prompt)
+
+final_answer_generation_prompt = """
+You will be given a complex question and you need to solve it step by step by decomposing it to a series of subtasks that can be solved using a single tool(functions).
+At this step, you need to analyse the previous subtasks and their execution result to generate the answer to the original question reasonably and accurately.
+Please note that:
+1. Answer the question in natural language based on the subtask results reasonably and effectively.
+2. The user cannot directly get the subtask results, so you need to make full use of the subtask results and give the information in the response that can satisfy the user's question in as much detail as possible.
+This is the user's question:
+{question}
+There are logs of previous subtasks and execution results: 
+{previous_log}
+Output:"""
+final_answer_generation_prompt = PromptTemplate.from_template(final_answer_generation_prompt)
+
+answer_generation_direct_prompt = """"You need to answer the user's question.
+This is the user's question: {question}
+Output:"""
+answer_generation_direct_prompt = PromptTemplate.from_template(answer_generation_direct_prompt)
+
+hotpot_answer_parser_prompt = """
+You will need to extract a concise answer from the detailed answer to answer the question in a consice language style.
+Only output your concise answer to the answer.
+For example:
+Question:
+VIVA Media AG changed it's name in 2004. What does their new acronym stand for?
+Detailed Answer:
+The new name of VIVA Media AG since its change in 2004 is "VIVA Media GmbH". In this acronym, "GmbH" is a German term which means "Gesellschaft mit beschränkter Haftung", translating to "company with limited liability" in English. So, the acronym denotes that it is a type of business organization similar to a limited liability company (LLC). VIVA Media GmbH is a company that specializes in publishing, producing, and developing high-quality games for different platforms.
+Output:
+Gesellschaft mit beschränkter Haftung
+
+Question:
+Jaclyn Stapp is married to the former frontman of a band that disbanded in what year?
+Detailed Answer:
+The band Creed effectively ended on December 29, 2002. However, they had a reunion tour that started on August 6, 2009, and ended on October 20, 2009. They also released an album called "Full Circle" on October 27, 2009. Despite these reunions, the band's meteoric rise came to a halt when it split up again in 2004.
+Output:
+2004
+
+Question:
+{question}
+Detailed Answer:
+{detailed_answer}
+Output:"""

Smurfs/agents/base.py

@@ -0,0 +1,51 @@
+from abc import abstractmethod
+from pydantic import BaseModel, Field
+from Smurfs.model.base import BaseLM
+# import json
+import os
+from langchain.prompts import PromptTemplate
+from typing import Any
+from termcolor import colored  
+
+class BaseAgent(BaseModel):
+    name: str
+    llm: BaseLM
+    prompt: Any
+    logger_dir: str
+    memory: dict = Field(default={})
+    max_retry: int = Field(default=10)
+
+    @abstractmethod
+    def run(self, **kwargs):
+        """agent run one step"""
+        pass
+
+    @abstractmethod
+    def get_memory(self, **kwargs):
+        """get relevant memory and add it to agent's memory"""
+        pass
+    
+    @abstractmethod
+    def get_prompt(self, **kwargs):
+        """get the prompt for the agent"""
+        pass
+    
+    def log(self, query_id, content):
+        """write log to the logger file"""
+        logger_file = os.path.join(self.logger_dir, f"{query_id}.txt")
+        with open(logger_file, "a+") as file:
+            file.write("\n##########\n")
+            file.write(f"{self.name}: \n\n")
+            file.write(str(content))
+            file.write("\n##########\n")
+
+    def colorful_print(self, content, task):
+        """print out message in different color"""
+        role_to_color = {
+        "Answer Agent": "red",
+        "Executor Agent": "green",
+        "Planning Agent": "blue",
+        "Verifier Agent": "yellow",
+        }
+
+        print(colored(f"##########{task}##########\n{content}\n", role_to_color[self.name]))

Smurfs/agents/executor_agent/executor.py

@@ -0,0 +1,246 @@
+from Smurfs.agents.base import BaseAgent
+from Smurfs.agents.executor_agent.prompt import generate_thought_prompt, choose_tool_prompt, choose_parameter_prompt
+from Smurfs.inference.utils import change_name, standardize, contain
+from Smurfs.inference.server import get_rapidapi_response
+from typing import Any
+import json
+import os
+import time
+import requests
+
+class executor_agent(BaseAgent):
+    thought_prompt: Any
+    tool_prompt: Any
+    parameter_prompt: Any
+
+    def __init__(self, *args, **kwargs):
+        thought_prompt = generate_thought_prompt
+        tool_prompt = choose_tool_prompt
+        parameter_prompt = choose_parameter_prompt
+        name = "Executor Agent"
+        kwargs.update({"thought_prompt": thought_prompt})
+        kwargs.update({"tool_prompt": tool_prompt})
+        kwargs.update({"parameter_prompt": parameter_prompt})
+        kwargs.update({"name": name})
+        super().__init__(
+            *args,
+            **kwargs,
+        )
+
+    def run(self, query_id, task, **kwargs):
+        """agent run one step"""
+        if task == "thought":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Thought Generation")
+                    return result
+                except Exception as e:
+                    print(f"generating thought fails: {e}")
+                    self.log(query_id, f"generating thought fails: {e}")
+                    if ind > self.max_retry:
+                        return -1
+                    ind += 1
+                    continue
+        
+        elif task == "tool":
+            thought = kwargs["thought"]
+            kwargs["question"] = kwargs["question"]+f"thought: {thought}\n"
+            del kwargs["thought"]
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    start = result.find("{")
+                    end = result.rfind("}")
+                    result = eval(result[start:end+1])
+                    self.colorful_print(result, "Choose Tool")
+                    tool = result['ID']
+                    self.log(query_id, result)
+                    return tool
+                except Exception as e:
+                    print(f"choosing tool fails: {e}")
+                    self.log(query_id, f"choosing tool fails: {e}")
+                    if ind > self.max_retry:
+                        return -1
+                    ind += 1
+                    continue
+
+        elif task == "parameter":
+            thought = kwargs["thought"]
+            del kwargs["thought"]
+            kwargs["question"] = kwargs["question"]+f"thought: {thought}\n"
+            api_dic = kwargs["api_dic"]
+            if len(api_dic["required_parameters"]) == 0 and len(api_dic["optional_parameters"]) == 0:
+                return {}
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    start = result.find("{")
+                    end = result.rfind("}")
+                    self.colorful_print(result[start:end+1], "Generate Parameters")
+                    result = result[start:end+1]
+                    clean_answer = eval(
+                        result.replace(": true", ": True").replace(": false", ": False").replace("```", "").strip())
+                    # a = clean_answer["Parameters"]
+                    # clean_answer = clean_answer["Parameters"]
+                    self.log(query_id, clean_answer)
+                    return clean_answer
+                except Exception as e:
+                    print(f"choose parameter fails: {e}")
+                    self.log(query_id, f"choose parameter fails: {e}")
+                    if ind > self.max_retry:
+                        return -1
+                    ind += 1
+                    continue
+
+    def get_memory(self, **kwargs):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = kwargs
+    
+    def get_prompt(self, task):
+        """get the prompt for the agent"""
+        if task == "thought":
+            agent_prompt = self.thought_prompt.format(**self.memory)
+        elif task == "tool":
+            agent_prompt = self.tool_prompt.format(**self.memory)
+        elif task == "parameter":
+            agent_prompt = self.parameter_prompt.format(**self.memory)
+            
+        return agent_prompt
+
+class stream_executor_agent(BaseAgent):
+    thought_prompt: Any
+    tool_prompt: Any
+    parameter_prompt: Any
+
+    def __init__(self, *args, **kwargs):
+        thought_prompt = generate_thought_prompt
+        tool_prompt = choose_tool_prompt
+        parameter_prompt = choose_parameter_prompt
+        name = "Executor Agent"
+        kwargs.update({"thought_prompt": thought_prompt})
+        kwargs.update({"tool_prompt": tool_prompt})
+        kwargs.update({"parameter_prompt": parameter_prompt})
+        kwargs.update({"name": name})
+        super().__init__(
+            *args,
+            **kwargs,
+        )
+
+    def run(self, query_id, task, **kwargs):
+        """agent run one step"""
+        if task == "thought":
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    self.log(query_id, result)
+                    self.colorful_print(result, "Thought Generation")
+                    return result, "Thought Generation", self.name, result
+                except Exception as e:
+                    print(f"generating thought fails: {e}")
+                    self.log(query_id, f"generating thought fails: {e}")
+                    if ind > self.max_retry:
+                        return -1, "Thought Generation", self.name, str(e)
+                    ind += 1
+                    continue
+        
+        elif task == "tool":
+            thought = kwargs["thought"]
+            kwargs["question"] = kwargs["question"]+f"thought: {thought}\n"
+            del kwargs["thought"]
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    start = result.find("{")
+                    end = result.rfind("}")
+                    result = eval(result[start:end+1])
+                    self.colorful_print(result, "Choose Tool")
+                    tool = result['ID']
+                    self.log(query_id, result)
+                    return tool, "Choose Tool", self.name, result
+                except Exception as e:
+                    print(f"choosing tool fails: {e}")
+                    self.log(query_id, f"choosing tool fails: {e}")
+                    if ind > self.max_retry:
+                        return -1, "Choose Tool", self.name, str(e)
+                    ind += 1
+                    continue
+
+        elif task == "parameter":
+            thought = kwargs["thought"]
+            del kwargs["thought"]
+            kwargs["question"] = kwargs["question"]+f"thought: {thought}\n"
+            api_dic = kwargs["api_dic"]
+            if len(api_dic["required_parameters"]) == 0 and len(api_dic["optional_parameters"]) == 0:
+                return {}
+            self.get_memory(**kwargs)
+            agent_prompt = self.get_prompt(task)
+            message = [{'role': 'user', 
+                    'content': agent_prompt}]
+            
+            ind = 0
+            while True:
+                try:
+                    result = self.llm.prediction(message)
+                    start = result.find("{")
+                    end = result.rfind("}")
+                    self.colorful_print(result[start:end+1], "Generate Parameters")
+                    result = result[start:end+1]
+                    clean_answer = eval(
+                        result.replace(": true", ": True").replace(": false", ": False").replace("```", "").strip())
+                    # a = clean_answer["Parameters"]
+                    # clean_answer = clean_answer["Parameters"]
+                    self.log(query_id, clean_answer)
+                    return clean_answer, "Generate Parameters", self.name, result[start:end+1]
+                except Exception as e:
+                    print(f"choose parameter fails: {e}")
+                    self.log(query_id, f"choose parameter fails: {e}")
+                    if ind > self.max_retry:
+                        return -1, "Generate Parameters", self.name, str(e)
+                    ind += 1
+                    continue
+
+    def get_memory(self, **kwargs):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = kwargs
+    
+    def get_prompt(self, task):
+        """get the prompt for the agent"""
+        if task == "thought":
+            agent_prompt = self.thought_prompt.format(**self.memory)
+        elif task == "tool":
+            agent_prompt = self.tool_prompt.format(**self.memory)
+        elif task == "parameter":
+            agent_prompt = self.parameter_prompt.format(**self.memory)
+            
+        return agent_prompt
+

Smurfs/agents/executor_agent/prompt.py

@@ -0,0 +1,58 @@
+from langchain.prompts import PromptTemplate
+
+generate_thought_prompt = """\
+You need to analyse the previous execution history and generate your internal reasoning and thoughts on the task, and how you plan to solve it based on the current attempts.
+
+Do not output thought that is too long. Output in 2-3 sentences is OK.
+
+This is the user's task: 
+{question}
+
+This is the Tool List: 
+{tool_list}
+
+This is the previous execution history:
+{previous_log}
+
+This is the hint comes from the evaluator:
+{hint}
+
+Output:"""
+generate_thought_prompt = PromptTemplate.from_template(generate_thought_prompt)
+
+choose_tool_prompt = """\
+This is the user's question: 
+{question}
+These are the tools you can select to solve the question:
+Tool List:
+{tool_list}
+
+Please note that: 
+1. You should only chooce one tool from the Tool List to solve this question.
+2. You must ONLY output the ID of the tool and your reason for choosing it in a parsible JSON format. An example output looks like:
+'''
+Example: {{\"ID\": ID of the tool, \"Reason\": The reason for choosing the tool}}
+'''
+Output: """
+choose_tool_prompt = PromptTemplate.from_template(choose_tool_prompt)
+
+choose_parameter_prompt="""\
+Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.
+Please note that: 
+1. The Example in the API tool documentation can help you better understand the use of the API.
+2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{}}
+3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.
+4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference.
+5. You must ONLY output in a parsible JSON Format. The example output looks like:
+'''
+Example: {{\"keyword\": \"Artificial Intelligence\", \"language\": \"English\"}}
+'''
+
+There are logs of previous questions and answers: 
+{previous_log}
+
+This is the current user's question: {question}
+
+This is API tool documentation: {api_dic}
+Output:"""
+choose_parameter_prompt = PromptTemplate.from_template(choose_parameter_prompt)

Smurfs/agents/memory_agent/prompt.py

@@ -0,0 +1,16 @@
+from langchain.prompts import PromptTemplate
+
+mem_choose_prompt = """You are a memory agent that controls the memory of the agent system.
+The agent system is trying to solve a complex question step by step by solving its subtasks one by one.
+Among those subtasks, some subtask may need execution history of other subtasks to be solved.
+Your task is to decide which subtasks' execution history is needed by the agent system to solve the current subtask.
+Please note that:
+1. If the current subtask is independent of the other subtasks, just output {{\"task\":}}
+2.  
+You must only output in a parsible JSON format. Two example outputs look like:
+Example 1: {{\"Reason\": \"The reason why you think you do not need to call an external API to solve the user's question\", \"Choice\": \"No\"}}
+Example 2: {{\"Reason\": \"The reason why you think you need to call an external API to solve the user's question\", \"Choice\": \"Yes\"}}
+This is the current subtask: {question}
+This is the previous execution history: {history}
+Output: """
+tool_check_prompt = PromptTemplate.from_template(mem_choose_prompt)

Smurfs/agents/planning_agent/planner.py

@@ -0,0 +1,137 @@
+from Smurfs.agents.base import BaseAgent
+from Smurfs.agents.planning_agent.prompt import task_decompose_prompt, hotpot_task_decompose_prompt
+
+class planning_agent(BaseAgent):
+    def __init__(self, llm, logger_dir):
+        super().__init__(
+            prompt = task_decompose_prompt,
+            llm = llm,
+            name = "Planning Agent",
+            logger_dir = logger_dir
+        )
+
+    def run(self, question, query_id):
+        """agent run one step"""
+        self.get_memory(question)
+        agent_prompt = self.get_prompt()
+        message = [{'role': 'user', 
+                 'content': agent_prompt}]
+        ind = 0
+        while True:
+            try:
+                result = self.llm.prediction(message)
+                # print(result)
+                start = result.find("{")
+                end = result.find("}")
+                result = eval(result[start:end+1])
+                self.colorful_print(result, "Task Decompose")
+                subtasks = result['Tasks']
+                self.log(query_id, result)
+                # print(a)
+                return subtasks
+            except Exception as e:
+                print(f"task deompose fails: {e}")
+                self.log(query_id, f"task deompose fails: {e}")
+                if ind > self.max_retry:
+                    return -1
+                ind += 1
+                continue
+
+    def get_memory(self, question):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = {"question": question}
+    
+    def get_prompt(self):
+        """get the prompt for the agent"""
+        agent_prompt = self.prompt.format(**self.memory)
+        return agent_prompt
+    
+class hotpot_planning_agent(BaseAgent):
+    def __init__(self, llm, logger_dir):
+        super().__init__(
+            prompt = hotpot_task_decompose_prompt,
+            llm = llm,
+            name = "Planning Agent",
+            logger_dir = logger_dir
+        )
+
+    def run(self, question, query_id):
+        """agent run one step"""
+        self.get_memory(question)
+        agent_prompt = self.get_prompt()
+        message = [{'role': 'user', 
+                 'content': agent_prompt}]
+        ind = 0
+        while True:
+            try:
+                result = self.llm.prediction(message)
+                # print(result)
+                start = result.find("{")
+                end = result.find("}")
+                result = eval(result[start:end+1])
+                self.colorful_print(result, "Task Decompose")
+                subtasks = result['Tasks']
+                self.log(query_id, result)
+                # print(a)
+                return subtasks
+            except Exception as e:
+                print(f"task deompose fails: {e}")
+                self.log(query_id, f"task deompose fails: {e}")
+                if ind > self.max_retry:
+                    return -1
+                ind += 1
+                continue
+
+    def get_memory(self, question):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = {"question": question}
+    
+    def get_prompt(self):
+        """get the prompt for the agent"""
+        agent_prompt = self.prompt.format(**self.memory)
+        return agent_prompt
+
+class stream_hotpot_planning_agent(BaseAgent):
+    def __init__(self, llm, logger_dir):
+        super().__init__(
+            prompt = hotpot_task_decompose_prompt,
+            llm = llm,
+            name = "Planning Agent",
+            logger_dir = logger_dir
+        )
+
+    def run(self, question, query_id):
+        """agent run one step"""
+        self.get_memory(question)
+        agent_prompt = self.get_prompt()
+        message = [{'role': 'user', 
+                 'content': agent_prompt}]
+        ind = 0
+        while True:
+            try:
+                result = self.llm.prediction(message)
+                # print(result)
+                start = result.find("{")
+                end = result.find("}")
+                result = eval(result[start:end+1])
+                self.colorful_print(result, "Task Decompose")
+                subtasks = result['Tasks']
+                self.log(query_id, result)
+                # print(a)
+                return subtasks, "Task Decompose", self.name, result
+            except Exception as e:
+                print(f"task deompose fails: {e}")
+                self.log(query_id, f"task deompose fails: {e}")
+                if ind > 5:
+                    return -1, "Task Decompose", self.name, str(e)
+                ind += 1
+                continue
+
+    def get_memory(self, question):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = {"question": question}
+    
+    def get_prompt(self):
+        """get the prompt for the agent"""
+        agent_prompt = self.prompt.format(**self.memory)
+        return agent_prompt

Smurfs/agents/planning_agent/prompt.py

@@ -0,0 +1,44 @@
+from langchain.prompts import PromptTemplate
+
+task_decompose_prompt = """
+You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step.
+Please note that: 
+1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using a single tool.
+2. Each simple subtask should be expressed into natural language.
+3. Each subtask should contain the necessary information from the original question and should be complete, explicit and self-consistent.
+4. You must ONLY output in a parsible JSON format. An example output looks like:
+'''
+{{\"Tasks\": [\"Task 1\", \"Task 2\", ...]}}
+'''
+
+This is the user's question: I'm planning a trip to Turkey and need information about postal codes in Istanbul. Can you provide me with the postal code and district for Istanbul province with plate number 34? Additionally, I would like to know if there are any transit agencies available in Istanbul. Please fetch their names and contact numbers.
+Output: {{\"Tasks\": [\"Find the postal codes and districts for plate number 34 in Istanbul.\", \"Search for transit agencies and their contact numbers in Istanbul.\"]}}
+
+This is the user's question: I recently moved to a new address and I need to update my information. Can you retrieve my address details using the postal code 75094080? Additionally, I would like to know the companies that offer shipping services.
+Output: {{\"Tasks\": [\"retrieve the address details using the postal code 75094080\", \"search for companies that offer shipping services to my address\"]}}
+
+This is the user's question: {question}
+Output:
+"""
+task_decompose_prompt = PromptTemplate.from_template(task_decompose_prompt)
+
+hotpot_task_decompose_prompt = """
+You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step.
+Please note that: 
+1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using a single tool.
+2. Each simple subtask should be expressed into natural language.
+3. Each subtask should contain the necessary information from the original question and should be complete, explicit and self-consistent.
+4. You must ONLY output in a parsible JSON format. An example output looks like:
+'''
+{{\"Tasks\": [\"Task 1\", \"Task 2\", ...]}}
+'''
+
+This is the user's question: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?
+Output: {{\"Tasks\": [\"In the film Kiss and Tell, who is the woman who portrayed Corliss Archer?\", \"What government position was held by this woman?\"]}}
+
+This is the user's question: Were Scott Derrickson and Ed Wood of the same nationality?
+Output: {{\"Tasks\": [\"search for the nationality of Scott Derrickson\", \"search for the nationality for Ed Wood\", \"Compare whether they have the same nationality\"]}}
+
+This is the user's question: {question}
+Output:
+"""

Smurfs/agents/verifier_agent/prompt.py

@@ -0,0 +1,25 @@
+from langchain.prompts import PromptTemplate
+
+final_answer_check_prompt = """
+An agent is trying to solve the query proposed by the user. \
+You need to evaluate whether the given query has been completed reasonably and accurately. If so, summarize the solution to the user. If not, summarize the current progress, and propose what is missing.
+
+You response contains following elements:
+Speak: (your words to the agent if the task isn't completed, or a complete answer based on the full execution log to the user if the task is finished)
+Status: (0 or 1. 0 for unfinished and 1 for finished)
+
+Please note that: 
+1. If the answer says the query can't be solved or it can't answer the query given the current information, please output Status as 0.
+2. Only output Status as 1 if the query has been answered correctly and accurately.
+3. If the answer only give a plan instead of a detailed answer, output Status as 0.
+
+You must only output in a parsible JSON format. Two example outputs look like:
+Example 1: {{\"Speak\": \"answer based on the full execution log to the user\", \"Status\": \"1\"}}
+Example 2: {{\"Speak\": \"your words to the group if the task isn't solved\", \"Status\": \"0\"}}
+
+This is the answer from the previous execution result:
+{answer}
+
+This is the original question: {question}
+Output: """ 
+final_answer_check_prompt = PromptTemplate.from_template(final_answer_check_prompt)

Smurfs/agents/verifier_agent/verifier.py

@@ -0,0 +1,90 @@
+from Smurfs.agents.base import BaseAgent
+from Smurfs.agents.verifier_agent.prompt import final_answer_check_prompt
+
+class verifier_agent(BaseAgent):
+    def __init__(self, llm, logger_dir):
+        super().__init__(
+            prompt = final_answer_check_prompt,
+            llm = llm,
+            name = "Verifier Agent",
+            logger_dir = logger_dir
+        )
+
+    def run(self, question, answer, query_id):
+        """agent run one step"""
+        self.get_memory(question, answer)
+        agent_prompt = self.get_prompt()
+        message = [{'role': 'user', 
+                 'content': agent_prompt}]
+        ind = 0
+        while True:
+            try:
+                result = self.llm.prediction(message)
+                start = result.find("{")
+                end = result.find("}")
+                self.colorful_print(result, "Answer Verify")
+                self.log(query_id, result)
+                clean_result = eval(result[start:end+1])
+                speak = clean_result["Speak"]
+                status = clean_result["Status"]
+                return speak, status
+            except Exception as e:
+                print(f"final answer check fails: {e}")
+                self.log(query_id, f"final answer check fails: {e}")
+                if ind > self.max_retry:
+                    return -1, -1
+                ind += 1
+                continue
+
+    def get_memory(self, question, answer):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = {"question": question, "answer": answer}
+    
+    def get_prompt(self):
+        """get the prompt for the agent"""
+        agent_prompt = self.prompt.format(**self.memory)
+        return agent_prompt
+
+class stream_verifier_agent(BaseAgent):
+    def __init__(self, llm, logger_dir):
+        super().__init__(
+            prompt = final_answer_check_prompt,
+            llm = llm,
+            name = "Verifier Agent",
+            logger_dir = logger_dir
+        )
+
+    def run(self, question, answer, query_id):
+        """agent run one step"""
+        self.get_memory(question, answer)
+        agent_prompt = self.get_prompt()
+        message = [{'role': 'user', 
+                 'content': agent_prompt}]
+        ind = 0
+        while True:
+            try:
+                result = self.llm.prediction(message)
+                start = result.find("{")
+                end = result.find("}")
+                self.colorful_print(result, "Answer Verify")
+                self.log(query_id, result)
+                clean_result = eval(result[start:end+1])
+                speak = clean_result["Speak"]
+                status = clean_result["Status"]
+                return speak, status, "Answer Verify", self.name, result
+            except Exception as e:
+                print(f"final answer check fails: {e}")
+                self.log(query_id, f"final answer check fails: {e}")
+                if ind > self.max_retry:
+                    return -1, -1, "Answer Verify", self.name, str(e)
+                ind += 1
+                continue
+
+    def get_memory(self, question, answer):
+        """get relevant memory and add it to agent's memory"""
+        self.memory = {"question": question, "answer": answer}
+    
+    def get_prompt(self):
+        """get the prompt for the agent"""
+        agent_prompt = self.prompt.format(**self.memory)
+        return agent_prompt

Smurfs/data/post_process.py

@@ -0,0 +1,65 @@
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+# print(sys.path)
+import json
+from Smurfs.data.utils import tree_steps_counter, total_path_transform
+import argparse
+
+def parse_arg():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input_dir', type=str, default="dir_to_your_data", required=False, help='the directory of the data that needs post-processing')
+    parser.add_argument('--example_dir', type=str, default="dir_to_example_data", required=False, help='the directory of the example data')
+    parser.add_argument('--test_sets', nargs='+', type=str, required=False, help='the test sets that need processing. It should be G2_instruction, G2_category or G3_instruction')
+    args = parser.parse_args()
+    return args
+
+def main():
+    args = parse_arg()
+    input_dir = args.input_dir
+    test_sets = args.test_sets
+    example_dir = args.example_dir
+    for test_set in test_sets:
+        data_path = os.path.join(input_dir, f"{test_set}_raw.json")
+        example_data_path = os.path.join(example_dir, f"{test_set}.json")
+
+        with open(data_path, 'r') as file:
+            g_data = json.load(file)
+        with open(example_data_path, 'r') as file:
+            g_example_data = json.load(file)
+        g_new_data = {}
+        for g_d in g_data:
+            m = False
+            for d in g_example_data:
+                if g_data[g_d]["query"] == g_example_data[d]["query"]:
+                    g_new_data_ele = {"query":"", "available_tools": [], "answer":{}}
+                    g_new_answer_ele = {
+                        "method": "smurfs",
+                        "total_steps": 0,
+                        "final_answer": "",
+                        "answer_details": []
+                        }
+                    g_new_data_ele["query"] = g_data[g_d]["query"]
+                    g_new_data_ele["available_tools"] = g_example_data[d]["available_tools"]
+                    g_new_answer_ele["answer_details"] = [g_data[g_d]["answer"]["answer_details"]]
+                    counter = tree_steps_counter(0)
+                    counter.count_total_steps(g_data[g_d]["answer"]["answer_details"])
+                    g_new_answer_ele["total_steps"] = counter.get_steps()
+                    g_new_answer_ele["final_answer"] = g_data[g_d]["answer"]["final_answer"]
+                    g_new_data_ele["answer"] = g_new_answer_ele
+                    g_new_data[d] = g_new_data_ele
+                    m = True
+                    break
+            if not m:
+                print(f"{test_set} mismatch! The key is: {g_d}")
+        if test_set == "G2_category":
+            duplicate = g_new_data["43201"]
+            g_new_data["43200"] = duplicate
+        
+        output_path = os.path.join(input_dir, f"{test_set}.json")
+        print(output_path)
+        with open(output_path, 'w') as file:
+            json.dump(g_new_data, file, indent=4, ensure_ascii=False)
+ 
+if __name__ == '__main__':
+    main()

Smurfs/data/utils.py

@@ -0,0 +1,53 @@
+class tree_steps_counter:
+    def __init__(self, steps):
+        self.steps = steps
+
+    def count_total_steps(self, root):
+        self.steps += 1
+        if root["next"] == []:
+            return
+        for i in range(len(root["next"])):
+            self.count_total_steps(root["next"][i])
+
+    def get_steps(self):
+        return self.steps
+
+def total_path_transform(data, index):
+    finish_template = [{
+                                "role": "tool",
+                                "message": {
+                                    "name": "Finish",
+                                    "arguments": {
+                                        "return_type": "give_answer",
+                                        "final_answer": data[index]["final_answer"]
+                                    },
+                                    "response": ""
+                                },
+                                "next": []
+                            }]
+    answer_path = data[index]["total_path"]["next"][0]["next"]
+    for i in range(len(answer_path)-1, -1, -1):
+        if answer_path[i]["role"] ==  "plan_global":
+            if answer_path[i]["next"] != []:
+                current_log = answer_path[i]["next"][-1]
+                answer_path[i] = answer_path[i]["next"]
+            else:
+                if i == len(answer_path)-1:
+                    answer_path[i] = finish_template[0]
+                continue
+        else:
+            current_log = answer_path[i]
+        while current_log["next"] != []:
+            current_log = current_log["next"][-1]
+        if i == len(answer_path)-1:
+            current_log["next"] = finish_template
+        else:
+            if not isinstance(answer_path[i+1], list):
+                current_log["next"] = [answer_path[i+1]]
+            else:
+                current_log["next"] = answer_path[i+1]
+    if not isinstance(answer_path[0], list):
+        data[index]["total_path"]["next"][0]["next"] = [answer_path[0]]
+    else:
+        data[index]["total_path"]["next"][0]["next"] = answer_path[0]
+

Smurfs/deploy/__init__.py

@@ -0,0 +1,3 @@
+global_dict = {
+    "knowledge_base" : None
+}

Smurfs/deploy/cli_inference.py

@@ -0,0 +1,58 @@
+import warnings
+
+# 抑制所有警告
+warnings.filterwarnings('ignore')
+
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.inference.smurfs_worker import smurfs_hotpot_worker, smurfs_worker
+# from Smurfs.tools.tool_env import HotpotToolEnv
+from Smurfs.tools.tool_env import tool_env
+from Smurfs.model.openai_model.openai_model import OpenAI_Model, OpenRouter_Model
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import executor_agent
+from Smurfs.agents.planning_agent.planner import hotpot_planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+import json
+import threading
+import joblib
+from tqdm import tqdm
+import time
+
+def run(worker, query, query_id):
+    # global lock
+    final_answer, output_file_ele, solution_file_ele = worker.run(query, query_id)
+    # lock.acquire()
+    worker.save_solution(output_file_ele, solution_file_ele, query_id)
+    # lock.release()
+    return final_answer
+
+def cli_run(query, worker):
+    pre = run(worker, query, 0)
+    return pre
+
+if __name__ == '__main__':
+    # model_name = "mistralai/mistral-7b-instruct-v0.2"
+    model_name = "mistralai/mistral-7b-instruct-v0.2"
+    method_name = "cli_inference"
+    tool_doc_path = "Smurfs/tools/math_search.json"
+    # llm = OpenAI_Model(model_name=model_name)
+    llm = OpenRouter_Model(model_name=model_name)
+    # parser_llm = OpenAI_Model(model_name="gpt-4")
+    with open(tool_doc_path, "r") as f:
+        available_tools = json.load(f)
+
+    test_set = "cli"
+        
+    output_dir = f"data/{method_name}/{test_set}/answer"
+    results_dir = f"data/{method_name}/{test_set}/results.json"
+    if not os.path.exists(f"data/{method_name}/{test_set}/parser_log"):
+        os.makedirs(f"data/{method_name}/{test_set}/parser_log")
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    # HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/parser_log")
+    # worker = smurfs_hotpot_worker(available_tools, HotpotToolEnv, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+    worker = smurfs_worker(available_tools, tool_env, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+    query = input("Please Enter Your Task: ")
+    cli_run(query, worker)

Smurfs/deploy/gradio_inference.py

@@ -0,0 +1,223 @@
+import gradio as gr
+
+import warnings
+
+# 抑制所有警告
+warnings.filterwarnings('ignore')
+
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.inference.smurfs_worker import smurfs_hotpot_worker, smurfs_worker, stream_smurfs_worker
+# from Smurfs.tools.tool_env import HotpotToolEnv
+from Smurfs.deploy import global_dict
+from Smurfs.tools.tool_env import tool_env
+from Smurfs.model.openai_model.openai_model import OpenAI_Model
+from Smurfs.agents.answer_agent.answer import stream_answer_agent
+from Smurfs.agents.executor_agent.executor import stream_executor_agent
+from Smurfs.agents.planning_agent.planner import stream_hotpot_planning_agent
+from Smurfs.agents.verifier_agent.verifier import stream_verifier_agent
+
+from Smurfs.tools.docqa.api import tool_env as docqa_tool_env
+from Smurfs.tools.hotpotQA.api import tool_env as hotpot_tool_env
+from Smurfs.tools.math.api import tool_env as math_tool_env
+from Smurfs.tools.shell.api import tool_env as shell_tool_env
+from Smurfs.tools.websearch.api import tool_env as websearch_tool_env
+
+
+import json
+import threading
+import joblib
+from tqdm import tqdm
+import time
+
+from PyPDF2 import PdfReader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain_openai import OpenAIEmbeddings
+from datetime import datetime
+current_datetime = datetime.now()
+# user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=10).style(container=False)
+# inp = gr.Textbox(placeholder="Enter your task")
+# css = """
+# .btn {background-color: blue; color: white;}
+# #bot {background-color: blue; color: white;}
+# #e {display: inline-block; vertical-align: middle;}
+# """
+
+# def update(name):
+#     return f"<span style='color: red'>Welcome to Gradio, {name}!</span>"
+
+tool_env_map = {
+    "shell": shell_tool_env, 
+    "math": math_tool_env, 
+    "docqa": docqa_tool_env, 
+    "hotpotQA": hotpot_tool_env, 
+    "websearch": websearch_tool_env
+}
+
+total_env, env_name_list = {}, []
+
+def loading():
+    return "Loading..."
+
+def load_text_from_pdf(up, key=None):
+    global global_dict
+    if key == None:
+        key = os.environ.get("OPENAI_API_KEY")
+    pdf_path = up.name
+    pdf_reader = PdfReader(pdf_path)
+    text = ""
+    for page in pdf_reader.pages:
+        text += page.extract_text()
+    
+    # split into chunks
+    text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=1000,
+    chunk_overlap=200,
+    add_start_index=True
+    )
+    chunks = text_splitter.split_text(text)
+
+    # create embeddings
+    # embeddings = OpenAIEmbeddings()
+    embeddings = OpenAIEmbeddings(openai_api_key=key)
+    global_dict["knowledge_base"] = FAISS.from_texts(chunks, embeddings)
+    return "upload success!"
+    #return knowledge_base
+
+def update(query, OPENAI_API_KEY, BING_SUBSCRIPT_KEY, WOLFRAMALPH_APP_ID, WEATHER_API_KEYS):
+    global total_env, env_name_list
+    # print(total_env)
+    # print(BING_SUBSCRIPT_KEY)
+    # print(WOLFRAMALPH_APP_ID)
+    # print(WEATHER_API_KEYS)
+    # model_name = "mistralai/mistral-7b-instruct-v0.2"
+    model_name = "gpt-4"
+    method_name = "cli_inference"
+    tool_doc_path = "Smurfs/tools/tool_doc.json"
+    if OPENAI_API_KEY == None or OPENAI_API_KEY == '':
+        yield [(query, "No OPENAI KEY provided!")]
+        raise KeyError
+    if (BING_SUBSCRIPT_KEY == None or BING_SUBSCRIPT_KEY == ''):
+        yield [(query, "No BING_SUBSCRIPT_KEY provided! Please register one from https://www.microsoft.com/en-us/bing/apis/bing-web-search-api and add it to your keys")]
+        raise KeyError
+    if WOLFRAMALPH_APP_ID == None or WOLFRAMALPH_APP_ID == '':
+        yield [(query, "No WOLFRAMALPH_APP_ID provided! please register one from https://products.wolframalpha.com/api/ and add it to your keys")]
+        raise KeyError
+    if WEATHER_API_KEYS == None or WEATHER_API_KEYS == '':
+        yield [(query, "No WEATHER_API_KEYS provided! Please register one from https://www.weatherapi.com/ and add it to")]
+        raise KeyError
+    llm = OpenAI_Model(model_name=model_name, api_key=OPENAI_API_KEY)
+    #llm = OpenRouter_Model(model_name=model_name)
+    if "docqa" in total_env:
+        sys_prompt = llm.sys_prompt + "You already have access to the file uploaded by the user. So just answer the question from the user, you don't need to find the file first."
+        llm.change_sys_prompt(sys_prompt)
+    else:
+        llm.set_default_sys_prompt()
+    # parser_llm = OpenAI_Model(model_name="gpt-4")
+    with open(tool_doc_path, "r") as f:
+        tool_doc = json.load(f)
+    tool_doc["bing_search"]["api_description"] += f"Today is {current_datetime.year}.{current_datetime.month}.{current_datetime.day}"
+    available_tools = []
+    for env_name in env_name_list:
+        available_tools.append(tool_doc[env_name])
+
+    test_set = "cli"
+        
+    output_dir = f"data/{method_name}/{test_set}/answer"
+    results_dir = f"data/{method_name}/{test_set}/results.json"
+    if not os.path.exists(f"data/{method_name}/{test_set}/parser_log"):
+        os.makedirs(f"data/{method_name}/{test_set}/parser_log")
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    # HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/parser_log")
+    # worker = smurfs_hotpot_worker(available_tools, HotpotToolEnv, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+    #print(total_env)
+    worker = stream_smurfs_worker(available_tools, total_env, llm, method_name, test_set, stream_answer_agent, stream_executor_agent, stream_hotpot_planning_agent, stream_verifier_agent, OPENAI_API_KEY, BING_SUBSCRIPT_KEY, WOLFRAMALPH_APP_ID, WEATHER_API_KEYS)
+    stream_generator = worker.run(query, 0)
+    # messages = []
+    while True:
+        try:
+            response = next(stream_generator)
+            messages = [(query, response)]
+            yield messages
+        except StopIteration:
+            break
+    # query = input("Please Enter Your Task: ")
+    # cli_run(query, worker)
+
+def update_tools(rs):
+    global total_env, env_name_list
+    total_env = {}
+    env_name_list = []
+    for tool_system in rs:
+        tool = tool_system.split(": ")[0]
+        env = tool_env_map[tool]
+        print(f"env: {env}")
+        for e in env:
+            if e not in env_name_list:
+                total_env[e] = env[e]
+                env_name_list.append(e)
+    print(total_env)
+    #return total_env, env_name_list
+
+def user(user_msg):
+    return user_msg
+
+tools = ["math: Tool that can handle mathematical problems", 
+         "docqa: Tool that can answer questions about your uploaded file", 
+         "hotpotQA: Tool that can do multi-hop commonsense reasoning", 
+         "websearch: Tool that can do web search to answer your question"]
+websearch_example = ["请根据深圳明天的天气推荐给我推荐一套穿搭方案，结果用中文输出。", "今年的中秋节是哪天？用中文输出"]
+math_example = ["Calc integral of sin(x)+2x^2+3x+1 from 0 to 1", "When both sides of a right triangle are 6 and 8, what is the length of the other side?"]
+inp = gr.Textbox(placeholder="Please input your task", label="Task")
+with gr.Blocks() as demo:
+    gr.HTML("""<h1 align="center">Smurfs</h1>""")
+    #gr.Markdown("""<figure><a href=https://yoursmiles.org/h-smurf.php><img src=https://yoursmiles.org/hsmile/smurf/h3602.gif></a><a href=https://yoursmiles.org/h-smurf.php><img src=https://yoursmiles.org/hsmile/smurf/h3607.gif></a><a href=https://yoursmiles.org/h-smurf.php><img src=https://yoursmiles.org/hsmile/smurf/h3623.gif></a><a href=https://yoursmiles.org/h-smurf.php><img src=https://yoursmiles.org/hsmile/smurf/h3625.gif></a></figure>""")    
+    #gr.HTML("""<a href=https://yoursmiles.org/h-smurf.php><img src=https://yoursmiles.org/hsmile/smurf/h3602.gif>""")
+    with gr.Row():
+        with gr.Column(scale=1):
+            inp.render()
+            rs = gr.Dropdown(choices=tools, label="Tool Systems", multiselect=True)
+            file_output = gr.File(file_types=[".pdf"])
+            with gr.Accordion("Keys", open=False):
+                # model_name = gr.Dropdown(label="Moel Name", choices=["gpt-3.5", "gpt-4o", "gpt-4"])
+                openai_key = gr.Textbox(label="OpenAI API Key", placeholder="Please Enter Your OpenAI API Key")
+                bing_search_key = gr.Textbox(label="BingSearch Key", placeholder="Please Enter Your BingSearch Key from https://www.microsoft.com/en-us/bing/apis/bing-web-search-api")
+                wolframalpha_key = gr.Textbox(label="Wolframalpha API Key", placeholder="Please Enter Your WOLFRAMALPH_APP_ID from https://products.wolframalpha.com/api/")
+                weather_key = gr.Textbox(label="Weather API Key", placeholder="Please Enter Your Weather API Key from https://www.weatherapi.com/")
+
+                
+            gr.Examples(["Who is the brother of the 2022 NBA FMVP?", "How much older is Lebron James than his oldest son?", "Calc integral of sin(x)+2x^2+3x+1 from 0 to 1", "Calculate the length of the hypotenuse of a right triangle when the other two sides are 6 and 8", "请根据深圳明天的天气推荐给我推荐一套穿搭方案，结果用中文输出。", "今年的中秋节是哪天？用中文输出"], inp)
+            _submit = gr.Button("Submit")
+            stop = gr.Button("Stop")
+            clear = gr.Button("Clear")
+            #upload = gr.UploadButton("Click to upload your pdf file")
+            # btn = gr.Button("Run", elem_id="bot", elem_classes="btn")
+        #with gr.Column(scale=1, elem_id="e"):
+            # chatbox = gr.HTML()
+        chatbox = gr.Chatbot(height=300)
+        # btn.click(fn=update, inputs=inp, outputs=chatbox)
+        file_output.upload(load_text_from_pdf, [file_output, openai_key], None)
+        #upload.upload(loading, None, inp).then(load_text_from_pdf, upload, inp)
+        rs.change(update_tools, rs, None)
+        click_event = _submit.click(user, inp, inp).then(update, [inp, openai_key, bing_search_key, wolframalpha_key, weather_key], chatbox)
+        stop.click(None, None, None, cancels=[click_event])
+        #inp.submit(user, inp, inp).then(update, inp, chatbox)
+        clear.click(lambda: (None, None), None, [inp, chatbox], queue=False)
+# theme=gr.themes.Default().set(button_primary_border_color_dark=, hover)
+    demo.load(
+        None,
+        None,
+        _js="""
+        () => {
+        const params = new URLSearchParams(window.location.search);
+        if (!params.has('__theme')){
+            params.set('__theme', 'dark');
+            window.location.search = params.toString();
+        }
+        }    
+        """,
+    )
+demo.queue().launch(server_name='0.0.0.0', share=True, inbrowser=False, server_port=7001)

Smurfs/eval/hotpot_qa/post_process.py

@@ -0,0 +1,109 @@
+import re
+def remove_tags(text):
+    # 使用正则表达式去除形如<xxx>的字符
+    cleaned_text = re.sub(r'<[^>]*>', '', text)
+    return cleaned_text
+
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+from Smurfs.inference.smurfs_worker import smurfs_hotpot_worker
+from Smurfs.tools.tool_env import HotpotToolEnv
+from Smurfs.model.openai_model.openai_model import OpenAI_Model, OpenRouter_Model
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import executor_agent
+from Smurfs.agents.planning_agent.planner import hotpot_planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+from Smurfs.eval.hotpot_qa.utils import eval_result
+import json
+import threading
+import joblib
+from tqdm import tqdm
+import time
+
+def post_run(results, HP_answer_agent):
+    global new_results
+    for res in tqdm(results):
+        pre = res["pre_ans"]
+        query_id = res["id"]
+        ques = res["question"]
+        pre = remove_tags(pre)
+        if len(pre) == 0:
+            res["parsed_pre"] = ""
+            new_results.append(res)
+            continue
+        parsed_pre = HP_answer_agent.run(query_id=query_id, task="parse", question=ques, detailed_answer=pre)
+        res["parsed_pre"] = parsed_pre
+        new_results.append(res)
+
+if __name__ == "__main__":
+    levels = ['easy', 'medium', 'hard']
+    # model_name = "meta-llama/llama-2-70b-chat"
+    method_name = "llama-2-13b-Smurfs"
+    parser_llm = OpenAI_Model(model_name="gpt-4")
+    for level in levels:
+        # level = 'easy'
+        # model_name = "gpt-4-0613"
+        new_results = []
+        test_set = f"hotpot_qa_{level}"
+        HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/post_process/parser_log")
+        output_dir = f"data/{method_name}/{test_set}/answer"
+        new_results_path = f"data/{method_name}/{test_set}/post_process/results.json"
+        new_results_dir = f"data/{method_name}/{test_set}/post_process/"
+        results_dir = f"data/{method_name}/{test_set}/results.json"
+        if not os.path.exists(new_results_path):
+            os.makedirs(f"data/{method_name}/{test_set}/post_process/parser_log")
+            # os.makedirs(new_results_dir)
+            with open(results_dir, "r") as file:
+                    results = json.load(file)
+            
+            total_len = len(results)
+            print(total_len)
+
+            threads = []
+            if total_len < 20:
+                for i in range(total_len):
+                    if total_len == 0:
+                        break
+                
+                    start = i
+                    end = i+1
+                    if i == total_len-1:
+                        query_cur = results[start:]
+                    else:
+                        query_cur = results[start: end]
+                    t = threading.Thread(target=post_run, args=(query_cur, HP_answer_agent))
+                    t.start()
+                    threads.append(t)
+
+            else:
+                for i in range(20):        
+                    
+                    if total_len == 0:
+                        break
+                
+                    start = round(total_len/20)*i
+                    end = round(total_len/20)*(i+1)
+                    if i == 19:
+                        query_cur = results[start:]
+                    else:
+                        query_cur = results[start: end]
+                    t = threading.Thread(target=post_run, args=(query_cur, HP_answer_agent))
+                    t.start()
+                    threads.append(t)
+            
+            for thread in threads:
+                thread.join()
+                
+            with open(new_results_path, "w") as file:
+                json.dump(new_results, file, indent=4, ensure_ascii=False)
+            correct, reward, parsed_correct, parsed_reward, pre_dict, parsed_dict = eval_result(new_results)
+            print(f"correct rate for {test_set} is: {correct}, reward rate for {test_set} is: {reward}")
+            print(f"parsed correct rate for {test_set} is: {parsed_correct}, parsed reward rate for {test_set} is: {parsed_reward}")
+        else:
+            with open(new_results_path, "r") as file:
+                new_results = json.load(file)
+            correct, reward, parsed_correct, parsed_reward, pre_dict, parsed_dict = eval_result(new_results)
+            print(f"correct rate for {test_set} is: {correct}, reward rate for {test_set} is: {reward}")
+            print(f"parsed correct rate for {test_set} is: {parsed_correct}, parsed reward rate for {test_set} is: {parsed_reward}")
+            

Smurfs/eval/hotpot_qa/run_eval.py

@@ -0,0 +1,395 @@
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+from Smurfs.inference.smurfs_worker import smurfs_hotpot_worker
+from Smurfs.tools.tool_env import HotpotToolEnv
+from Smurfs.model.openai_model.openai_model import OpenAI_Model, OpenRouter_Model
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import executor_agent
+from Smurfs.agents.planning_agent.planner import hotpot_planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+from Smurfs.eval.hotpot_qa.utils import eval_result
+import json
+import threading
+import joblib
+from tqdm import tqdm
+import time
+import warnings
+
+# 抑制所有警告
+warnings.filterwarnings('ignore')
+
+def run(worker, query, query_id):
+    # global lock
+    final_answer, output_file_ele, solution_file_ele = worker.run(query, query_id)
+    # lock.acquire()
+    worker.save_solution(output_file_ele, solution_file_ele, query_id)
+    # lock.release()
+    return final_answer
+
+def run_one_hotpot(ques, ans, HP_answer_agent, worker, query_id):
+    global results
+    # ques, ans = task_instructions[0]
+    # print(ques)
+    # print(ans)
+    pre = run(worker, ques, query_id)
+    print(pre)
+
+    # question = "Where was the first governor after the The Missouri Compromise from?"
+    # detailed_answer = "The first governor of Missouri after the Missouri Compromise was Alexander McNair. He was originally from central Pennsylvania, specifically, he was born in Cumberland County on May 5, 1775, and later lived in Derry township, Lancaster (now Dauphin) County. He also pursued his education in Derry and attended the University of Pennsylvania in Philadelphia for a term."
+    parsed_pre = HP_answer_agent.run(query_id=query_id, task="parse", question=ques, detailed_answer=pre)
+    print(parsed_pre)
+
+    result_ele = {"question": ques, "gt_answer": ans, "pre_ans": pre, "parsed_pre": parsed_pre, "id": query_id}
+    lock.acquire()
+    results.append(result_ele)
+    lock.release()
+
+def run_hotpot(query_list, HP_answer_agent, worker):
+    with tqdm(total=len(query_list), desc="Processing files", initial=0) as pbar:
+        for i, test_task_ins in enumerate(query_list, start=0):
+            idx = test_task_ins[0]
+            ques, ans = test_task_ins[1]
+            while True:
+                try:
+                    run_one_hotpot(ques, ans, HP_answer_agent, worker, idx)
+                    break
+                except Exception as e:
+                    print(e)
+                    print("some error occurs, continue...")
+                    time.sleep(60)
+                    continue
+
+            pbar.update(1)
+        return
+
+#测试三个测试集
+# if __name__ == '__main__':
+#     #store true pre and parse pre in a same json, calculate together
+#     #dump them together
+#     lock = threading.Lock()
+#     levels = ['easy', 'medium', 'hard']
+#     model_name = "gpt-3.5-turbo"
+#     method_name = "GPT3-turbo-Smurfs"
+#     llm = OpenAI_Model(model_name=model_name)
+#     parser_llm = OpenAI_Model(model_name="gpt-4")
+#     task_path = "/Users/chenjunzhi/Desktop/smurfs_more/AutoAct/Self_Plan/Group_Planning/benchmark_run/data/hotpotqa"
+#     with open("/Users/chenjunzhi/Desktop/smurfs_more/Smurfs/Smurfs/tools/hotpot.json", "r") as f:
+#         available_tools = json.load(f)
+#     for level in levels:
+#         # level = 'hard'
+#         # model_name = "gpt-4-0613"
+#         results = []
+#         test_set = f"hotpot_qa_{level}"
+        
+#         output_dir = f"data/{method_name}/{test_set}/answer"
+#         results_dir = f"data/{method_name}/{test_set}/results.json"
+#         if not os.path.exists(f"data/{method_name}/{test_set}/parser_log"):
+#             os.makedirs(f"data/{method_name}/{test_set}/parser_log")
+#         if not os.path.exists(output_dir):
+#             os.makedirs(output_dir)
+#         if os.path.exists(results_dir):
+#             with open(results_dir, "r") as file:
+#                 results = json.load(file)
+
+#         items = os.listdir(output_dir)
+#         for i in range(len(items)):
+#             items[i] = items[i].split(".")[0]
+
+#         HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/parser_log")
+#         worker = smurfs_worker(available_tools, tool_env, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+#         hotpot = joblib.load(f'{task_path}/{level}.joblib').reset_index(drop = True)
+#         task_instructions = [(row['question'], row['answer']) for _, row in hotpot.iterrows()]
+
+
+#         query_to_do = []
+#         # if len(items) != 0:
+#         for idx, q in enumerate(task_instructions):
+#             # print(idx)
+#             if str(idx) in items:
+#                 continue
+#             # query_id = q["query_id"]
+#             # if str(query_id) not in test_ids:
+#             #     continue
+#             query_to_do_ele = (idx, q)
+#             query_to_do.append(query_to_do_ele)
+
+#         total_len = len(query_to_do)
+#         query_len = len(task_instructions)
+#         print(total_len)
+
+#         threads = []
+#         if total_len < 20:
+#             for i in range(total_len):
+#                 if total_len == 0:
+#                     break
+            
+#                 start = i
+#                 end = i+1
+#                 if i == total_len-1:
+#                     query_cur = query_to_do[start:]
+#                 else:
+#                     query_cur = query_to_do[start: end]
+#                 t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+#                 t.start()
+#                 threads.append(t)
+
+#         else:
+#             for i in range(20):        
+                
+#                 if total_len == 0:
+#                     break
+            
+#                 start = round(total_len/20)*i
+#                 end = round(total_len/20)*(i+1)
+#                 if i == 19:
+#                     query_cur = query_to_do[start:]
+#                 else:
+#                     query_cur = query_to_do[start: end]
+#                 t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+#                 t.start()
+#                 threads.append(t)
+        
+#         for thread in threads:
+#             thread.join()
+
+#         with open(results_dir, "w") as file:
+#             json.dump(results, file, indent=4, ensure_ascii=False)
+        
+#         with open(results_dir, "r") as file:
+#             eval_data = json.load(file)
+        
+#         correct, reward, parsed_correct, parsed_reward, pre_dict, parsed_dict = eval_result(eval_data)
+#         print(f"correct rate for {test_set} is: {correct}, reward rate for {test_set} is: {reward}")
+#         print(f"parsed correct rate for {test_set} is: {parsed_correct}, parsed reward rate for {test_set} is: {parsed_reward}")
+        
+#         with open(f"data/{method_name}/{test_set}/parsed_result.json", "w") as file:
+#             json.dump(parsed_dict, file, indent=4, ensure_ascii=False)
+        
+#         with open(f"data/{method_name}/{test_set}/original_result.json", "w") as file:
+#             json.dump(pre_dict, file, indent=4, ensure_ascii=False)
+
+
+
+
+
+
+
+# 测试一个query
+# if __name__ == '__main__':
+#     #store true pre and parse pre in a same json, calculate together
+#     #dump them together
+#     lock = threading.Lock()
+#     levels = ['easy', 'medium', 'hard']
+#     model_name = "gpt-3.5-turbo"
+#     method_name = "GPT3-test-Smurfs"
+#     llm = OpenAI_Model(model_name=model_name)
+#     parser_llm = OpenAI_Model(model_name="gpt-4")
+#     task_path = "/Users/chenjunzhi/Desktop/smurfs_more/AutoAct/Self_Plan/Group_Planning/benchmark_run/data/hotpotqa"
+#     with open("/Users/chenjunzhi/Desktop/smurfs_more/Smurfs/Smurfs/tools/hotpot.json", "r") as f:
+#         available_tools = json.load(f)
+#     # for level in levels:
+#     level = 'hard'
+#     # model_name = "gpt-4-0613"
+#     results = []
+#     test_set = f"hotpot_qa_{level}"
+    
+#     output_dir = f"data/{method_name}/{test_set}/answer"
+#     results_dir = f"data/{method_name}/{test_set}/results.json"
+#     if not os.path.exists(f"data/{method_name}/{test_set}/parser_log"):
+#         os.makedirs(f"data/{method_name}/{test_set}/parser_log")
+#     if not os.path.exists(output_dir):
+#         os.makedirs(output_dir)
+#     if os.path.exists(results_dir):
+#         with open(results_dir, "r") as file:
+#             results = json.load(file)
+
+#     items = os.listdir(output_dir)
+#     for i in range(len(items)):
+#         items[i] = items[i].split(".")[0]
+
+#     HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/parser_log")
+#     worker = smurfs_worker(available_tools, tool_env, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+#     hotpot = joblib.load(f'{task_path}/{level}.joblib').reset_index(drop = True)
+#     task_instructions = [(row['question'], row['answer']) for _, row in hotpot.iterrows()]
+#     ques, ans = task_instructions[15][0], task_instructions[15][1]
+#     run_one_hotpot(ques, ans, HP_answer_agent, worker, 0)
+
+
+#     # query_to_do = []
+#     # if len(items) != 0:
+#     # for idx, q in enumerate(task_instructions):
+#     #     # print(idx)
+#     #     if str(idx) in items:
+#     #         continue
+#     #     # query_id = q["query_id"]
+#     #     # if str(query_id) not in test_ids:
+#     #     #     continue
+#     #     query_to_do_ele = (idx, q)
+#     #     query_to_do.append(query_to_do_ele)
+
+#     # total_len = len(query_to_do)
+#     # query_len = len(task_instructions)
+#     # print(total_len)
+
+#     # threads = []
+#     # if total_len < 20:
+#     #     for i in range(total_len):
+#     #         if total_len == 0:
+#     #             break
+        
+#     #         start = i
+#     #         end = i+1
+#     #         if i == total_len-1:
+#     #             query_cur = query_to_do[start:]
+#     #         else:
+#     #             query_cur = query_to_do[start: end]
+#     #         t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+#     #         t.start()
+#     #         threads.append(t)
+
+#     # else:
+#     #     for i in range(20):        
+            
+#     #         if total_len == 0:
+#     #             break
+        
+#     #         start = round(total_len/20)*i
+#     #         end = round(total_len/20)*(i+1)
+#     #         if i == 19:
+#     #             query_cur = query_to_do[start:]
+#     #         else:
+#     #             query_cur = query_to_do[start: end]
+#     #         t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+#     #         t.start()
+#     #         threads.append(t)
+    
+#     # for thread in threads:
+#     #     thread.join()
+
+#     with open(results_dir, "w") as file:
+#         json.dump(results, file, indent=4, ensure_ascii=False)
+    
+#     # with open(results_dir, "r") as file:
+#     #     eval_data = json.load(file)
+    
+#     # correct, reward, parsed_correct, parsed_reward, pre_dict, parsed_dict = eval_result(eval_data)
+#     # print(f"correct rate for {test_set} is: {correct}, reward rate for {test_set} is: {reward}")
+#     # print(f"parsed correct rate for {test_set} is: {parsed_correct}, parsed reward rate for {test_set} is: {parsed_reward}")
+    
+#     # with open(f"data/{method_name}/{test_set}/parsed_result.json", "w") as file:
+#     #     json.dump(parsed_dict, file, indent=4, ensure_ascii=False)
+    
+#     # with open(f"data/{method_name}/{test_set}/original_result.json", "w") as file:
+#     #     json.dump(pre_dict, file, indent=4, ensure_ascii=False)
+
+
+
+
+
+#测试一个个测试集
+if __name__ == '__main__':
+    #store true pre and parse pre in a same json, calculate together
+    #dump them together
+    lock = threading.Lock()
+    levels = ['easy', 'medium', 'hard']
+    model_name = "meta-llama/llama-2-70b-chat"
+    method_name = "llama-2-13b-Smurfs"
+    # llm = OpenAI_Model(model_name=model_name)
+    llm = OpenRouter_Model(model_name=model_name)
+    parser_llm = OpenAI_Model(model_name="gpt-4")
+    task_path = "/Users/chenjunzhi/Desktop/smurfs_more/AutoAct/Self_Plan/Group_Planning/benchmark_run/data/hotpotqa"
+    with open("/Users/chenjunzhi/Desktop/smurfs_more/Smurfs/Smurfs/tools/hotpot.json", "r") as f:
+        available_tools = json.load(f)
+    for level in levels:
+    # level = 'easy'
+    # model_name = "gpt-4-0613"
+        results = []
+        test_set = f"hotpot_qa_{level}"
+        
+        output_dir = f"data/{method_name}/{test_set}/answer"
+        results_dir = f"data/{method_name}/{test_set}/results.json"
+        if not os.path.exists(f"data/{method_name}/{test_set}/parser_log"):
+            os.makedirs(f"data/{method_name}/{test_set}/parser_log")
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        if os.path.exists(results_dir):
+            with open(results_dir, "r") as file:
+                results = json.load(file)
+
+        items = os.listdir(output_dir)
+        for i in range(len(items)):
+            items[i] = items[i].split(".")[0]
+
+        HP_answer_agent = answer_agent(llm=parser_llm, logger_dir=f"data/{method_name}/{test_set}/parser_log")
+        worker = smurfs_hotpot_worker(available_tools, HotpotToolEnv, llm, method_name, test_set, answer_agent, executor_agent,hotpot_planning_agent, verifier_agent)
+        hotpot = joblib.load(f'{task_path}/{level}.joblib').reset_index(drop = True)
+        task_instructions = [(row['question'], row['answer']) for _, row in hotpot.iterrows()]
+
+
+        query_to_do = []
+        # if len(items) != 0:
+        for idx, q in enumerate(task_instructions):
+            # print(idx)
+            if str(idx) in items:
+                continue
+            # query_id = q["query_id"]
+            # if str(query_id) not in test_ids:
+            #     continue
+            query_to_do_ele = (idx, q)
+            query_to_do.append(query_to_do_ele)
+
+        total_len = len(query_to_do)
+        query_len = len(task_instructions)
+        print(total_len)
+
+        threads = []
+        if total_len < 20:
+            for i in range(total_len):
+                if total_len == 0:
+                    break
+            
+                start = i
+                end = i+1
+                if i == total_len-1:
+                    query_cur = query_to_do[start:]
+                else:
+                    query_cur = query_to_do[start: end]
+                t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+                t.start()
+                threads.append(t)
+
+        else:
+            for i in range(20):        
+                
+                if total_len == 0:
+                    break
+            
+                start = round(total_len/20)*i
+                end = round(total_len/20)*(i+1)
+                if i == 19:
+                    query_cur = query_to_do[start:]
+                else:
+                    query_cur = query_to_do[start: end]
+                t = threading.Thread(target=run_hotpot, args=(query_cur, HP_answer_agent, worker))
+                t.start()
+                threads.append(t)
+        
+        for thread in threads:
+            thread.join()
+
+        with open(results_dir, "w") as file:
+            json.dump(results, file, indent=4, ensure_ascii=False)
+        
+        with open(results_dir, "r") as file:
+            eval_data = json.load(file)
+        
+        correct, reward, parsed_correct, parsed_reward, pre_dict, parsed_dict = eval_result(eval_data)
+        print(f"correct rate for {test_set} is: {correct}, reward rate for {test_set} is: {reward}")
+        print(f"parsed correct rate for {test_set} is: {parsed_correct}, parsed reward rate for {test_set} is: {parsed_reward}")
+        
+        with open(f"data/{method_name}/{test_set}/parsed_result.json", "w") as file:
+            json.dump(parsed_dict, file, indent=4, ensure_ascii=False)
+        
+        with open(f"data/{method_name}/{test_set}/original_result.json", "w") as file:
+            json.dump(pre_dict, file, indent=4, ensure_ascii=False)

Smurfs/eval/hotpot_qa/utils.py

@@ -0,0 +1,117 @@
+import re
+from collections import Counter
+import string
+import json
+
+def format_step(step: str) -> str:
+    step = step.strip('\n').strip().replace('\n', '')
+    if step.startswith("Thought") or step.startswith("Action"):
+        step = step.split()[2:]
+        step = " ".join(step)
+    if "Thought" in step:
+        step = step.split("Thought")[0].strip()
+    if "Action" in step:
+        step = step.split("Action")[0].strip()
+    if "Observation" in step:
+        step = step.split("Observation")[0].strip()
+    return step
+
+def normalize_answer(s):
+    def remove_articles(text):
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+    
+    def white_space_fix(text):
+        return " ".join(text.split())
+    
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+    
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+def f1_score(prediction, ground_truth):
+    normalized_prediction = normalize_answer(prediction)
+    normalized_ground_truth = normalize_answer(ground_truth)
+
+    ZERO_METRIC = (0, 0, 0)
+
+    if normalized_prediction in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:
+        return ZERO_METRIC
+    if normalized_ground_truth in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:
+        return ZERO_METRIC
+  
+    prediction_tokens = normalized_prediction.split()
+    ground_truth_tokens = normalized_ground_truth.split()
+    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+    num_same = sum(common.values())
+    if num_same == 0:
+        return ZERO_METRIC
+    precision = 1.0 * num_same / len(prediction_tokens)
+    recall = 1.0 * num_same / len(ground_truth_tokens)
+    f1 = (2 * precision * recall) / (precision + recall)
+    return f1, precision, recall
+
+def EM(answer, key) -> bool:
+    return normalize_answer(answer) == normalize_answer(key)
+
+def score_string_similarity(str1, str2):
+    if str1 == str2:
+        return 2.0
+    elif " " in str1 or " " in str2:
+        str1_split = str1.split(" ")
+        str2_split = str2.split(" ")
+        overlap = list(set(str1_split) & set(str2_split))
+        return len(overlap) / max(len(str1_split), len(str2_split))
+    else:
+        return 0.0    
+
+def eval_result_once(question, pre, gt):
+    correct = EM(pre, gt)
+    reward = f1_score(pre, gt)[0]
+    # halted = agent.is_halted()
+    # error = agent.run_error
+    # prompt = agent._build_agent_prompt()
+    save_dict = {"question":question, "answer":gt, "prediction": pre, "EM":correct, "reward":reward}
+    # with open(file_path, 'a') as f:
+    #     json.dump(save_dict, f)
+    #     f.write("\n")
+    return save_dict
+
+def eval_result(eval_data):
+    result = []
+    parsed_result = []
+    correct = 0
+    reward = 0
+    parsed_correct = 0
+    parsed_reward = 0
+    total_len = len(eval_data)
+    for d in eval_data:
+        pre = d["pre_ans"]
+        parsed_pre = d["parsed_pre"]
+        gt = d["gt_answer"]
+        question = d["question"]
+        pre_dict = eval_result_once(question, pre, gt)
+        parsed_dict = eval_result_once(question, parsed_pre, gt)
+        result.append(pre_dict)
+        parsed_result.append(parsed_dict)
+
+        correct += pre_dict["EM"]
+        reward += pre_dict["reward"]
+
+        parsed_correct += parsed_dict["EM"]
+        parsed_reward += parsed_dict["reward"]
+
+    correct /= total_len
+    reward /= total_len
+
+    parsed_correct /= total_len
+    parsed_reward /= total_len
+
+    return correct, reward, parsed_correct, parsed_reward, result, parsed_result
+
+
+
+    

Smurfs/inference/functioncall_inference.py

@@ -0,0 +1,533 @@
+# — coding: utf-8 –
+import json
+import sys
+import argparse
+import time
+import requests
+import os
+from utils import change_name, standardize, get_white_list, get_answer_log, get_observation_log, build_tree, get_answer_details, test_sets
+from tqdm import tqdm
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.model.vllm_model.vllm_model import vllm_Model
+from Smurfs.inference.server import get_rapidapi_response
+import threading
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import function_call_executor_agent
+from Smurfs.agents.planning_agent.planner import planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+
+def _Call_function(category, tool_name, api_name, tool_input, strip, white_list, toolbench_key, args):
+    use_rapidapi_key = args.use_rapidapi_key
+    rapidapi_key = os.environ.get("rapidapi_key")
+    api_customization = args.api_customization
+
+    api_name = change_name(standardize(api_name))
+    tool_name = standardize(tool_name)
+    if tool_name not in white_list.keys():
+        print(f"tool name doesn't exist: {tool_name}")
+        return {}, 1
+    standard_tool_name = white_list[tool_name]["standard_tool_name"]
+    payload = {
+                    "category": category,
+                    "tool_name": standard_tool_name,
+                    "api_name": api_name,
+                    "tool_input": tool_input,
+                    "strip": strip,
+                    "toolbench_key": toolbench_key
+                }
+    if use_rapidapi_key or api_customization:
+        payload["rapidapi_key"] = rapidapi_key
+        response = get_rapidapi_response(payload, api_customization=api_customization)
+    else:
+        time.sleep(2) # rate limit: 30 per minute
+        headers = {"toolbench_key": toolbench_key}
+        print(payload)
+        # if tool_input == {}:
+        #     response = requests.post("http://8.218.239.54:8080/rapidapi", headers=headers, timeout=15)
+        # else:
+        response = requests.post("http://8.218.239.54:8080/rapidapi", json=payload, headers=headers, timeout=15)
+        if response.status_code != 200:
+            return json.dumps({"error": f"request invalid, data error. status_code={response.status_code}", "response": ""}), 12
+        try:
+            response = response.json()
+        except:
+            print(response)
+            return json.dumps({"error": f"request invalid, data error", "response": ""}), 12
+    # 1 Hallucinating function names
+    # 4 means that the model decides to pruning by itself
+    # 5 represents api call timeout
+    # 6 for 404
+    # 7 means not subscribed
+    # 8 represents unauthorized
+    # 9 represents too many requests
+    # 10 stands for rate limit
+    # 11 message contains "error" field
+    # 12 error sending request
+    if response["error"] == "API not working error...":
+        status_code = 6
+    elif response["error"] == "Unauthorized error...":
+        status_code = 7
+    elif response["error"] == "Unsubscribed error...":
+        status_code = 8
+    elif response["error"] == "Too many requests error...":
+        status_code = 9
+    elif response["error"] == "Rate limit per minute error...":
+        print("Reach api calling limit per minute, sleeping...")
+        time.sleep(10)
+        status_code = 10
+    elif response["error"] == "Message error...":
+        status_code = 11
+    elif response["error"] != "":
+        status_code = "unexpected error, try again!"
+    else:
+        status_code = 0
+    return json.dumps(response), status_code
+
+def Call_function(category, tool_name, api_name, tool_input, strip, white_list, args):
+    toolbench_key = os.environ.get("toolbench_key")
+    response, status_code = _Call_function(category, tool_name, api_name, tool_input, strip, white_list, toolbench_key, args)
+    if status_code == "unexpected error, try again!":
+        arg = {change_name(k.lower()): v for k, v in tool_input.items()}
+        response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+        if status_code == "unexpected error, try again!":
+            arg = {change_name(k.replace("-", "_")): v for k, v in tool_input.items()}
+            response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+            if status_code == "unexpected error, try again!":
+                arg = {change_name(k.replace("\\", "")): v for k, v in tool_input.items()}
+                response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+                if status_code == "unexpected error, try again!":
+                    print(f"Call function fails")
+                    with open('wrong_log.json', 'a+', encoding='utf-8') as f:
+                        line = json.dumps({
+                            "id": 0,
+                            "parameters": arg,
+                            "wrong": response
+                        }, ensure_ascii=False)
+                        f.write(line + '\n')
+                    return -1
+    return response
+
+def inference(query, relevant_APIs, white_list, subtask, Answer_Agent, Executor_Agent, Verifier_Agent, query_id, args, max_step=3):
+    tool_check_num = Answer_Agent.run(question=query, task="tool_check", query_id=query_id)
+    #direct answer
+    if tool_check_num == 1:
+        input_dic = {"task": query}
+        answer = Answer_Agent.run(input_dic)
+        return answer, answer, None, None
+
+    previous_log = []
+    history_log = []
+    tool_used_dic = {}
+    relevant_APIs_ids = []
+    for idx in relevant_APIs:
+        ele = relevant_APIs[idx]
+        relevant_APIs_ids.append(str(ele["ID"]))
+    restart_time = 0
+    step_num = 0
+    hint = "Beginnig of the agent. No hint yet"
+    retry_tool_id = 0
+    retry_parameter = 0
+    re_time = 0
+    subtask_id = 0
+    restart = 0
+    while True:
+        if step_num >= max_step:
+            print("\n\nReach steps limits, return answers!\n\n")
+            answer_log = get_answer_log(history_log)
+            answer = Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+            return answer, previous_log, re_time, history_log
+        
+        if step_num not in tool_used_dic.keys():
+            tool_used_dic[step_num] = []
+
+        tool_used = tool_used_dic[step_num]
+
+        tool_list = []
+        for idx in relevant_APIs:
+            ele = idx
+            ID = str(ele['api_name'])
+            if ID in tool_used:
+                continue
+            # des = ele['description']
+            # name = ele["tool_name"]
+            # tool_list.append({"ID": ID, "tool_name": name, "description": des})
+            tool_list.append(ele)
+
+        if len(tool_list) == 0:
+            if len(previous_log) == 0:
+                answer_log = get_answer_log(history_log)
+                partial_answer = Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                answer = f"Sorry, I can't answer this question accurately using the existing tools. A partial answer is: {partial_answer}"
+                return answer, previous_log, re_time, history_log
+            else:
+                delete_log = previous_log.pop()
+                tool_used_dic[step_num] = []
+                step_num -= 1
+                tool_used_dic[step_num].append(delete_log["tool"])
+                restart_time += 1
+                re_time += 1
+                continue
+
+        current_log = {"thought": "", "action": "", "action_input": {}, "observation": "", "answer": "", "tool": "","id": subtask_id}
+
+        answer_log = get_answer_log(previous_log)
+       
+        if retry_tool_id == 4:
+            # tool_id = tool_list[0]["ID"]
+            tool_list = tool_list[0]
+            thought = Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+
+        else:
+            thought = Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+            # tool_id = Executor_Agent.run(question=subtask, tool_list=tool_list, thought=thought, query_id=query_id, task="tool")
+        
+        try:
+            tool_id = int(tool_id)
+            tool_id = str(tool_id)
+            if tool_id not in relevant_APIs_ids:
+                re_time += 1
+                retry_tool_id += 1
+                print("Tool ID wrong! Generate tool_id that do not exist!")
+                continue
+            tool_des_json = relevant_APIs[str(tool_id)]
+            retry_tool_id = 0
+        except:
+            retry_tool_id += 1
+            print("Tool ID wrong! Generate tool_id that do not exist!")
+            continue
+        
+        tool_name_list = tool_des_json["tool_name"].split(":")
+        category_name = tool_name_list[0]
+        tool_name = tool_name_list[1]
+        api_name = tool_name_list[2]
+        API_doc = tool_des_json
+        
+        while True:
+            try:
+                parameters = {}
+
+                if retry_parameter == 4:
+                    restart = 1
+                    retry_parameter = 0
+                    print("No Para! Restart!")
+                    break
+                
+                parameter = Executor_Agent.run(api_dic=API_doc, question=query, previous_log=answer_log, thought=thought, query_id=query_id, task="parameter")
+                if parameter == -1:
+                    retry_parameter += 1
+                    re_time += 1
+                    continue
+
+                if parameter == {}:
+                    retry_parameter = 0
+                    parameters = {}
+                    break
+
+                for key in parameter:
+                    value = parameter[key]
+                    key = change_name(key)
+                    parameters[key] = value
+
+                retry_parameter = 0
+                break
+
+            except:
+                if retry_parameter == 4:
+                    parameters = {}
+                    retry_parameter = 0
+                    restart = 1
+                    break
+                retry_parameter += 1
+                print("parameter generation fails, try again!")
+                re_time += 1
+                continue
+    
+        
+        api_name = change_name(standardize(api_name))
+
+        if restart != 1:
+            try:
+                observation = Call_function(category_name, tool_name, api_name, parameters, "truncate", white_list, args)
+            except:
+                observation = -1
+                
+            if observation == -1:
+                restart = 1
+                observation = str({"error": "", "response": "call API fails"})
+
+        if restart == 1:
+            tool_used_dic[step_num].append(str(tool_id))
+            print('****Try Again For This Step****')
+            re_time += 1
+            restart = 0
+            continue
+        
+
+        if len(previous_log) != 0:
+            previous_id = previous_log[-1]["id"]
+        else:
+            previous_id = -1
+
+        current_log["tool"] = str(tool_id)
+        current_log["thought"] = thought
+        current_log["action"] = api_name
+        current_log["action_input"] = parameters
+        current_log["observation"] = observation
+        previous_log.append(current_log)
+
+        observation_log = get_observation_log(previous_log)
+
+        answer = Answer_Agent.run(question=subtask, call_result=observation_log, query_id=query_id, task="answer")
+
+        previous_log[-1]["answer"] = answer
+
+        history_log_ele = {"thought": thought, "action": tool_name, "action_input": parameters, "observation": observation, "answer": answer, "previous_id": previous_id, "id": subtask_id}
+        history_log.append(history_log_ele)
+        subtask_id += 1
+
+        speak, status = Verifier_Agent.run(question=subtask, answer=answer, query_id=query_id)
+        if speak == -1 and status == -1:
+            step_num += 1
+            continue
+
+        try:
+            if int(status) == 0:
+                hint = speak
+                step_num += 1
+                continue
+        except:
+                step_num += 1
+                continue
+            
+        else:
+            return answer, previous_log, re_time, history_log
+
+def decompose_inference(query, relevant_APIs, api_list, white_list, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, query_id, args):
+    while True:
+        subtasks = Planning_Agent.run(question=query, query_id=query_id)
+        if subtasks == -1:
+            continue
+        break
+    task_log = ""
+    history_log = []
+    previous_log_totals = []
+    re_time_total = 0
+    print(subtasks)
+    relevant_API_list = []
+    # tool_id = 0
+    for api in api_list:
+        for relevant_API in relevant_APIs:
+            if relevant_API[0] == api["tool_name"] and relevant_API[1] == api["api_name"]:
+                # new_tool_name = api["category_name"]+":"+api["tool_name"]+":"+api["api_name"]
+                ele = api
+                # ele = {"ID": tool_id, "tool_name": new_tool_name, "description": api["api_description"], "required_parameters": api["required_parameters"], "optional_parameters": api["optional_parameters"]}
+                # for para in api["required_parameters"]:
+                #     para_ele = {
+                #         para["name"]: {
+                #             "type": para["type"],
+                #             "description": para["description": ]
+                #         }
+                #     }
+                relevant_API_list.append(ele)
+                # tool_id += 1
+
+    for subtask in subtasks:
+        task_log += f"question: {subtask}\n"
+        answer, previous_log, re_time, previous_log_total = inference(task_log, relevant_API_list, white_list, subtask, Answer_Agent, Executor_Agent, Verifier_Agent, query_id, args)
+        previous_log_totals.append(previous_log_total)
+        print(answer)
+        history_log += previous_log
+        re_time_total += re_time
+        task_log += f"answer: {answer}\n"
+    final_answer = Answer_Agent.run(question=query, previous_log=task_log, task="final", query_id=query_id)
+    return final_answer, history_log, task_log, re_time_total, previous_log_totals 
+
+def test(query_json, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args):
+    while True:
+        try:
+            global lock
+            total_query = len(query_json)
+            with tqdm(total=total_query, desc="Processing files", initial=0) as pbar:
+                for i, test_query in enumerate(query_json, start=0):
+                    idx = test_query[0]
+                    test_query = test_query[1]
+                    query = test_query["query"]
+                    relevant_APIs = test_query["relevant APIs"]
+                    api_list = test_query["api_list"]
+                    final_answer, previous_log, task_log,re_time, previous_log_totals = decompose_inference(query, relevant_APIs, api_list, white_list, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, idx, args)
+                    answer_details, total_steps = get_answer_details(final_answer, previous_log)
+                    solution_tree, solution_total_steps = build_tree(previous_log_totals, task_log)
+                    output_file_ele = {
+                        "query": query,
+                        "restart_time": re_time,
+                        "answer": {
+                            "method": "decompose_dfs",
+                            "total_steps": total_steps,
+                            "final_answer": final_answer,
+                            "answer_details": answer_details
+                        }
+                    }
+                    
+                    solution_file_ele = {
+                        "query": query,
+                        "total_steps": solution_total_steps,
+                        "task_log": task_log,
+                        "final_answer": final_answer,
+                        "answer_path": answer_details,
+                        "total_path": solution_tree
+                    }
+                    file_name = f"{idx}.json"
+                    output_file = os.path.join(output_dir, file_name)
+                    whole_solution_file = os.path.join(whole_solution_dir, file_name)
+                    lock.acquire()
+                    with open(output_file, "w") as file:
+                        json.dump(output_file_ele, file, ensure_ascii=False, indent=4)
+                    with open(whole_solution_file, "w") as file:
+                        json.dump(solution_file_ele, file, ensure_ascii=False, indent=4)
+                    lock.release()
+                    pbar.update(1)
+            return
+                
+        except Exception as e:
+            print(e)
+            print("some error occurs, continue...")
+            time.sleep(60)
+            continue
+
+def parse_arg():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--test_query_id_path', type=str, default="toolbench_data/data/test_query_ids", required=False, help='test query ids for different test sets')
+    parser.add_argument('--method_name', type=str, default="smurfs-all", required=False, help='the inference method')
+    parser.add_argument('--model_name', type=str, default="your_model_name", required=False, help='the model name for the vllm model')
+    parser.add_argument('--query_file_dir', type=str, default="toolbench_data/data/test_instruction", required=False, help='the directory that contains test sets')
+    parser.add_argument('--tool_env_dir', type=str, default="toolbench_data/data/toolenv/tools", required=False, help='tool environment for the toolbench')
+    parser.add_argument('--toolbench_key', type=str, default="",required=False, help='your toolbench key to request rapidapi service')
+    parser.add_argument('--rapidapi_key', type=str, default="",required=False, help='your rapidapi key to request rapidapi service')
+    parser.add_argument('--use_rapidapi_key', action="store_true", help="To use customized rapidapi service or not.")
+    parser.add_argument('--api_customization', action="store_true", help="To use customized api or not.")
+    args = parser.parse_args()
+
+    return args
+
+if __name__ == '__main__':
+    threads = []
+    lock = threading.Lock()
+
+    args = parse_arg()
+
+    test_query_id_path = args.test_query_id_path
+    method_name = args.method_name
+    model_name = args.model_name
+    query_file_dir = args.query_file_dir
+    tool_env_dir = args.tool_env_dir
+
+    toolbench_key = args.toolbench_key
+    rapidapi_key = args.rapidapi_key
+    use_rapidapi_key = args.use_rapidapi_key
+    api_customization = args.api_customization
+
+    chat = vllm_Model(model_name=model_name)
+    
+
+    for test_set in test_sets:
+        total_output_file = f"data/{method_name}/{test_set}_raw.json"
+        
+        test_ids = list(json.load(open(os.path.join(test_query_id_path, test_set+".json"), "r")).keys())
+
+        query_file = f'{query_file_dir}/{test_set}.json'
+        
+        output_dir = f"data/{method_name}/{test_set}/answer"
+
+        whole_solution_dir = f"data/{method_name}/{test_set}/whole"
+
+        logger_dir = f"data/{method_name}/{test_set}/agent_log"
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        if not os.path.exists(whole_solution_dir):
+            os.makedirs(whole_solution_dir)
+        
+        if not os.path.exists(logger_dir):
+            os.makedirs(logger_dir)
+
+        Answer_Agent = answer_agent(llm=chat, logger_dir=logger_dir)
+        Executor_Agent = function_call_executor_agent(llm=chat, logger_dir=logger_dir, max_observation_length=4096, use_rapidapi_key=use_rapidapi_key, api_customization=api_customization, toolbench_key="HcnlyY4DUKOr3mMas51dewgfzAhBVST7EWv0FPtNRjpoi6buJk", service_url="http://8.218.239.54:8080/rapidapi")
+        Planning_Agent = planning_agent(llm=chat, logger_dir=logger_dir)
+        Verifier_Agent = verifier_agent(llm=chat, logger_dir=logger_dir)
+
+        items = os.listdir(output_dir)
+        for i in range(len(items)):
+            items[i] = items[i].split(".")[0]
+
+        white_list = get_white_list(tool_env_dir)
+
+        with open(query_file) as file:
+            query_json = json.load(file)
+        # with open(tool_doc_dir) as file:
+        #     tool_doc = json.load(file)
+        
+        print(len(items))
+        query_json_to_do = []
+        # if len(items) != 0:
+        for idx, q in enumerate(query_json):
+            # print(idx)
+            if str(idx) in items:
+                continue
+            query_id = q["query_id"]
+            if str(query_id) not in test_ids:
+                continue
+            query_json_to_do_ele = (idx, q)
+            query_json_to_do.append(query_json_to_do_ele)
+        # else:
+        #     query_json_to_do = query_json
+        
+        total_len = len(query_json_to_do)
+        query_len = len(query_json)
+        print(total_len)
+
+        if total_len < 20:
+            for i in range(total_len):
+                if total_len == 0:
+                    break
+            
+                start = i
+                end = i+1
+                if i == total_len-1:
+                    query_json_cur = query_json_to_do[start:]
+                else:
+                    query_json_cur = query_json_to_do[start: end]
+                t = threading.Thread(target=test, args=(query_json_cur, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args))
+                t.start()
+                threads.append(t)
+            
+                
+        else:
+            for i in range(20):        
+                
+                if total_len == 0:
+                    break
+            
+                start = round(total_len/20)*i
+                end = round(total_len/20)*(i+1)
+                if i == 19:
+                    query_json_cur = query_json_to_do[start:]
+                else:
+                    query_json_cur = query_json_to_do[start: end]
+                t = threading.Thread(target=test, args=(query_json_cur, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args))
+                t.start()
+                threads.append(t)
+        
+        for thread in threads:
+            thread.join()
+
+        total_json = {}
+        items = os.listdir(output_dir)
+        for item in items:
+            item_path = os.path.join(output_dir, item)
+            idx = item.split(".")[0]
+            total_json[str(idx)] = json.load(open(item_path, 'r'))
+
+        with open(total_output_file, 'w') as file:
+            json.dump(total_json, file, indent=4, ensure_ascii=False)
+    

Smurfs/inference/inference.py

@@ -0,0 +1,527 @@
+# — coding: utf-8 –
+import json
+import sys
+import argparse
+import time
+import requests
+import os
+from tqdm import tqdm
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.inference.utils import change_name, standardize, get_white_list, get_answer_log, get_observation_log, build_tree, get_answer_details, test_sets
+from Smurfs.model.vllm_model.vllm_model import vllm_Model
+from Smurfs.inference.server import get_rapidapi_response
+import threading
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import executor_agent
+from Smurfs.agents.planning_agent.planner import planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+import warnings
+
+warnings.filterwarnings('ignore')
+
+def _Call_function(category, tool_name, api_name, tool_input, strip, white_list, toolbench_key, args):
+    use_rapidapi_key = args.use_rapidapi_key
+    rapidapi_key = os.environ.get("rapidapi_key")
+    api_customization = args.api_customization
+
+    api_name = change_name(standardize(api_name))
+    tool_name = standardize(tool_name)
+    if tool_name not in white_list.keys():
+        print(f"tool name doesn't exist: {tool_name}")
+        return {}, 1
+    standard_tool_name = white_list[tool_name]["standard_tool_name"]
+    payload = {
+                    "category": category,
+                    "tool_name": standard_tool_name,
+                    "api_name": api_name,
+                    "tool_input": tool_input,
+                    "strip": strip,
+                    "toolbench_key": toolbench_key
+                }
+    if use_rapidapi_key or api_customization:
+        payload["rapidapi_key"] = rapidapi_key
+        response = get_rapidapi_response(payload, api_customization=api_customization)
+    else:
+        time.sleep(2) # rate limit: 30 per minute
+        headers = {"toolbench_key": toolbench_key}
+        print(payload)
+        # if tool_input == {}:
+        #     response = requests.post("http://8.218.239.54:8080/rapidapi", headers=headers, timeout=15)
+        # else:
+        response = requests.post("http://8.218.239.54:8080/rapidapi", json=payload, headers=headers, timeout=15)
+        if response.status_code != 200:
+            return json.dumps({"error": f"request invalid, data error. status_code={response.status_code}", "response": ""}), 12
+        try:
+            response = response.json()
+        except:
+            print(response)
+            return json.dumps({"error": f"request invalid, data error", "response": ""}), 12
+    # 1 Hallucinating function names
+    # 4 means that the model decides to pruning by itself
+    # 5 represents api call timeout
+    # 6 for 404
+    # 7 means not subscribed
+    # 8 represents unauthorized
+    # 9 represents too many requests
+    # 10 stands for rate limit
+    # 11 message contains "error" field
+    # 12 error sending request
+    if response["error"] == "API not working error...":
+        status_code = 6
+    elif response["error"] == "Unauthorized error...":
+        status_code = 7
+    elif response["error"] == "Unsubscribed error...":
+        status_code = 8
+    elif response["error"] == "Too many requests error...":
+        status_code = 9
+    elif response["error"] == "Rate limit per minute error...":
+        print("Reach api calling limit per minute, sleeping...")
+        time.sleep(10)
+        status_code = 10
+    elif response["error"] == "Message error...":
+        status_code = 11
+    elif response["error"] != "":
+        status_code = "unexpected error, try again!"
+    else:
+        status_code = 0
+    return json.dumps(response), status_code
+
+def Call_function(category, tool_name, api_name, tool_input, strip, white_list, args):
+    toolbench_key = os.environ.get("toolbench_key")
+    response, status_code = _Call_function(category, tool_name, api_name, tool_input, strip, white_list, toolbench_key, args)
+    if status_code == "unexpected error, try again!":
+        arg = {change_name(k.lower()): v for k, v in tool_input.items()}
+        response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+        if status_code == "unexpected error, try again!":
+            arg = {change_name(k.replace("-", "_")): v for k, v in tool_input.items()}
+            response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+            if status_code == "unexpected error, try again!":
+                arg = {change_name(k.replace("\\", "")): v for k, v in tool_input.items()}
+                response, status_code = _Call_function(category, tool_name, api_name, arg, strip, white_list, toolbench_key, args)
+                if status_code == "unexpected error, try again!":
+                    print(f"Call function fails")
+                    with open('wrong_log.json', 'a+', encoding='utf-8') as f:
+                        line = json.dumps({
+                            "id": 0,
+                            "parameters": arg,
+                            "wrong": response
+                        }, ensure_ascii=False)
+                        f.write(line + '\n')
+                    return -1
+    return response
+
+def inference(query, relevant_APIs, white_list, subtask, Answer_Agent, Executor_Agent, Verifier_Agent, query_id, args, max_step=3):
+    tool_check_num = Answer_Agent.run(question=query, task="tool_check", query_id=query_id)
+    #direct answer
+    if tool_check_num == 1:
+        input_dic = {"task": query}
+        answer = Answer_Agent.run(input_dic)
+        return answer, answer, None, None
+
+    previous_log = []
+    history_log = []
+    tool_used_dic = {}
+    relevant_APIs_ids = []
+    for idx in relevant_APIs:
+        ele = relevant_APIs[idx]
+        relevant_APIs_ids.append(str(ele["ID"]))
+    restart_time = 0
+    step_num = 0
+    hint = "Beginnig of the agent. No hint yet"
+    retry_tool_id = 0
+    retry_parameter = 0
+    re_time = 0
+    subtask_id = 0
+    restart = 0
+    while True:
+        if step_num >= max_step:
+            print("\n\nReach steps limits, return answers!\n\n")
+            answer_log = get_answer_log(history_log)
+            answer = Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+            return answer, previous_log, re_time, history_log
+        
+        if step_num not in tool_used_dic.keys():
+            tool_used_dic[step_num] = []
+
+        tool_used = tool_used_dic[step_num]
+
+        tool_list = []
+        for idx in relevant_APIs:
+            ele = relevant_APIs[idx]
+            ID = str(ele['ID'])
+            if ID in tool_used:
+                continue
+            des = ele['description']
+            name = ele["tool_name"]
+            tool_list.append({"ID": ID, "tool_name": name, "description": des})
+
+        if len(tool_list) == 0:
+            if len(previous_log) == 0:
+                answer_log = get_answer_log(history_log)
+                partial_answer = Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                answer = f"Sorry, I can't answer this question accurately using the existing tools. A partial answer is: {partial_answer}"
+                return answer, previous_log, re_time, history_log
+            else:
+                delete_log = previous_log.pop()
+                tool_used_dic[step_num] = []
+                step_num -= 1
+                tool_used_dic[step_num].append(delete_log["tool"])
+                restart_time += 1
+                re_time += 1
+                continue
+
+        current_log = {"thought": "", "action": "", "action_input": {}, "observation": "", "answer": "", "tool": "","id": subtask_id}
+
+        answer_log = get_answer_log(previous_log)
+       
+        if retry_tool_id == 4:
+            tool_id = tool_list[0]["ID"]
+            tool_list = tool_list[0]
+            thought = Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+
+        else:
+            thought = Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+            tool_id = Executor_Agent.run(question=subtask, tool_list=tool_list, thought=thought, query_id=query_id, task="tool")
+        
+        try:
+            tool_id = int(tool_id)
+            tool_id = str(tool_id)
+            if tool_id not in relevant_APIs_ids:
+                re_time += 1
+                retry_tool_id += 1
+                print("Tool ID wrong! Generate tool_id that do not exist!")
+                continue
+            tool_des_json = relevant_APIs[str(tool_id)]
+            retry_tool_id = 0
+        except:
+            retry_tool_id += 1
+            print("Tool ID wrong! Generate tool_id that do not exist!")
+            continue
+        
+        tool_name_list = tool_des_json["tool_name"].split(":")
+        category_name = tool_name_list[0]
+        tool_name = tool_name_list[1]
+        api_name = tool_name_list[2]
+        API_doc = tool_des_json
+        
+        while True:
+            try:
+                parameters = {}
+
+                if retry_parameter == 4:
+                    restart = 1
+                    retry_parameter = 0
+                    print("No Para! Restart!")
+                    break
+                
+                parameter = Executor_Agent.run(api_dic=API_doc, question=query, previous_log=answer_log, thought=thought, query_id=query_id, task="parameter")
+                if parameter == -1:
+                    retry_parameter += 1
+                    re_time += 1
+                    continue
+
+                if parameter == {}:
+                    retry_parameter = 0
+                    parameters = {}
+                    break
+
+                for key in parameter:
+                    value = parameter[key]
+                    key = change_name(key)
+                    parameters[key] = value
+
+                retry_parameter = 0
+                break
+
+            except:
+                if retry_parameter == 4:
+                    parameters = {}
+                    retry_parameter = 0
+                    restart = 1
+                    break
+                retry_parameter += 1
+                print("parameter generation fails, try again!")
+                re_time += 1
+                continue
+    
+        
+        api_name = change_name(standardize(api_name))
+
+        if restart != 1:
+            try:
+                observation = Call_function(category_name, tool_name, api_name, parameters, "truncate", white_list, args)
+            except:
+                observation = -1
+                
+            if observation == -1:
+                restart = 1
+                observation = str({"error": "", "response": "call API fails"})
+
+        if restart == 1:
+            tool_used_dic[step_num].append(str(tool_id))
+            print('****Try Again For This Step****')
+            re_time += 1
+            restart = 0
+            continue
+        
+
+        if len(previous_log) != 0:
+            previous_id = previous_log[-1]["id"]
+        else:
+            previous_id = -1
+
+        current_log["tool"] = str(tool_id)
+        current_log["thought"] = thought
+        current_log["action"] = api_name
+        current_log["action_input"] = parameters
+        current_log["observation"] = observation
+        previous_log.append(current_log)
+
+        observation_log = get_observation_log(previous_log)
+
+        answer = Answer_Agent.run(question=subtask, call_result=observation_log, query_id=query_id, task="answer")
+
+        previous_log[-1]["answer"] = answer
+
+        history_log_ele = {"thought": thought, "action": tool_name, "action_input": parameters, "observation": observation, "answer": answer, "previous_id": previous_id, "id": subtask_id}
+        history_log.append(history_log_ele)
+        subtask_id += 1
+
+        speak, status = Verifier_Agent.run(question=subtask, answer=answer, query_id=query_id)
+        if speak == -1 and status == -1:
+            step_num += 1
+            continue
+
+        try:
+            if int(status) == 0:
+                hint = speak
+                step_num += 1
+                continue
+        except:
+                step_num += 1
+                continue
+            
+        else:
+            return answer, previous_log, re_time, history_log
+
+def decompose_inference(query, relevant_APIs, api_list, white_list, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, query_id, args):
+    while True:
+        subtasks = Planning_Agent.run(question=query, query_id=query_id)
+        if subtasks == -1:
+            continue
+        break
+    task_log = ""
+    history_log = []
+    previous_log_totals = []
+    re_time_total = 0
+    print(subtasks)
+    relevant_API_list = {}
+    tool_id = 0
+    for api in api_list:
+        for relevant_API in relevant_APIs:
+            if relevant_API[0] == api["tool_name"] and relevant_API[1] == api["api_name"]:
+                new_tool_name = api["category_name"]+":"+api["tool_name"]+":"+api["api_name"]
+                ele = {"ID": tool_id, "tool_name": new_tool_name, "description": api["api_description"], "required_parameters": api["required_parameters"], "optional_parameters": api["optional_parameters"]}
+                relevant_API_list[str(tool_id)] = ele
+                tool_id += 1
+
+    for subtask in subtasks:
+        task_log += f"question: {subtask}\n"
+        answer, previous_log, re_time, previous_log_total = inference(task_log, relevant_API_list, white_list, subtask, Answer_Agent, Executor_Agent, Verifier_Agent, query_id, args)
+        previous_log_totals.append(previous_log_total)
+        print(answer)
+        history_log += previous_log
+        re_time_total += re_time
+        task_log += f"answer: {answer}\n"
+    final_answer = Answer_Agent.run(question=query, previous_log=task_log, task="final", query_id=query_id)
+    return final_answer, history_log, task_log, re_time_total, previous_log_totals 
+
+def test(query_json, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args):
+    while True:
+        try:
+            global lock
+            total_query = len(query_json)
+            with tqdm(total=total_query, desc="Processing files", initial=0) as pbar:
+                for i, test_query in enumerate(query_json, start=0):
+                    idx = test_query[0]
+                    test_query = test_query[1]
+                    query = test_query["query"]
+                    relevant_APIs = test_query["relevant APIs"]
+                    api_list = test_query["api_list"]
+                    final_answer, previous_log, task_log,re_time, previous_log_totals = decompose_inference(query, relevant_APIs, api_list, white_list, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, idx, args)
+                    answer_details, total_steps = get_answer_details(final_answer, previous_log)
+                    solution_tree, solution_total_steps = build_tree(previous_log_totals, task_log)
+                    output_file_ele = {
+                        "query": query,
+                        "restart_time": re_time,
+                        "answer": {
+                            "method": "decompose_dfs",
+                            "total_steps": total_steps,
+                            "final_answer": final_answer,
+                            "answer_details": answer_details
+                        }
+                    }
+                    
+                    solution_file_ele = {
+                        "query": query,
+                        "total_steps": solution_total_steps,
+                        "task_log": task_log,
+                        "final_answer": final_answer,
+                        "answer_path": answer_details,
+                        "total_path": solution_tree
+                    }
+                    file_name = f"{idx}.json"
+                    output_file = os.path.join(output_dir, file_name)
+                    whole_solution_file = os.path.join(whole_solution_dir, file_name)
+                    lock.acquire()
+                    with open(output_file, "w") as file:
+                        json.dump(output_file_ele, file, ensure_ascii=False, indent=4)
+                    with open(whole_solution_file, "w") as file:
+                        json.dump(solution_file_ele, file, ensure_ascii=False, indent=4)
+                    lock.release()
+                    pbar.update(1)
+            return
+                
+        except Exception as e:
+            print(e)
+            print("some error occurs, continue...")
+            time.sleep(60)
+            continue
+
+def parse_arg():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--test_query_id_path', type=str, default="toolbench_data/data/test_query_ids", required=False, help='test query ids for different test sets')
+    parser.add_argument('--method_name', type=str, default="smurfs-all", required=False, help='the inference method')
+    parser.add_argument('--model_name', type=str, default="your_model_name", required=False, help='the model name for the vllm model')
+    parser.add_argument('--query_file_dir', type=str, default="toolbench_data/data/test_instruction", required=False, help='the directory that contains test sets')
+    parser.add_argument('--tool_env_dir', type=str, default="toolbench_data/data/toolenv/tools", required=False, help='tool environment for the toolbench')
+    parser.add_argument('--toolbench_key', type=str, default="",required=False, help='your toolbench key to request rapidapi service')
+    parser.add_argument('--rapidapi_key', type=str, default="",required=False, help='your rapidapi key to request rapidapi service')
+    parser.add_argument('--use_rapidapi_key', action="store_true", help="To use customized rapidapi service or not.")
+    parser.add_argument('--api_customization', action="store_true", help="To use customized api or not.")
+    args = parser.parse_args()
+
+    return args
+
+if __name__ == '__main__':
+    threads = []
+    lock = threading.Lock()
+
+    args = parse_arg()
+
+    test_query_id_path = args.test_query_id_path
+    method_name = args.method_name
+    model_name = args.model_name
+    query_file_dir = args.query_file_dir
+    tool_env_dir = args.tool_env_dir
+
+    toolbench_key = args.toolbench_key
+    rapidapi_key = args.rapidapi_key
+    use_rapidapi_key = args.use_rapidapi_key
+    api_customization = args.api_customization
+
+    chat = vllm_Model(model_name=model_name)
+    
+
+    for test_set in test_sets:
+        total_output_file = f"data/{method_name}/{test_set}_raw.json"
+        
+        test_ids = list(json.load(open(os.path.join(test_query_id_path, test_set+".json"), "r")).keys())
+
+        query_file = f'{query_file_dir}/{test_set}.json'
+        
+        output_dir = f"data/{method_name}/{test_set}/answer"
+
+        whole_solution_dir = f"data/{method_name}/{test_set}/whole"
+
+        logger_dir = f"data/{method_name}/{test_set}/agent_log"
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        if not os.path.exists(whole_solution_dir):
+            os.makedirs(whole_solution_dir)
+        
+        if not os.path.exists(logger_dir):
+            os.makedirs(logger_dir)
+
+        Answer_Agent = answer_agent(llm=chat, logger_dir=logger_dir)
+        Executor_Agent = executor_agent(llm=chat, logger_dir=logger_dir)
+        Planning_Agent = planning_agent(llm=chat, logger_dir=logger_dir)
+        Verifier_Agent = verifier_agent(llm=chat, logger_dir=logger_dir)
+
+        items = os.listdir(output_dir)
+        for i in range(len(items)):
+            items[i] = items[i].split(".")[0]
+
+        white_list = get_white_list(tool_env_dir)
+
+        with open(query_file) as file:
+            query_json = json.load(file)
+        # with open(tool_doc_dir) as file:
+        #     tool_doc = json.load(file)
+        
+        print(len(items))
+        query_json_to_do = []
+        # if len(items) != 0:
+        for idx, q in enumerate(query_json):
+            # print(idx)
+            if str(idx) in items:
+                continue
+            query_id = q["query_id"]
+            if str(query_id) not in test_ids:
+                continue
+            query_json_to_do_ele = (idx, q)
+            query_json_to_do.append(query_json_to_do_ele)
+        # else:
+        #     query_json_to_do = query_json
+        
+        total_len = len(query_json_to_do)
+        query_len = len(query_json)
+        print(total_len)
+
+        if total_len < 20:
+            for i in range(total_len):
+                if total_len == 0:
+                    break
+            
+                start = i
+                end = i+1
+                if i == total_len-1:
+                    query_json_cur = query_json_to_do[start:]
+                else:
+                    query_json_cur = query_json_to_do[start: end]
+                t = threading.Thread(target=test, args=(query_json_cur, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args))
+                t.start()
+                threads.append(t)
+            
+                
+        else:
+            for i in range(20):        
+                
+                if total_len == 0:
+                    break
+            
+                start = round(total_len/20)*i
+                end = round(total_len/20)*(i+1)
+                if i == 19:
+                    query_json_cur = query_json_to_do[start:]
+                else:
+                    query_json_cur = query_json_to_do[start: end]
+                t = threading.Thread(target=test, args=(query_json_cur, white_list, output_dir, whole_solution_dir, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, args))
+                t.start()
+                threads.append(t)
+        
+        for thread in threads:
+            thread.join()
+
+        total_json = {}
+        items = os.listdir(output_dir)
+        for item in items:
+            item_path = os.path.join(output_dir, item)
+            idx = item.split(".")[0]
+            total_json[str(idx)] = json.load(open(item_path, 'r'))
+
+        with open(total_output_file, 'w') as file:
+            json.dump(total_json, file, indent=4, ensure_ascii=False)
+    

Smurfs/inference/server.py

@@ -0,0 +1,179 @@
+from pydantic import BaseModel
+import json
+import os
+from typing import Union
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.inference.utils import standardize, change_name
+import random
+
+
+class Info(BaseModel):
+    category: str
+    tool_name: str
+    api_name: str
+    tool_input: Union[str, dict]
+    strip: str
+
+def prepare_tool_name_and_url(tools_root, info):
+    category = info.category
+    standard_category = category.replace(" ", "_").replace(",", "_").replace("/", "_")
+    while " " in standard_category or "," in standard_category:
+        standard_category = standard_category.replace(" ", "_").replace(",", "_")
+    standard_category = standard_category.replace("__", "_")
+    
+    tool_name = info.tool_name
+    api_name = change_name(standardize(info.api_name))
+    if not tool_name.endswith(f"_for_{standard_category}"):
+        tool_name = standardize(info.tool_name)
+        code_string = f"""from {tools_root}.{standard_category}.{tool_name}.api import {api_name}"""
+        tool_name += f"_for_{standard_category}"
+    else:
+        tmp_tool_name = standardize(tool_name.replace(f"_for_{standard_category}", ""))
+        code_string = f"""from {tools_root}.{standard_category}.{tmp_tool_name}.api import {api_name}"""
+    return tool_name, standard_category, api_name, code_string
+
+def process_error(response):
+    save_cache_flag = False
+    switch_flag = False
+    if "The request to the API has timed out. Please try again later, or if the issue persists" in str(response):
+        return_dict = {"error": "API temporarily not working error...", "response": response}
+
+    if "Your Client (working) ---> Gateway (working) ---> API (not working)" in str(response):
+        return_dict = {"error": "API not working error...", "response": response}
+        
+    elif "Unauthorized" in str(response) or "unauthorized" in str(response):
+        save_cache_flag = True
+        return_dict = {"error": "Unauthorized error...", "response": response}
+    
+    elif "You are not subscribed to this API." in str(response):
+        switch_flag = True
+        return_dict = {"error": "Unsubscribed error...", "response": response}
+    
+    elif "Too many requests" in str(response):
+        switch_flag = True
+        return_dict = {"error": "Too many requests error...", "response": response}
+
+    elif "You have exceeded" in str(response) or "you are being rate limited"  in str(response):
+        switch_flag = True
+        return_dict = {"error": "Rate limit error...", "response": response}
+
+    elif "Access restricted. Check credits balance or enter the correct API key." in str(response):
+        switch_flag = True
+        return_dict = {"error": "Rate limit error...", "response": response}
+    
+    elif "Oops, an error in the gateway has occurred." in str(response):
+        switch_flag = True
+        return_dict = {"error": "Gateway error...", "response": response}
+
+    elif "Blocked User. Please contact your API provider." in str(response):
+        switch_flag = True
+        return_dict = {"error": "Blocked error...", "response": response}
+    
+    elif "error" in str(response):
+        return_dict = {"error": "Message error...", "response": response}
+
+    else:
+        save_cache_flag = True
+        return_dict = {"error": "", "response": response}
+    return return_dict, save_cache_flag, switch_flag
+
+def run(toolbench_code_string, toolbench_api_name, toolbench_input_params_str):
+    # get observation
+    success_flag = False
+    switch_flag = False
+    save_cache = False
+    exec(toolbench_code_string)
+    try:
+        eval_func_str = f"{toolbench_api_name}({toolbench_input_params_str})"
+        new_func = eval(eval_func_str)
+        response, save_cache, switch_flag = process_error(new_func)
+        success_flag = True
+    except Exception as e:
+        response = {"error": f"Function executing {toolbench_code_string} error...\n{e}", "response": ""}
+        save_cache = False
+    return success_flag, switch_flag, response, save_cache
+
+
+def dict_shorten(origin: dict, schema: dict):
+    for key, value in list(origin.items()):
+        if key not in schema:
+            del origin[key]
+        else:
+            if isinstance(value, dict):
+                dict_shorten(value, schema[key]) # schema[key] should be a dict
+            elif isinstance(value, list):
+                if value:
+                    if isinstance(value[0], dict):
+                        for item in value:
+                            dict_shorten(item, schema[key][0]) # schema[key] should be a list with only one dict element
+    return origin
+
+def observation_shorten(schema_root, response_dict, category, tool_name, api_name, strip_method):
+    print(random.random())
+    if strip_method == "filter" or (strip_method == "random" and random.random() > 0.5):
+        if isinstance(response_dict["response"], dict):
+            if os.path.exists(os.path.join(schema_root, category)):
+                if os.path.exists(os.path.join(schema_root, category, tool_name+".json")):
+                    schema_dicts = json.load(open(os.path.join(schema_root, category, tool_name+".json"), "r"))
+                    api_list = schema_dicts["api_list"]
+                    schema = None
+                    for schema_dict in api_list:
+                        schema_api_name = change_name(standardize(schema_dict["name"]))
+                        if schema_api_name == api_name and len(schema_dict["schema"]) > 0:
+                            schema = schema_dict["schema"]
+                            break
+                    if schema is not None:
+                        response_dict["response"] = dict_shorten(response_dict["response"], schema)
+    return str(response_dict["response"])
+
+
+def get_rapidapi_response(input_dict: dict, api_customization: bool=False, tools_root: str="data.toolenv.tools", schema_root: str="data/toolenv/response_examples"):
+    info = Info
+    info.category = input_dict['category']
+    info.tool_name = input_dict['tool_name']
+    info.api_name = input_dict['api_name']
+    info.tool_input = input_dict['tool_input']
+    info.strip = input_dict['strip']
+    rapidapi_key = input_dict['rapidapi_key']
+
+    tool_name, standard_category, api_name, code_string = prepare_tool_name_and_url(tools_root, info)
+    tool_input = info.tool_input
+    
+    strip_method = info.strip
+    
+    try:
+        tool_input = json.loads(tool_input)
+    except Exception as e:
+        if tool_input == "":
+            tool_input = {}
+        else:
+            print(f"Can not parse tool input into json: {tool_input}")
+            response_dict = {"error": f"Tool input parse error...\n", "response": ""}
+            return response_dict
+    
+    input_params_str = ""
+    if len(tool_input) > 0:
+        for key, value in tool_input.items():
+            if isinstance(value, str):
+                input_params_str += f'{key}="{value}", '
+            else:
+                input_params_str += f'{key}={value}, '
+    if not api_customization:
+        input_params_str += f"toolbench_rapidapi_key='{rapidapi_key}'"
+    success_flag, switch_flag, response_dict, save_cache = run(code_string, api_name, input_params_str)
+    observation = observation_shorten(schema_root, response_dict, standard_category, tool_name.replace(f"_for_{standard_category}", ""), api_name, strip_method)
+    result = str(observation)[:2048]
+    return {"error": response_dict['error'], "response": result}
+
+
+if __name__ == "__main__":
+    result = get_rapidapi_response({
+        "category": "Social",
+        "tool_name": "olato_quotes",
+        "api_name": "love_quote",
+        "tool_input": '{}',
+        "strip": "filter",
+        "rapidapi_key": ""
+    })
+    print(result)

Smurfs/inference/smurfs_worker.py

@@ -0,0 +1,1040 @@
+# — coding: utf-8 –
+import json
+import sys
+import argparse
+import time
+import requests
+import os
+from Smurfs.inference.utils import change_name, standardize, get_white_list, get_answer_log, get_observation_log, build_tree, get_answer_details, test_sets
+from tqdm import tqdm
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+from Smurfs.model.vllm_model.vllm_model import vllm_Model
+from Smurfs.inference.server import get_rapidapi_response
+import threading
+from Smurfs.agents.answer_agent.answer import answer_agent
+from Smurfs.agents.executor_agent.executor import executor_agent
+from Smurfs.agents.planning_agent.planner import planning_agent
+from Smurfs.agents.verifier_agent.verifier import verifier_agent
+from termcolor import colored  
+class smurfs_worker:
+    def __init__(self, available_tools, tool_env, llm, method_name, test_set, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent):
+        #available_tools的格式形如toolbench里面的api_list里的格式，只需要api_name
+        #tool_env是一个工具函数里用来存储工具代码的py文件中的所有函数的字典，key为函数名，value是函数对象
+        self.available_tools = available_tools
+        
+        self.output_dir = f"data/{method_name}/{test_set}/answer"
+
+        self.whole_solution_dir = f"data/{method_name}/{test_set}/whole"
+
+        self.logger_dir = f"data/{method_name}/{test_set}/agent_log"
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+        if not os.path.exists(self.whole_solution_dir):
+            os.makedirs(self.whole_solution_dir)
+        
+        if not os.path.exists(self.logger_dir):
+            os.makedirs(self.logger_dir)
+
+        self.Answer_Agent = Answer_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Executor_Agent = Executor_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Planning_Agent = Planning_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Verifier_Agent = Verifier_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.tool_env = tool_env
+    
+    def inference(self, query, relevant_APIs, subtask, query_id, max_step=3):
+        # tool_check_num, reason = self.Answer_Agent.run(question=query, task="tool_check", query_id=query_id)
+        # #direct answer
+        # if tool_check_num == 1:
+        #     # input_dic = {"task": query}
+        #     answer = self.Answer_Agent.run(question=query, task="direct", query_id=query_id)
+        #     previous_log = [{"thought": reason, "action": "", "action_input": "", "observation": "", "answer": answer, "tool": "","id": 0}]
+        #     history_log = [{"thought": reason, "action": "", "action_input": "", "observation": "", "answer": answer, "previous_id": -1, "id": 0}]
+        #     return answer, previous_log, 0, history_log
+
+        previous_log = []
+        history_log = []
+        tool_used_dic = {}
+        relevant_APIs_ids = []
+        for idx in relevant_APIs:
+            ele = relevant_APIs[idx]
+            relevant_APIs_ids.append(str(ele["ID"]))
+        restart_time = 0
+        step_num = 0
+        hint = "Beginnig of the agent. No hint yet"
+        retry_tool_id = 0
+        retry_parameter = 0
+        re_time = 0
+        subtask_id = 0
+        restart = 0
+        while True:
+            if step_num >= max_step:
+                print("\n\nReach steps limits, return answers!\n\n")
+                answer_log = get_answer_log(history_log)
+                answer = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                return answer, previous_log, re_time, history_log
+            
+            if step_num not in tool_used_dic.keys():
+                tool_used_dic[step_num] = []
+
+            tool_used = tool_used_dic[step_num]
+
+            tool_list = []
+            for idx in relevant_APIs:
+                ele = relevant_APIs[idx]
+                ID = str(ele['ID'])
+                if ID in tool_used:
+                    continue
+                des = ele['description']
+                name = ele["tool_name"]
+                tool_list.append({"ID": ID, "tool_name": name, "description": des})
+
+            if len(tool_list) == 0:
+                if len(previous_log) == 0:
+                    answer_log = get_answer_log(history_log)
+                    partial_answer = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                    answer = f"Sorry, I can't answer this question accurately using the existing tools. A partial answer is: {partial_answer}"
+                    return answer, previous_log, re_time, history_log
+                else:
+                    delete_log = previous_log.pop()
+                    tool_used_dic[step_num] = []
+                    step_num -= 1
+                    tool_used_dic[step_num].append(delete_log["tool"])
+                    restart_time += 1
+                    re_time += 1
+                    continue
+
+            current_log = {"thought": "", "action": "", "action_input": {}, "observation": "", "answer": "", "tool": "","id": subtask_id}
+
+            answer_log = get_answer_log(previous_log)
+        
+            if retry_tool_id == 4:
+                tool_id = tool_list[0]["ID"]
+                tool_list = tool_list[0]
+                thought = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+
+            else:
+                thought = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+                tool_id = self.Executor_Agent.run(question=subtask, tool_list=tool_list, thought=thought, query_id=query_id, task="tool")
+            
+            try:
+                tool_id = int(tool_id)
+                tool_id = str(tool_id)
+                if tool_id not in relevant_APIs_ids:
+                    re_time += 1
+                    retry_tool_id += 1
+                    print("Tool ID wrong! Generate tool_id that do not exist!")
+                    continue
+                tool_des_json = relevant_APIs[str(tool_id)]
+                retry_tool_id = 0
+            except:
+                retry_tool_id += 1
+                print("Tool ID wrong! Generate tool_id that do not exist!")
+                continue
+            
+            # tool_name_list = tool_des_json["tool_name"].split(":")
+            # category_name = tool_name_list[0]
+            # tool_name = tool_name_list[1]
+            api_name = tool_des_json["tool_name"]
+            API_doc = tool_des_json
+            
+            while True:
+                try:
+                    parameters = {}
+
+                    if retry_parameter == 4:
+                        restart = 1
+                        retry_parameter = 0
+                        print("No Para! Restart!")
+                        break
+                    
+                    parameter = self.Executor_Agent.run(api_dic=API_doc, question=query, previous_log=answer_log, thought=thought, query_id=query_id, task="parameter")
+                    if parameter == -1:
+                        retry_parameter += 1
+                        re_time += 1
+                        continue
+
+                    if parameter == {}:
+                        retry_parameter = 0
+                        parameters = {}
+                        break
+
+                    for key in parameter:
+                        value = parameter[key]
+                        key = change_name(key)
+                        parameters[key] = value
+
+                    retry_parameter = 0
+                    break
+
+                except:
+                    if retry_parameter == 4:
+                        parameters = {}
+                        retry_parameter = 0
+                        restart = 1
+                        break
+                    retry_parameter += 1
+                    print("parameter generation fails, try again!")
+                    re_time += 1
+                    continue
+        
+            
+            # api_name = change_name(standardize(api_name))
+
+            if restart != 1:
+                try:
+                    observation = self.Call_function(api_name, parameters)
+                except:
+                    observation = -1
+                    
+                if observation == -1:
+                    restart = 1
+                    observation = str({"error": "", "response": "call API fails"})
+
+            if restart == 1:
+                tool_used_dic[step_num].append(str(tool_id))
+                print('****Try Again For This Step****')
+                re_time += 1
+                restart = 0
+                continue
+            
+
+            if len(previous_log) != 0:
+                previous_id = previous_log[-1]["id"]
+            else:
+                previous_id = -1
+
+            current_log["tool"] = str(tool_id)
+            current_log["thought"] = thought
+            current_log["action"] = api_name
+            current_log["action_input"] = parameters
+            current_log["observation"] = observation
+            previous_log.append(current_log)
+            print("##########Tool Response##########")
+            print(f"{observation}\n")
+            observation_log = get_observation_log(previous_log)
+
+            answer = self.Answer_Agent.run(question=subtask, call_result=observation_log, query_id=query_id, task="answer")
+
+            previous_log[-1]["answer"] = answer
+
+            history_log_ele = {"thought": thought, "action": api_name, "action_input": parameters, "observation": observation, "answer": answer, "previous_id": previous_id, "id": subtask_id}
+            history_log.append(history_log_ele)
+            subtask_id += 1
+
+            speak, status = self.Verifier_Agent.run(question=subtask, answer=answer, query_id=query_id)
+            if speak == -1 and status == -1:
+                step_num += 1
+                continue
+
+            try:
+                if int(status) == 0:
+                    hint = speak
+                    step_num += 1
+                    continue
+            except:
+                    step_num += 1
+                    continue
+                
+            else:
+                return answer, previous_log, re_time, history_log
+
+    def decompose_inference(self, query, query_id):
+        while True:
+            subtasks = self.Planning_Agent.run(question=query, query_id=query_id)
+            if subtasks == -1:
+                continue
+            break
+        task_log = ""
+        history_log = []
+        previous_log_totals = []
+        re_time_total = 0
+        # print(subtasks)
+        relevant_API_list = {}
+        tool_id = 0
+        for api in self.available_tools:
+            tool_name = api["api_name"]
+            ele = {"ID": tool_id, "tool_name": tool_name, "description": api["api_description"], "required_parameters": api["required_parameters"], "optional_parameters": api["optional_parameters"]}
+            relevant_API_list[str(tool_id)] = ele
+            tool_id += 1
+
+        for subtask in subtasks:
+            task_log += f"question: {subtask}\n"
+            answer, previous_log, re_time, previous_log_total = self.inference(task_log, relevant_API_list, subtask, query_id)
+            previous_log_totals.append(previous_log_total)
+            # print(answer)
+            history_log += previous_log
+            re_time_total += re_time
+            task_log += f"answer: {answer}\n"
+        final_answer = self.Answer_Agent.run(question=query, previous_log=task_log, task="final", query_id=query_id)
+        return final_answer, history_log, task_log, re_time_total, previous_log_totals 
+
+    def run(self, input, query_id):
+        # result = {}
+        # st = time.time()
+        final_answer, previous_log, task_log,re_time, previous_log_totals = self.decompose_inference(input, query_id)
+        answer_details, total_steps = get_answer_details(final_answer, previous_log)
+        solution_tree, solution_total_steps = build_tree(previous_log_totals, task_log)
+        output_file_ele = {
+            "query": input,
+            "restart_time": re_time,
+            "answer": {
+                "method": "decompose_dfs",
+                "total_steps": total_steps,
+                "final_answer": final_answer,
+                "answer_details": answer_details
+            }
+        }
+                    
+        solution_file_ele = {
+            "query": input,
+            "total_steps": solution_total_steps,
+            "task_log": task_log,
+            "final_answer": final_answer,
+            "answer_path": answer_details,
+            "total_path": solution_tree
+        }
+        return final_answer, output_file_ele, solution_file_ele
+
+    def save_solution(self, output_file_ele, solution_file_ele, idx):
+        file_name = f"{idx}.json"
+        output_file = os.path.join(self.output_dir, file_name)
+        whole_solution_file = os.path.join(self.whole_solution_dir, file_name)
+        with open(output_file, "w") as file:
+            json.dump(output_file_ele, file, ensure_ascii=False, indent=4)
+        with open(whole_solution_file, "w") as file:
+            json.dump(solution_file_ele, file, ensure_ascii=False, indent=4)
+
+    def Call_function(self, tool_name, args):
+        try:
+            print(tool_name)
+            func = self.tool_env[tool_name]
+            observation = func(**args)
+            return observation
+        except Exception as e:
+            print(e)
+            print(f"Call function fails")
+            with open('wrong_log.json', 'a+', encoding='utf-8') as f:
+                line = json.dumps({
+                    "id": 0,
+                    "parameters": args,
+                    "tool": tool_name,
+                    "wrong": str(e)
+                }, ensure_ascii=False)
+                f.write(line + '\n')
+            return -1
+    
+class smurfs_hotpot_worker:
+    def __init__(self, available_tools, tool_env, llm, method_name, test_set, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent):
+        #available_tools的格式形如toolbench里面的api_list里的格式，只需要api_name
+        #tool_env是一个工具函数里用来存储工具代码的py文件中的所有函数的字典，key为函数名，value是函数对象
+        self.available_tools = available_tools
+        
+        self.output_dir = f"data/{method_name}/{test_set}/answer"
+
+        self.whole_solution_dir = f"data/{method_name}/{test_set}/whole"
+
+        self.logger_dir = f"data/{method_name}/{test_set}/agent_log"
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+        if not os.path.exists(self.whole_solution_dir):
+            os.makedirs(self.whole_solution_dir)
+        
+        if not os.path.exists(self.logger_dir):
+            os.makedirs(self.logger_dir)
+
+        self.Answer_Agent = Answer_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Executor_Agent = Executor_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Planning_Agent = Planning_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Verifier_Agent = Verifier_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.tool_class = tool_env
+        self.tool_env = {}
+    
+    def inference(self, query, relevant_APIs, subtask, query_id, max_step=3):
+        # tool_check_num = self.Answer_Agent.run(question=query, task="tool_check", query_id=query_id)
+        # #direct answer
+        # if tool_check_num == 1:
+        #     input_dic = {"task": query}
+        #     answer = self.Answer_Agent.run(input_dic)
+        #     return answer, answer, None, None
+
+        previous_log = []
+        history_log = []
+        tool_used_dic = {}
+        relevant_APIs_ids = []
+        for idx in relevant_APIs:
+            ele = relevant_APIs[idx]
+            relevant_APIs_ids.append(str(ele["ID"]))
+        restart_time = 0
+        step_num = 0
+        hint = "Beginnig of the agent. No hint yet"
+        retry_tool_id = 0
+        retry_parameter = 0
+        re_time = 0
+        subtask_id = 0
+        restart = 0
+        while True:
+            if step_num >= max_step:
+                print("\n\nReach steps limits, return answers!\n\n")
+                answer_log = get_answer_log(history_log)
+                answer = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                return answer, previous_log, re_time, history_log
+            
+            if step_num not in tool_used_dic.keys():
+                tool_used_dic[step_num] = []
+
+            tool_used = tool_used_dic[step_num]
+
+            tool_list = []
+            for idx in relevant_APIs:
+                ele = relevant_APIs[idx]
+                ID = str(ele['ID'])
+                if ID in tool_used:
+                    continue
+                des = ele['description']
+                name = ele["tool_name"]
+                tool_list.append({"ID": ID, "tool_name": name, "description": des})
+
+            if len(tool_list) == 0:
+                if len(previous_log) == 0:
+                    answer_log = get_answer_log(history_log)
+                    partial_answer = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                    answer = f"Sorry, I can't answer this question accurately using the existing tools. A partial answer is: {partial_answer}"
+                    return answer, previous_log, re_time, history_log
+                else:
+                    delete_log = previous_log.pop()
+                    tool_used_dic[step_num] = []
+                    step_num -= 1
+                    tool_used_dic[step_num].append(delete_log["tool"])
+                    restart_time += 1
+                    re_time += 1
+                    continue
+
+            current_log = {"thought": "", "action": "", "action_input": {}, "observation": "", "answer": "", "tool": "","id": subtask_id}
+
+            answer_log = get_answer_log(previous_log)
+        
+            if retry_tool_id == 4:
+                tool_id = tool_list[0]["ID"]
+                tool_list = tool_list[0]
+                thought = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+
+            else:
+                thought = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+                tool_id = self.Executor_Agent.run(question=subtask, tool_list=tool_list, thought=thought, query_id=query_id, task="tool")
+            
+            try:
+                tool_id = int(tool_id)
+                tool_id = str(tool_id)
+                if tool_id not in relevant_APIs_ids:
+                    re_time += 1
+                    retry_tool_id += 1
+                    print("Tool ID wrong! Generate tool_id that do not exist!")
+                    continue
+                tool_des_json = relevant_APIs[str(tool_id)]
+                retry_tool_id = 0
+            except:
+                retry_tool_id += 1
+                print("Tool ID wrong! Generate tool_id that do not exist!")
+                continue
+            
+            # tool_name_list = tool_des_json["tool_name"].split(":")
+            # category_name = tool_name_list[0]
+            # tool_name = tool_name_list[1]
+            api_name = tool_des_json["tool_name"]
+            API_doc = tool_des_json
+            
+            while True:
+                try:
+                    parameters = {}
+
+                    if retry_parameter == 4:
+                        restart = 1
+                        retry_parameter = 0
+                        print("No Para! Restart!")
+                        break
+                    
+                    parameter = self.Executor_Agent.run(api_dic=API_doc, question=query, previous_log=answer_log, thought=thought, query_id=query_id, task="parameter")
+                    if parameter == -1:
+                        retry_parameter += 1
+                        re_time += 1
+                        continue
+
+                    if parameter == {}:
+                        retry_parameter = 0
+                        parameters = {}
+                        break
+
+                    for key in parameter:
+                        value = parameter[key]
+                        key = change_name(key)
+                        parameters[key] = value
+
+                    retry_parameter = 0
+                    break
+
+                except:
+                    if retry_parameter == 4:
+                        parameters = {}
+                        retry_parameter = 0
+                        restart = 1
+                        break
+                    retry_parameter += 1
+                    print("parameter generation fails, try again!")
+                    re_time += 1
+                    continue
+        
+            
+            # api_name = change_name(standardize(api_name))
+
+            if restart != 1:
+                try:
+                    observation = self.Call_function(api_name, parameters)
+                except:
+                    observation = -1
+                    
+                if observation == -1:
+                    restart = 1
+                    observation = str({"error": "", "response": "call API fails"})
+
+            if restart == 1:
+                tool_used_dic[step_num].append(str(tool_id))
+                print('****Try Again For This Step****')
+                re_time += 1
+                restart = 0
+                continue
+            
+
+            if len(previous_log) != 0:
+                previous_id = previous_log[-1]["id"]
+            else:
+                previous_id = -1
+
+            current_log["tool"] = str(tool_id)
+            current_log["thought"] = thought
+            current_log["action"] = api_name
+            current_log["action_input"] = parameters
+            current_log["observation"] = observation
+            print("##########Tool Response##########")
+            print(f"{observation}\n")
+            previous_log.append(current_log)
+
+            observation_log = get_observation_log(previous_log)
+
+            answer = self.Answer_Agent.run(question=subtask, call_result=observation_log, query_id=query_id, task="answer")
+
+            previous_log[-1]["answer"] = answer
+
+            history_log_ele = {"thought": thought, "action": api_name, "action_input": parameters, "observation": observation, "answer": answer, "previous_id": previous_id, "id": subtask_id}
+            history_log.append(history_log_ele)
+            subtask_id += 1
+
+            speak, status = self.Verifier_Agent.run(question=subtask, answer=answer, query_id=query_id)
+            if speak == -1 and status == -1:
+                step_num += 1
+                continue
+
+            try:
+                if int(status) == 0:
+                    hint = speak
+                    step_num += 1
+                    continue
+            except:
+                    step_num += 1
+                    continue
+                
+            else:
+                return answer, previous_log, re_time, history_log
+
+    def decompose_inference(self, query, query_id):
+        while True:
+            subtasks = self.Planning_Agent.run(question=query, query_id=query_id)
+            if subtasks == -1:
+                continue
+            break
+        task_log = ""
+        history_log = []
+        previous_log_totals = []
+        re_time_total = 0
+        # print(subtasks)
+        relevant_API_list = {}
+        tool_id = 0
+        for api in self.available_tools:
+            tool_name = api["api_name"]
+            ele = {"ID": tool_id, "tool_name": tool_name, "description": api["api_description"], "required_parameters": api["required_parameters"], "optional_parameters": api["optional_parameters"]}
+            relevant_API_list[str(tool_id)] = ele
+            tool_id += 1
+
+        for subtask in subtasks:
+            task_log += f"question: {subtask}\n"
+            answer, previous_log, re_time, previous_log_total = self.inference(task_log, relevant_API_list, subtask, query_id)
+            previous_log_totals.append(previous_log_total)
+            # print(answer)
+            history_log += previous_log
+            re_time_total += re_time
+            task_log += f"answer: {answer}\n"
+        final_answer = self.Answer_Agent.run(question=query, previous_log=task_log, task="final", query_id=query_id)
+        return final_answer, history_log, task_log, re_time_total, previous_log_totals 
+
+    def run(self, input, query_id):
+        # result = {}
+        # st = time.time()
+        HPEnv = self.tool_class()
+        self.tool_env = {
+            "BingSearch": HPEnv.BingSearch,
+            "Retrieve": HPEnv.Retrieve,
+            "Lookup": HPEnv.Lookup
+            }
+        final_answer, previous_log, task_log,re_time, previous_log_totals = self.decompose_inference(input, query_id)
+        answer_details, total_steps = get_answer_details(final_answer, previous_log)
+        solution_tree, solution_total_steps = build_tree(previous_log_totals, task_log)
+        output_file_ele = {
+            "query": input,
+            "restart_time": re_time,
+            "answer": {
+                "method": "decompose_dfs",
+                "total_steps": total_steps,
+                "final_answer": final_answer,
+                "answer_details": answer_details
+            }
+        }
+                    
+        solution_file_ele = {
+            "query": input,
+            "total_steps": solution_total_steps,
+            "task_log": task_log,
+            "final_answer": final_answer,
+            "answer_path": answer_details,
+            "total_path": solution_tree
+        }
+        return final_answer, output_file_ele, solution_file_ele
+
+    def save_solution(self, output_file_ele, solution_file_ele, idx):
+        file_name = f"{idx}.json"
+        output_file = os.path.join(self.output_dir, file_name)
+        whole_solution_file = os.path.join(self.whole_solution_dir, file_name)
+        with open(output_file, "w") as file:
+            json.dump(output_file_ele, file, ensure_ascii=False, indent=4)
+        with open(whole_solution_file, "w") as file:
+            json.dump(solution_file_ele, file, ensure_ascii=False, indent=4)
+
+    def Call_function(self, tool_name, args):
+        try:
+            print(tool_name)
+            func = self.tool_env[tool_name]
+            observation = func(**args)
+            return observation
+        except Exception as e:
+            print(e)
+            print(f"Call function fails")
+            with open('wrong_log.json', 'a+', encoding='utf-8') as f:
+                line = json.dumps({
+                    "id": 0,
+                    "parameters": args,
+                    "tool": tool_name,
+                    "wrong": str(e)
+                }, ensure_ascii=False)
+                f.write(line + '\n')
+            return -1
+
+
+class stream_smurfs_worker:
+    def __init__(self, available_tools, tool_env, llm, method_name, test_set, Answer_Agent, Executor_Agent, Planning_Agent, Verifier_Agent, OPENAI_API_KEY, BING_SUBSCRIPT_KEY, WOLFRAMALPH_APP_ID, WEATHER_API_KEYS):
+        #available_tools的格式形如toolbench里面的api_list里的格式，只需要api_name
+        #tool_env是一个工具函数里用来存储工具代码的py文件中的所有函数的字典，key为函数名，value是函数对象
+        self.OPENAI_API_KEY = OPENAI_API_KEY
+        self.BING_SUBSCRIPT_KEY = BING_SUBSCRIPT_KEY
+        self.WOLFRAMALPH_APP_ID = WOLFRAMALPH_APP_ID
+        self.WEATHER_API_KEYS = WEATHER_API_KEYS
+        #print(self.BING_SUBSCRIPT_KEY)
+        self.available_tools = available_tools
+        
+        self.output_dir = f"data/{method_name}/{test_set}/answer"
+
+        self.whole_solution_dir = f"data/{method_name}/{test_set}/whole"
+
+        self.logger_dir = f"data/{method_name}/{test_set}/agent_log"
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+        if not os.path.exists(self.whole_solution_dir):
+            os.makedirs(self.whole_solution_dir)
+        
+        if not os.path.exists(self.logger_dir):
+            os.makedirs(self.logger_dir)
+
+        self.Answer_Agent = Answer_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Executor_Agent = Executor_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Planning_Agent = Planning_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.Verifier_Agent = Verifier_Agent(llm=llm, logger_dir=self.logger_dir)
+        self.tool_env = tool_env
+    
+    # def colorful_html(self, task, content, name):
+    #     """print out message in different color"""
+    #     role_to_color = {
+    #     "Answer Agent": "red",
+    #     "Executor Agent": "green",
+    #     "Planning Agent": "blue",
+    #     "Verifier Agent": "yellow",
+    #     }
+    #     color = role_to_color[name]
+    #     html_text = f"<span style='color: {color}'>##########{task}##########<br>{content}<br></span>"
+    #     # print(colored(f"##########{task}##########\n{content}\n", role_to_color[name]))
+    #     return html_text
+
+    def colorful_html(self, task, content, name):
+        """print out message in different color"""
+        role_to_color = {
+        "Answer Agent": "red",
+        "Executor Agent": "green",
+        "Planning Agent": "blue",
+        "Verifier Agent": "yellow",
+        }
+        color = role_to_color[name]
+        if task != "Final Answer Generation":
+            html_text = f"""<details><summary>{task}<br></summary>{content}<br></details>"""
+        else:
+            html_text = content
+        # html_text = f"<span style='color: {color}'>##########{task}##########<br>{content}<br></span>"
+        # print(colored(f"##########{task}##########\n{content}\n", role_to_color[name]))
+        return html_text
+
+    def inference(self, query, relevant_APIs, subtask, query_id, max_step=3):
+        # tool_check_num, reason = self.Answer_Agent.run(question=query, task="tool_check", query_id=query_id)
+        # #direct answer
+        # if tool_check_num == 1:
+        #     # input_dic = {"task": query}
+        #     answer = self.Answer_Agent.run(question=query, task="direct", query_id=query_id)
+        #     previous_log = [{"thought": reason, "action": "", "action_input": "", "observation": "", "answer": answer, "tool": "","id": 0}]
+        #     history_log = [{"thought": reason, "action": "", "action_input": "", "observation": "", "answer": answer, "previous_id": -1, "id": 0}]
+        #     return answer, previous_log, 0, history_log
+
+        previous_log = []
+        history_log = []
+        tool_used_dic = {}
+        relevant_APIs_ids = []
+        for idx in relevant_APIs:
+            ele = relevant_APIs[idx]
+            relevant_APIs_ids.append(str(ele["ID"]))
+        restart_time = 0
+        step_num = 0
+        hint = "Beginnig of the agent. No hint yet"
+        retry_tool_id = 0
+        retry_parameter = 0
+        re_time = 0
+        subtask_id = 0
+        restart = 0
+        while True:
+            if step_num >= max_step:
+                yield("<br><br>Reach steps limits, return answers!<br><br>")
+                answer_log = get_answer_log(history_log)
+                answer, task, agent_name, result = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                yield self.colorful_html(task, result, agent_name)
+                yield answer, previous_log, re_time, history_log
+                yield "stop"
+            
+            if step_num not in tool_used_dic.keys():
+                tool_used_dic[step_num] = []
+
+            tool_used = tool_used_dic[step_num]
+
+            tool_list = []
+            for idx in relevant_APIs:
+                ele = relevant_APIs[idx]
+                ID = str(ele['ID'])
+                if ID in tool_used:
+                    continue
+                des = ele['description']
+                name = ele["tool_name"]
+                tool_list.append({"ID": ID, "tool_name": name, "description": des})
+
+            if len(tool_list) == 0:
+                if len(previous_log) == 0:
+                    answer_log = get_answer_log(history_log)
+                    partial_answer, task, agent_name, result = self.Answer_Agent.run(question=query, previous_log=answer_log, task="final", query_id=query_id)
+                    answer = f"Sorry, I can't answer this question accurately using the existing tools. A partial answer is: {partial_answer}"
+                    yield self.colorful_html(task, result, agent_name)
+                    yield answer, previous_log, re_time, history_log
+                    yield "stop"
+                else:
+                    delete_log = previous_log.pop()
+                    tool_used_dic[step_num] = []
+                    step_num -= 1
+                    tool_used_dic[step_num].append(delete_log["tool"])
+                    restart_time += 1
+                    re_time += 1
+                    continue
+
+            current_log = {"thought": "", "action": "", "action_input": {}, "observation": "", "answer": "", "tool": "","id": subtask_id}
+
+            answer_log = get_answer_log(previous_log)
+        
+            if retry_tool_id == 4:
+                tool_id = tool_list[0]["ID"]
+                tool_list = tool_list[0]
+                thought, task, agent_name, result = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+                yield self.colorful_html(task, result, agent_name)
+            else:
+                thought, task, agent_name, result = self.Executor_Agent.run(question=query, tool_list=tool_list, previous_log=answer_log, hint=hint, query_id=query_id, task="thought")
+                yield self.colorful_html(task, result, agent_name)
+                tool_id, task, agent_name, result = self.Executor_Agent.run(question=subtask, tool_list=tool_list, thought=thought, query_id=query_id, task="tool")
+                yield self.colorful_html(task, result, agent_name)
+            
+            try:
+                tool_id = int(tool_id)
+                tool_id = str(tool_id)
+                if tool_id not in relevant_APIs_ids:
+                    re_time += 1
+                    retry_tool_id += 1
+                    yield("Tool ID wrong! Generate tool_id that do not exist!<br>")
+                    continue
+                tool_des_json = relevant_APIs[str(tool_id)]
+                retry_tool_id = 0
+            except:
+                retry_tool_id += 1
+                yield("Tool ID wrong! Generate tool_id that do not exist!<br>")
+                continue
+            
+            # tool_name_list = tool_des_json["tool_name"].split(":")
+            # category_name = tool_name_list[0]
+            # tool_name = tool_name_list[1]
+            api_name = tool_des_json["tool_name"]
+            API_doc = tool_des_json
+            
+            while True:
+                try:
+                    parameters = {}
+
+                    if retry_parameter == 4:
+                        restart = 1
+                        retry_parameter = 0
+                        yield("No Para! Restart!<br>")
+                        break
+                    
+                    parameter, task, agent_name, result = self.Executor_Agent.run(api_dic=API_doc, question=query, previous_log=answer_log, thought=thought, query_id=query_id, task="parameter")
+                    yield self.colorful_html(task, result, agent_name)
+
+                    if parameter == -1:
+                        retry_parameter += 1
+                        re_time += 1
+                        continue
+
+                    if parameter == {}:
+                        retry_parameter = 0
+                        parameters = {}
+                        break
+
+                    for key in parameter:
+                        value = parameter[key]
+                        key = change_name(key)
+                        parameters[key] = value
+
+                    retry_parameter = 0
+                    break
+
+                except:
+                    if retry_parameter == 4:
+                        parameters = {}
+                        retry_parameter = 0
+                        restart = 1
+                        break
+                    retry_parameter += 1
+                    yield("parameter generation fails, try again!<br>")
+                    re_time += 1
+                    continue
+        
+            
+            # api_name = change_name(standardize(api_name))
+
+            if restart != 1:
+                try:
+                    observation = self.Call_function(api_name, parameters)
+                except:
+                    observation = -1
+                    
+                if observation == -1:
+                    restart = 1
+                    observation = str({"error": "", "response": "call API fails"})
+
+            if restart == 1:
+                tool_used_dic[step_num].append(str(tool_id))
+                yield('****Try Again For This Step****<br>')
+                re_time += 1
+                restart = 0
+                continue
+            
+
+            if len(previous_log) != 0:
+                previous_id = previous_log[-1]["id"]
+            else:
+                previous_id = -1
+
+            current_log["tool"] = str(tool_id)
+            current_log["thought"] = thought
+            current_log["action"] = api_name
+            current_log["action_input"] = parameters
+            current_log["observation"] = observation
+            previous_log.append(current_log)
+            yield(f"<details><summary>Tool Response</summary>{observation}<br></details>")
+            # print(f"{observation}\n")
+            observation_log = get_observation_log(previous_log)
+
+            answer, task, agent_name, result = self.Answer_Agent.run(question=subtask, call_result=observation_log, query_id=query_id, task="answer")
+            yield self.colorful_html(task, result, agent_name)
+            previous_log[-1]["answer"] = answer
+
+            history_log_ele = {"thought": thought, "action": api_name, "action_input": parameters, "observation": observation, "answer": answer, "previous_id": previous_id, "id": subtask_id}
+            history_log.append(history_log_ele)
+            subtask_id += 1
+
+            speak, status, task, agent_name, result = self.Verifier_Agent.run(question=subtask, answer=answer, query_id=query_id)
+            yield self.colorful_html(task, result, agent_name)
+
+            if speak == -1 and status == -1:
+                step_num += 1
+                continue
+
+            try:
+                if int(status) == 0:
+                    hint = speak
+                    step_num += 1
+                    continue
+            except:
+                    step_num += 1
+                    continue
+                
+            else:
+                yield answer, previous_log, re_time, history_log
+                yield "stop"
+
+    def run(self, query, query_id):
+        output = ""
+        count = 0
+        while True:
+            subtasks, task, agent_name, result = self.Planning_Agent.run(question=query, query_id=query_id)
+            if subtasks == -1:
+                count += 1
+                if count >= 1:
+                    yield "Task Decompose Fails! Your OpenAI Key can not function correctly."
+                    raise RuntimeError
+                continue
+            
+            break
+        output += self.colorful_html(task, result, agent_name)
+        yield output
+        task_log = ""
+        history_log = []
+        previous_log_totals = []
+        re_time_total = 0
+        # print(subtasks)
+        relevant_API_list = {}
+        tool_id = 0
+        for api in self.available_tools:
+            tool_name = api["api_name"]
+            ele = {"ID": tool_id, "tool_name": tool_name, "description": api["api_description"], "required_parameters": api["required_parameters"], "optional_parameters": api["optional_parameters"]}
+            relevant_API_list[str(tool_id)] = ele
+            tool_id += 1
+
+        for subtask in subtasks:
+            sub_output = ""
+            output_ele = "<details><summary>subtask: {subtask}</summary>{sub_output}</details>"
+            task_log += f"question: {subtask}\n"
+            inference_generator = self.inference(task_log, relevant_API_list, subtask, query_id)
+            # old = None
+            while True:
+                try:
+                    result = next(inference_generator)
+                    # if result == "stop":
+                    #     break
+                    if isinstance(result, str):
+                        sub_output += result
+                        sub_out = output_ele.format(subtask=subtask, sub_output=sub_output)
+                        output_ins = output + sub_out
+                        yield output_ins
+                    else:
+                        break
+                except StopIteration:
+                    break
+            output += sub_out
+            answer, previous_log, re_time, previous_log_total = result
+            #answer, previous_log, re_time, previous_log_total = self.inference(task_log, relevant_API_list, subtask, query_id)
+            previous_log_totals.append(previous_log_total)
+            # print(answer)
+            history_log += previous_log
+            re_time_total += re_time
+            task_log += f"answer: {answer}\n"
+        final_answer, task, agent_name, result = self.Answer_Agent.run(question=query, previous_log=task_log, task="final", query_id=query_id)
+        output += self.colorful_html(task, result, agent_name)
+        yield output
+        # return final_answer, history_log, task_log, re_time_total, previous_log_totals 
+
+    # def run(self, input, query_id):
+    #     # result = {}
+    #     # st = time.time()
+    #     final_answer, previous_log, task_log,re_time, previous_log_totals = self.decompose_inference(input, query_id)
+    #     answer_details, total_steps = get_answer_details(final_answer, previous_log)
+    #     solution_tree, solution_total_steps = build_tree(previous_log_totals, task_log)
+    #     output_file_ele = {
+    #         "query": input,
+    #         "restart_time": re_time,
+    #         "answer": {
+    #             "method": "decompose_dfs",
+    #             "total_steps": total_steps,
+    #             "final_answer": final_answer,
+    #             "answer_details": answer_details
+    #         }
+    #     }
+                    
+    #     solution_file_ele = {
+    #         "query": input,
+    #         "total_steps": solution_total_steps,
+    #         "task_log": task_log,
+    #         "final_answer": final_answer,
+    #         "answer_path": answer_details,
+    #         "total_path": solution_tree
+    #     }
+    #     return final_answer, output_file_ele, solution_file_ele
+
+    def save_solution(self, output_file_ele, solution_file_ele, idx):
+        file_name = f"{idx}.json"
+        output_file = os.path.join(self.output_dir, file_name)
+        whole_solution_file = os.path.join(self.whole_solution_dir, file_name)
+        with open(output_file, "w") as file:
+            json.dump(output_file_ele, file, ensure_ascii=False, indent=4)
+        with open(whole_solution_file, "w") as file:
+            json.dump(solution_file_ele, file, ensure_ascii=False, indent=4)
+
+    def Call_function(self, tool_name, args):
+        try:
+            print(tool_name)
+            # print(self.BING_SUBSCRIPT_KEY)
+            if tool_name == "bing_search" or tool_name == "BingSearch":
+                args["key"] = self.BING_SUBSCRIPT_KEY
+            if tool_name == "forecast_weather" or tool_name == "get_weather_today":
+                args["KEY"] = self.WEATHER_API_KEYS
+            if tool_name == "getWolframAlphaResults":
+                args["APPID"] = self.WOLFRAMALPH_APP_ID
+            
+            print(args)
+            func = self.tool_env[tool_name]
+            observation = func(**args)
+            return observation
+        except Exception as e:
+            print(e)
+            print(f"Call function fails")
+            with open('wrong_log.json', 'a+', encoding='utf-8') as f:
+                line = json.dumps({
+                    "id": 0,
+                    "parameters": args,
+                    "tool": tool_name,
+                    "wrong": str(e)
+                }, ensure_ascii=False)
+                f.write(line + '\n')
+            return -1
+  
+
+        

Smurfs/inference/utils.py

@@ -0,0 +1,356 @@
+# — coding: utf-8 –
+import json
+import re
+import os
+from tqdm import tqdm
+
+def get_white_list(tool_root_dir):
+    # print(tool_root_dir)
+    white_list_dir = os.path.join(tool_root_dir)
+    white_list = {}
+    for cate in tqdm(os.listdir(white_list_dir)):
+        if not os.path.isdir(os.path.join(white_list_dir,cate)):
+            continue
+        for file in os.listdir(os.path.join(white_list_dir,cate)):
+            if not file.endswith(".json"):
+                continue
+            standard_tool_name = file.split(".")[0]
+            # print(standard_tool_name)
+            with open(os.path.join(white_list_dir,cate,file)) as reader:
+                js_data = json.load(reader)
+            origin_tool_name = js_data["tool_name"]
+            white_list[standardize(origin_tool_name)] = {"description": js_data["tool_description"], "standard_tool_name": standard_tool_name}
+    return white_list
+
+def build_index(base_path):
+    index = {}
+    for root, dirs, files in os.walk(base_path):
+        for dir_name in dirs:
+            if dir_name not in index:
+                index[dir_name] = []
+            index[dir_name].append(root)
+    return index
+
+
+def change_name(name):
+    change_list = ["from", "class", "return", "false", "true", "id", "and", "", "ID"]
+    if name in change_list:
+        name = "is_" + name.lower()
+    return name
+
+
+def standardize(string):
+    res = re.compile("[^\\u4e00-\\u9fa5^a-z^A-Z^0-9^_]")
+    string = res.sub("_", string)
+    string = re.sub(r"(_)\1+", "_", string).lower()
+    while True:
+        if len(string) == 0:
+            return string
+        if string[0] == "_":
+            string = string[1:]
+        else:
+            break
+    while True:
+        if len(string) == 0:
+            return string
+        if string[-1] == "_":
+            string = string[:-1]
+        else:
+            break
+    if string[0].isdigit():
+        string = "get_" + string
+    return string
+
+def get_answer_log(log):
+    if log == []:
+        return "Beginnig of the agent. No log yet"
+    answer_logs = []
+    for ele in log:
+        answer_log = {"thought": "", "answer": ""}
+        answer_log["thought"] = ele["thought"]
+        answer_log["answer"] = ele["answer"]
+        answer_logs.append(answer_log)
+    return answer_logs
+
+def get_observation_log(log):
+    if log == []:
+        return ""
+    answer_logs = []
+    for i, ele in enumerate(log):
+        if i == len(log)-1:
+            answer_log = {"thought": "", "observation": ""}
+            answer_log["thought"] = ele["thought"]
+            answer_log["observation"] = ele["observation"]
+            answer_logs.append(answer_log)
+        else:
+            answer_log = {"thought": "", "answer": ""}
+            answer_log["thought"] = ele["thought"]
+            answer_log["answer"] = ele["answer"]
+            answer_logs.append(answer_log)
+    return answer_logs
+
+def build_tree(previous_log_totals, task_log):
+    
+    total_root_list = []
+    total_total_steps = 0
+    task_log_list = task_log.split("question: ")[1:]
+    for i in range(len(task_log_list)):
+        task_log_list[i] = task_log_list[i].split("answer: ")
+    
+    for j, previous_log_total in enumerate(previous_log_totals):
+        
+        if previous_log_total == None:
+            
+            answer_detail = {
+                "role": "plan_global",
+                "message": {
+                    "subtask": task_log_list[j][0],
+                    "subtask_answer": task_log_list[j][1]
+                },
+                "total_steps": 0,
+                "next": []
+            }
+            total_root_list.append(answer_detail)
+            continue
+
+        next_list = []
+        root_list = []
+        total_steps = 0
+        for i in range(len(previous_log_total)):
+            
+            current_log = previous_log_total[i]
+            
+            tool_call_list = []
+            api_name = current_log["action"]
+            parameter = current_log["action_input"]
+            response = current_log["observation"]
+            next_ele = {
+                "role": "tool",
+                "message": {
+                    "name": api_name,
+                    "arguments": parameter,
+                    "response": response
+                },
+                "next": []
+            }
+            tool_call_list.append(next_ele)
+            total_steps += 1
+            if len(tool_call_list) > 1:
+                for k in range(len(tool_call_list)-2, -1, -1):
+                    tool_call_list[k]["next"].append(tool_call_list[k+1])
+            next_list.append(tool_call_list[0])
+        total_total_steps += total_steps
+        
+        for i in range(len(next_list)-1, -1, -1):
+            current_log = next_list[i]
+            current_log_pre_id = previous_log_total[i]["previous_id"]
+            if current_log_pre_id == -1:
+                # print(current_log)
+                root_list.append(current_log)
+            else:
+                next_list[current_log_pre_id]["next"].append(current_log)
+        answer_detail = {
+            "role": "plan_global",
+            "message": {
+                "subtask": task_log_list[j][0],
+                "subtask_answer": task_log_list[j][1]
+            },
+            "total_steps": total_steps,
+            "next": root_list
+        }
+        total_root_list.append(answer_detail)
+
+    answer_details = {
+        "role": "system",
+        "message": "",
+        "next": [
+            {
+                "role": "user",
+                "message": "",
+                "next": total_root_list
+            }
+        ]
+    }
+    return answer_details, total_total_steps
+
+def get_answer_details(final_answer, previous_log):
+   
+    next_list = []
+    total_steps = 0
+    for i in range(len(previous_log)):
+        current_log = previous_log[i]
+        if not isinstance(current_log, dict):
+            next_ele = {
+                "role": "assistant",
+                "message": current_log,
+                "next": []
+            }
+            next_list.append(next_ele)
+            total_steps += 1
+            continue
+       
+        api_name = current_log["action"]
+        parameter = current_log["action_input"]
+        response = current_log["observation"]
+        next_ele = {
+            "role": "tool",
+            "message": {
+                "name": api_name,
+                "arguments": parameter,
+                "response": response
+            },
+            "next": []
+        }
+        next_list.append(next_ele)
+        total_steps += 1
+    answer_ele = {
+        "role": "tool",
+        "message": {
+            "name": "Finish",
+            "arguments": {
+                "return_type": "give_answer",
+                "final_answer": final_answer
+            },
+            "response": ""
+        },
+        "next": []
+    }
+    next_list.append(answer_ele)
+    for i in range(len(next_list)-2, -1, -1):
+        next_list[i]["next"].append(next_list[i+1])
+    next_result = next_list[0]
+    answer_details = {
+        "role": "system",
+        "message": "",
+        "next": [
+            {
+                "role": "user",
+                "message": "",
+                "next": [next_result]
+            }
+        ]
+    }
+    return answer_details, total_steps
+
+def contain(candidate_list, white_list):
+    output = []
+    for cand in candidate_list:
+        if cand not in white_list.keys():
+            return False
+        output.append(white_list[cand])
+    return output
+
+# def fetch_api_json(api_list, tool_root_dir):
+#     data_dict = {"api_list":[]}
+#     for item in api_list:
+#         cate_name = item["category_name"]
+#         tool_name = standardize(item["tool_name"])
+#         api_name = change_name(standardize(item["api_name"]))
+#         tool_json = json.load(open(os.path.join(tool_root_dir, cate_name, tool_name + ".json"), "r"))
+#         append_flag = False
+#         api_dict_names = []
+#         for api_dict in tool_json["api_list"]:
+#             api_dict_names.append(api_dict["name"])
+#             pure_api_name = change_name(standardize(api_dict["name"]))
+#             if pure_api_name != api_name:
+#                 continue
+#             api_json = {}
+#             api_json["category_name"] = cate_name
+#             api_json["api_name"] = api_dict["name"]
+#             api_json["api_description"] = api_dict["description"]
+#             api_json["required_parameters"] = api_dict["required_parameters"]
+#             api_json["optional_parameters"] = api_dict["optional_parameters"]
+#             api_json["tool_name"] = tool_json["tool_name"]
+#             data_dict["api_list"].append(api_json)
+#             append_flag = True
+#             break
+#         if not append_flag:
+#             print(api_name, api_dict_names)
+#     return data_dict
+
+# def api_json_to_openai_json(api_json,standard_tool_name):
+#     description_max_length=256
+#     templete =     {
+#         "name": "",
+#         "description": "",
+#         "parameters": {
+#             "type": "object",
+#             "properties": {
+#             },
+#             "required": [],
+#             "optional": [],
+#         }
+#     }
+    
+#     map_type = {
+#         "NUMBER": "integer",
+#         "STRING": "string",
+#         "BOOLEAN": "boolean"
+#     }
+
+#     pure_api_name = change_name(standardize(api_json["api_name"]))
+#     templete["name"] = pure_api_name+ f"_for_{standard_tool_name}"
+#     templete["name"] = templete["name"][-64:]
+
+#     templete["description"] = f"This is the subfunction for tool \"{standard_tool_name}\", you can use this tool."
+    
+#     if api_json["api_description"].strip() != "":
+#         tuncated_description = api_json['api_description'].strip().replace(api_json['api_name'],templete['name'])[:description_max_length]
+#         templete["description"] = templete["description"] + f"The description of this function is: \"{tuncated_description}\""
+#     if "required_parameters" in api_json.keys() and len(api_json["required_parameters"]) > 0:
+#         for para in api_json["required_parameters"]:
+#             name = standardize(para["name"])
+#             name = change_name(name)
+#             if para["type"] in map_type:
+#                 param_type = map_type[para["type"]]
+#             else:
+#                 param_type = "string"
+#             prompt = {
+#                 "type":param_type,
+#                 "description":para["description"][:description_max_length],
+#             }
+
+#             default_value = para['default']
+#             if len(str(default_value)) != 0:    
+#                 prompt = {
+#                     "type":param_type,
+#                     "description":para["description"][:description_max_length],
+#                     "example_value": default_value
+#                 }
+#             else:
+#                 prompt = {
+#                     "type":param_type,
+#                     "description":para["description"][:description_max_length]
+#                 }
+
+#             templete["parameters"]["properties"][name] = prompt
+#             templete["parameters"]["required"].append(name)
+#         for para in api_json["optional_parameters"]:
+#             name = standardize(para["name"])
+#             name = change_name(name)
+#             if para["type"] in map_type:
+#                 param_type = map_type[para["type"]]
+#             else:
+#                 param_type = "string"
+
+#             default_value = para['default']
+#             if len(str(default_value)) != 0:    
+#                 prompt = {
+#                     "type":param_type,
+#                     "description":para["description"][:description_max_length],
+#                     "example_value": default_value
+#                 }
+#             else:
+#                 prompt = {
+#                     "type":param_type,
+#                     "description":para["description"][:description_max_length]
+#                 }
+
+#             templete["parameters"]["properties"][name] = prompt
+#             templete["parameters"]["optional"].append(name)
+
+#     return templete, api_json["category_name"],  pure_api_name
+
+
+
+test_sets = ["G2_category"]