random pick an incomplete task instead of go with sequential order

app.py

@@ -38,12 +38,13 @@ def find_random_incomplete_task(app_name):
     return random_idx
 
 
-def write_task_data(app_name, idx, task, task_complete, task_completed_step):
+def write_task_data(app_name, idx, task, task_complete, task_completed_step, trajectory):
     doc_ref = db.collection(app_name).document(f"{app_name}_{idx}")
     doc_ref.set({
         "task": task,
         "task_completed": task_complete,
-        "task_completed_steps": task_completed_step
+        "task_completed_steps": task_completed_step,
+        "trajectory": trajectory
     })
 
 
@@ -57,19 +58,20 @@ class AppSimulator:
 
     def initialize_simulator(self, sitemap_url, progress=gr.Progress(track_tqdm=True)):
         """Initialize the simulator with retries and elapsed time tracking."""
-        synthetic_sitemap = load_dataset(sitemap_url, "sitemap", split='train')
-        app_name, sitemap, page_details, user_state, system_data = None, None, None, None, None
+        synthetic_sitemap = load_dataset(sitemap_url, "schema", split='train')
+        app_name, app_description, sitemap, relevant_tables_per_page, jinjia_prerender_page = None, None, None, None, None
+
         for row in synthetic_sitemap:
             if row['name'] == 'app_name':
-                app_name = row['value']  # Use `eval` to convert the string to a list
+                app_name = row['value']
+            elif row['name'] == 'app_description':
+                app_description = row['value']
             elif row['name'] == 'sitemap':
                 sitemap = json_repair.loads(row['value'])
-            elif row['name'] == 'page_details':
-                page_details = json_repair.loads(row['value'])
-            elif row['name'] == 'user_state':
-                user_state = json_repair.loads(row['value'])
-            elif row['name'] == 'system_data':
-                system_data = json_repair.loads(row['value'])
+            elif row['name'] == 'relevant_tables_per_page':
+                relevant_tables_per_page = json_repair.loads(row['value'])
+            elif row['name'] == 'jinjia_prerender_pages':
+                jinjia_prerender_page = json_repair.loads(row['value'])
 
         self.app_name = app_name
         smallest_index = find_random_incomplete_task(app_name)
@@ -78,25 +80,26 @@ class AppSimulator:
         self.smallest_index = smallest_index
 
         synthetic_tasks = load_dataset(sitemap_url, "tasks", split='train')
-        incomplete_task = synthetic_tasks[smallest_index]
-        task = incomplete_task["tasks"]
-        solution = incomplete_task["steps"]
-        user_data = incomplete_task["attributes"]["user_data"]
+        random_index = random.randint(0, len(synthetic_tasks) - 1)  # Generate a random index
+        random_row = synthetic_tasks[random_index]
+        task = random_row['task']
+        solution = random_row['solution']
+        database = random_row['database']
 
         self.task = task
 
         self.simulation = ChatbotSimulation(
             app_name=app_name,
+            app_description=app_description,
             site_map=sitemap,
-            page_details=page_details,
-            user_state=user_state,
-            system_data=system_data,
-            user_data=user_data,
+            relevant_tables_per_page=relevant_tables_per_page,
+            database=database,
+            jinjia_prerender_page=jinjia_prerender_page,
             task=task,
             solution=solution,
             log_location=f'conversation_log_{app_name}.txt',
             openai_api_key=openai_api_key,
-            agent='llm'
+            agent='Human'
         )
 
         initial_message = self.simulation.start_conversation()
@@ -124,7 +127,9 @@ def chat(user_input, history, simulator_app):
         app_name = simulator_app.app_name
         idx = simulator_app.smallest_index
         task = simulator_app.task
-        write_task_data(app_name, idx, task, task_complete, task_completed_step)
+        trajectory = simulator_app.simulation.trajectory
+
+        write_task_data(app_name, idx, task, task_complete, task_completed_step, trajectory)
 
     return response
 
@@ -137,8 +142,9 @@ def give_up(simulator_app):
     app_name = simulator_app.app_name
     idx = simulator_app.smallest_index
     task = simulator_app.task
+    trajectory = simulator_app.simulation.trajectory
 
-    write_task_data(app_name, idx, task, task_completed, task_completed_steps)
+    write_task_data(app_name, idx, task, task_completed, task_completed_steps, trajectory)
 
     return "Task marked as abandoned (Give-Up action)."
 

chatbot_simulator.py

@@ -5,24 +5,28 @@ from prompts import *
 import re
 from tenacity import retry, wait_fixed, stop_after_attempt, retry_if_exception_type
 from openai import RateLimitError
+from difflib import get_close_matches
 
 
 class ChatbotSimulation:
-    def __init__(self, app_name, site_map, page_details, user_state, system_data, user_data, task, solution,
+    def __init__(self, app_name, app_description, site_map, relevant_tables_per_page,
+                 database, jinjia_prerender_page, task, solution,
                  log_location, openai_api_key, agent='human',
-                 max_steps=50, max_tokens=8192, buffer_tokens=500):
+                 max_steps=30, max_tokens=8192, buffer_tokens=500):
         self.app_name = app_name
+        self.app_description = app_description
         self.sitemap = site_map
-        self.page_details = page_details
-        self.user_state = user_state
-        self.user_state['current_page'] = 'Home'  # Initialize current page
-        self.user_state['task_completed'] = 'False'
-        self.user_state['back'] = 'False'
-        self.system_data = system_data
-        self.user_data = user_data
+        self.relevant_tables_per_page = relevant_tables_per_page
+        self.database = database
+        self.jinjia_prerender_page = jinjia_prerender_page
         self.task = task
         self.solution = solution
 
+        self.user_state = dict()
+        self.user_state['current_page'] = self.sitemap['pages'][0]['id']  # Initialize current page
+        self.user_state['task_completed'] = 'False'
+        self.user_state['back'] = 'False'
+
         self.log_location = log_location
         self.agent = agent.lower()
         if self.agent not in ['human', 'llm']:
@@ -31,40 +35,53 @@ class ChatbotSimulation:
         self.max_tokens = max_tokens
         self.buffer_tokens = buffer_tokens
         self.conversation = []  # Stores recent conversation snippets
+        self.trajectory = [{"role": "system", "content": f"Welcome to {app_name} simulator! Your task is: {task}"}]
         self.prompt_count = 0
         self.client = OpenAI(api_key=openai_api_key)
         self.actions = []
         self.tokenizer = AutoTokenizer.from_pretrained("gpt2", clean_up_tokenization_spaces=True)
 
-        # back button
+        #back button
         self.page_history = ['Home']
 
-    def _get_page_uid(self, page_name):
-        """Retrieve the UID of the given page from the sitemap."""
-        return self.sitemap.get(page_name, {}).get('uid')
-
-    def _get_page_details(self, page_name):
-        """Retrieve the page details using its UID."""
-        uid = self._get_page_uid(page_name)
-        return self.page_details.get(uid, {})
+    def _get_relevant_data(self, current_page):
+        # Check if the current page exists as a key
+        if current_page in self.relevant_tables_per_page:
+            relevant_tables = self.relevant_tables_per_page[current_page]
+        else:
+            # Find the closest matching key
+            closest_match = get_close_matches(current_page, self.relevant_tables_per_page.keys(), n=1, cutoff=0.5)
+            if closest_match:
+                relevant_tables = self.relevant_tables_per_page[closest_match[0]]
+            else:
+                return self.database
+
+        return {table: self.database[table] for table in relevant_tables if table in self.database}
+
+    def _get_prerender_page(self, current_page):
+        if current_page in self.jinjia_prerender_page:
+            return self.jinjia_prerender_page[current_page]
+        else:
+            closest_match = get_close_matches(current_page, self.jinjia_prerender_page.keys(), n=1, cutoff=0)
+            return self.jinjia_prerender_page[closest_match[0]]
 
     def _generate_system_prompt(self):
         """Create a dynamic system prompt based on the current state."""
-        current_page = self.page_history[-1] if len(self.page_history) >= 1 else "Home"
-        last_page = self.page_history[-2] if len(self.page_history) > 1 else "Home"
-        page_info = self._get_page_details(current_page)
-        page_linkage = self.sitemap.get(current_page, {}).get('links_to')
+        current_page = self.page_history[-1] if len(self.page_history) >= 1 else self.sitemap['pages'][0]['id']
+        last_page = self.page_history[-2] if len(self.page_history) > 1 else self.sitemap['pages'][0]['id']
+        relevant_database = self._get_relevant_data(current_page)
+        relevant_sitemap = next((page for page in self.sitemap["pages"] if page["id"] == current_page), self.sitemap["pages"])
+        prerender_page = self._get_prerender_page(current_page)
 
         return get_system_prompt(app_name=self.app_name,
-                                 system_data=self.system_data,
+                                 app_description=self.app_description,
+                                 relevant_database=relevant_database,
                                  task=self.task,
-                                 user_data=self.user_data,
                                  current_page=current_page,
                                  last_page=last_page,
                                  actions=self.actions,
-                                 user_state=self.user_state,
-                                 page_info=page_info,
-                                 page_linkage=page_linkage
+                                 sitemap_page=relevant_sitemap,
+                                 jinjia_prerender=prerender_page,
                                  )
 
     @retry(
@@ -100,28 +117,33 @@ class ChatbotSimulation:
     def one_conversation_round(self, user_input):
         """Conduct one round of conversation between the user and the assistant."""
         # User provides input
+        self.trajectory.append({"role": "user", "content": f'Human: {user_input}'})
         valid_input = self._is_valid_input(user_input)
         if valid_input[0]:
             pass
         else:
-            return f"\n{self.app_name}: Invalid input. {valid_input[1]}"
+            self.prompt_count += 1
+            invalid_input_message = f"\n{self.app_name}: Invalid input. {valid_input[1]}"
+            self.trajectory.append({"role": "assistant", "content": invalid_input_message})
+            return invalid_input_message
 
         self.actions.append(user_input + f'on {self.user_state["current_page"]} page')
         self.conversation.append({"role": "user", "content": user_input})
         self.prompt_count += 1
 
         # Update user state using GPT's response
-        current_page = self.page_history[-1] if len(self.page_history) >= 1 else "Home"
+        current_page = self.page_history[-1] if len(self.page_history) >= 1 else self.sitemap['pages'][0]['id']
         update_prompt = get_user_state_update_prompt(user_input=user_input,
                                                      current_page=current_page,
                                                      task=self.task,
+                                                     database=self.database,
                                                      solution=self.solution,
                                                      user_state=self.user_state,
                                                      sitemap=self.sitemap)
 
-        self.conversation.append({"role": "assistant", "content": update_prompt})
+        self.conversation.append({"role": "user", "content": update_prompt})
         updated_state = self._get_openai_response(self.conversation).split("UPDATED", 1)[1].strip()
-        self.conversation.pop(-1) ## update prompt don't have to stay in conversation history
+        self.conversation.pop(-1) # update prompt don't have to stay in conversation history
 
         # Parse and update the user state
         updated_state = json_repair.loads(updated_state)
@@ -151,7 +173,9 @@ class ChatbotSimulation:
 
         try:
             if str(updated_state['task_completed']).lower() == 'true':
-                return f"Task completed! You took {self.prompt_count} steps."
+                complete_message = f"{self.app_name}: Task completed! You took {self.prompt_count} steps."
+                self.trajectory.append({"role": "assistant", "content": complete_message})
+                return complete_message
         except:
             updated_state['task_completed'] = 'False'
 
@@ -168,6 +192,7 @@ class ChatbotSimulation:
         self.conversation.append({"role": "system", "content": system_prompt})
         gpt_instruction = self._get_openai_response(self.conversation)
         self.conversation.append({"role": "assistant", "content": gpt_instruction})
+        self.trajectory.append({"role": "assistant", "content": gpt_instruction})
         return gpt_instruction
 
     def start_conversation(self):

prompts.py

@@ -1,97 +1,80 @@
+import textwrap
 
-def get_system_prompt(app_name, system_data, task, user_data, current_page, last_page, actions, user_state, page_info, page_linkage):
-    system_prompt = f"""
-You are a text-based {app_name}-like app simulator game.  
-The {app_name} app contains the following system data: {system_data}.  
-You are interacting with a user whose task is: {task}.  
-User's data: {user_data}.  
-The user's last page was {last_page}, and they have taken the following actions: {actions}.  
-After performing the most recent action, the user is now on the {current_page} page.  
-Current user state: {user_state}.  
-
-Details and buttons on the current page:  
-{page_info}  
-
-Current page links to these pages:
-{page_linkage}
-
-Each button on the page has a `name` and an `action_type`.  
-
-### Instruction Format:
-<If `actions` is non-empty: "You have successfully performed actions[-1]"> z 
-<If current_page == log in & user_state['logged_in'] == False: You need to log in first. Your credentials are: Username: user_data['credentials']['user_id'] Password: user_data['credentials']['password']>
-<If current_page == log in & user_state['logged_in'] == True: You have successfully logged in as user_data['credentials']['user_id']>
-You are currently on the **{current_page}** page. 
-
-<Display static information to users based on system data and user data on the current page (e.g., display search results)>
-
-You have the following options:  
-1. option 1: action type  
-2. option 2: action type  
-3. option 3: action type  
-4. option 4: action type  
-<If any linked page does not have button links to it, add the button.>
-5. Back to last page: click
-6. Back to Home page: click
-7. <If user_state['logged_in'] == False, display Log In: click to take user to log in page>
-8. Do nothing: noop
-
-Each button is in the format: `number. button name: action_type`.
-Please provide your choice in the format: `action_type(button name)`.  
-If you have a query, use the format: `type(button name, query)`.  
-Please provide only one choice per turn.
-
-### **Rules**:
-1. Display all available options from the buttons on the page.  
-2. Remain robotic and emotionless. Avoid offering any advice or opinions to the user.  
-3. Ensure there is always a button to navigate back to the **{last_page}** and a button to go to the **Home page**.
-4. Ensure there is always a button that allow user to do nothing.
-5. Ensure the action type is a single word, such as ‘type’ or ‘click.’ If the action type is not a single word, convert it to a single word. For example, type(SUMMER, apply) should be transformed to type.
-6. Group similar action types under ‘type’ and display them as ‘type’ (e.g., display ‘input’ or ‘search’ as ‘type’).
-7. Mimic the real {app_name} scenario as much as possible.
-    """
+
+def get_system_prompt(app_name, app_description, relevant_database, task,
+                      current_page, last_page, actions, sitemap_page, jinjia_prerender):
+    system_prompt = textwrap.dedent(f"""
+    You are a text-based CLI {app_name} ({app_description}) simulator.  
+    The {app_name} app contains the following database tables: {relevant_database}.  
+    You are interacting with a user whose task is: {task}.  
+    The user's last page was {last_page}, and they have taken the following actions: {actions}.  
+    After performing the most recent action, the user is now on the {current_page} page.  
+
+    Details and buttons on the current page:  
+    {sitemap_page}  
+
+    This page should display following pre-rendered page to user and making sure replace the placeholders with real data from database.
+
+    {jinjia_prerender}
+
+    ### **Rules**:
+    1. Display all information from the pre-rendered page.  
+    2. Remain robotic and emotionless. Avoid offering any advice or opinions to the user. 
+    3. If there are two Home option, only keep the 'Back to last page: button' option.
+    4. Add a button to interact with displayed item if such button doesn't exist.
+    5. Mimic the real {app_name} scenario as much as possible.
+    6. Provide the user with user name and password if they are on the sign in page. 
+    """)
     return system_prompt
 
-def get_user_state_update_prompt(user_input, current_page, task, solution, user_state, sitemap):
-    structure = """
-{
-   'current_page': 'Home',
-   'task_completed': False,
-   'back': False
-}
-    """
-    update_prompt = f"""
-You will now update the user state.  
-The user takes the action '{user_input}' on the {current_page} page. Determine which page the user will move to next.  
-The next page must from the sitemap: {sitemap}
-Recall the user's task: {task}. Solution to the user's task: {solution}.  
-Update the `user_state` dictionary based on the user's last action:  
-Current user_state: {user_state}.  
-
-### Instructions:
-1. If the 'current_page' has changed, update it to a valid page from the sitemap.
-2. If the task is completed, update 'task_completed' to `True`. Otherwise, leave it as `False`.
-3. If no updates are needed, return the `user_state` exactly as provided, without any changes.
-
-### Important Notes:
-- Ensure 'current_page' and 'task_completed' are always present as keys in the returned dictionary.
-- If the user go back a page from current page, set 'back' to `True`. Otherwise, it should remain `False`.
-- Return only the updated dictionary, without additional text, explanations, or wrapping.
-
-### Output Format:
-1. Start with "REASON": Explain which fields in `user_state` need to be updated and why.
-2. End with "UPDATED": Provide only the updated `user_state` dictionary. Follow the input user_state schema structure exactly without modifying its organization. Do not include explanations or any additional text after updated user state.
-    """
+
+def get_database_update_prompt(user_input, current_page, database):
+    update_database = textwrap.dedent(f"""
+    You will now update the database.
+    The user takes the action '{user_input}' on the {current_page} page. Determine how will database change. 
+    Recall the current Database: 
+    {database}
+
+    ### Output Format:
+    1. Start with "REASON": Explain which fields in database need to be updated and why.
+    2. End with "UPDATED": Provide only the updated database. Follow the input database schema structure exactly without modifying its organization. Do not include explanations or any additional text after updated database.
+    """)
+    return update_database
+
+
+def get_user_state_update_prompt(user_input, current_page, task, database, solution, user_state, sitemap):
+    update_prompt = textwrap.dedent(f"""
+    You will now update the user state.  
+    The user takes the action '{user_input}' on the {current_page} page. Determine which page the user will move to next.  
+    The next page must from the sitemap: {sitemap}
+    Recall the user's task: {task}. Recall the current Database: {database}. Solution to the user's task: {solution}.  
+    Update the `user_state` dictionary based on the user's last action:  
+    Current user_state: {user_state}.  
+
+    ### Instructions:
+    1. If the 'current_page' has changed, update it to a valid page from the sitemap.
+    2. If the task is completed, update 'task_completed' to `True`. Otherwise, leave it as `False`.
+    3. If no updates are needed, return the `user_state` exactly as provided, without any changes.
+
+    ### Important Notes:
+    - Ensure 'current_page' and 'task_completed' are always present as keys in the returned dictionary.
+    - If the user go back a page from current page, set 'back' to `True`. Otherwise, it should remain `False`.
+    - Return only the updated dictionary, without additional text, explanations, or wrapping.
+
+    ### Output Format:
+    1. Start with "REASON": Explain which fields in `user_state` need to be updated and why.
+    2. End with "UPDATED": Provide only the updated `user_state` dictionary. Follow the input user_state schema structure exactly without modifying its organization. Do not include explanations or any additional text after updated user state.
+    """)
     return update_prompt
 
 
 def get_agent_prompt(app_name, task, conversation):
-    agent_prompt = f"""
+    agent_prompt = textwrap.dedent(f"""
     Imagine you are an agent navigate through the {app_name} environment.
     Your overarching task is: {task}. You may have done some part of the task, or none at all.
     You will have access to all of your previous actions in the environment, as well as the last message from the assistant giving the current state of the environment.
     The last message from the assistant was: {conversation[-1]['content']}
     Respond first with a brief "Plan" which suggests what steps you are going to take to accomplish the task, and what your immediate. 
     Then generate an "Action" which is the immediate next step you can take.
-            """
+    """)
     return agent_prompt