Initial commit

.gitignore

@@ -0,0 +1,9 @@
+.env
+*/__pycache__
+.vscode
+notebooks/
+*.pyc
+local_tests/
+questions_tests.txt
+env/
+spinoza_env

LICENSE.txt

@@ -0,0 +1,15 @@
+Project Spinoza License
+
+This project, Spinoza, was developed by Ekimetrics, Reporters sans Frontières, and l'Alliance de la presse d'information générale, and funded by the French Ministry of Culture.
+License: GNU General Public License v3.0
+
+This project is licensed under the GNU General Public License, version 3.0 (GPL-3.0). A full copy of the GPL-3.0 license is available at https://www.gnu.org/licenses/gpl-3.0#license-text
+Key Provisions:
+
+    Any redistribution or reuse of the front-end interface of the Spinoza project must retain the footer that references the aforementioned organizations (Ekimetrics, Reporters sans Frontières, l'Alliance de la presse d'information générale) and the French Ministry of Culture.
+
+    The software includes components that rely on NVIDIA CUDA Runtime and other NVIDIA-specific packages. These packages impose constraints that restrict the software's deployment to environments running on NVIDIA GPUs, as is the case with similar platforms such as Hugging Face.
+
+    If this software or any of its components are made available outside the Spinoza repository, it is the responsibility of the person or organization making the software available to ensure compliance with all applicable licensing terms, including but not limited to, ensuring that all necessary legal conditions and technical constraints (such as deployment on NVIDIA hardware) are met.
+
+By using, modifying, or redistributing this software, you agree to these terms.

README.md

@@ -1,12 +1,59 @@
 ---
-title: Spinoza Public
-emoji: 👀
-colorFrom: purple
-colorTo: yellow
+title: Spinoza
+emoji: 🐨
+colorFrom: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 5.23.1
+sdk_version: 4.37.2
 app_file: app.py
 pinned: false
+hf_oauth: true
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Spinoza Project
+Reporters Without Borders (RSF) has teamed up with the General Information Press Alliance (the Alliance) to initiate the first step of its **Spinoza project** to develop an artificial intelligence tool by and for journalists that will safeguard the media’s intellectual property over what they publish. You can find more information <a href="https://rsf.org/en/rsf-and-french-general-press-alliance-launch-spinoza-project-develop-ai-tool-journalists" target="_blank">here</a> and <a href="https://rsf.org/en/journalists-france-test-spinoza-project-s-first-ai-prototype-launched-rsf-and-alliance" target="_blank">here</a>.
+
+A visual and guided introduction to the artificial intelligence tool can be found in the <a href="https://www.youtube.com/@reporterssansfrontieres" target="_blank">RSF's Youtube Channel</a>. You can have a look just below.
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/iBy6IBSxRkw?si=zjissjcLEOJv7cwP" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+
+## Table of Contents 
+- [Installation](#installation)
+- [Usage](#usage)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Installation
+1. Clone the repository (SSH):
+```bash
+ git clone https://huggingface.co/spaces/SpinozaProject/spinoza
+```
+
+2. Install dependencies:
+```bash
+ poetry install
+ ```
+
+Note that we highly recommend to create a new env :
+```bash
+ conda create -n myenv python=3.10
+ conda activate myenv
+
+ pip install poetry
+```
+
+## Usage
+To run the project in local workspace, use the following command:
+```bash
+python app.py
+```
+
+## Contributing
+1. Clone the repository.
+2. Create a new branch: `git checkout -b feature-name`.
+3. Make your changes.
+4. Push your branch: `git push origin feature-name`.
+5. Create a pull request.
+
+## License
+WIP

app.py

@@ -0,0 +1,640 @@
+import gradio as gr
+import time
+import os
+import yaml
+from qdrant_client import models
+from tqdm import tqdm
+from collections import defaultdict
+import pandas as pd
+
+from spinoza_project.source.backend.llm_utils import (
+    get_llm_api,
+)
+from spinoza_project.source.frontend.utils import (
+    init_env,
+    parse_output_llm_with_sources,
+)
+from spinoza_project.source.frontend.gradio_utils import (
+    get_sources,
+    set_prompts,
+    get_config,
+    get_prompts,
+    get_assets,
+    get_theme,
+    get_init_prompt,
+    get_synthesis_prompt,
+    get_qdrants,
+    start_agents,
+    end_agents,
+    next_call,
+    zip_longest_fill,
+    reformulate,
+    answer,
+    get_text,
+    update_translation,
+)
+
+from assets.utils_javascript import (
+    accordion_trigger,
+    accordion_trigger_end,
+    accordion_trigger_spinoza,
+    accordion_trigger_spinoza_end,
+    update_footer
+)
+
+init_env()
+
+with open("./spinoza_project/config.yaml") as f:
+    config = yaml.full_load(f)
+
+## Loading Prompts
+print("Loading Prompts")
+prompts = get_prompts(config)
+chat_qa_prompts, chat_reformulation_prompts = set_prompts(prompts, config)
+synthesis_prompt_template = get_synthesis_prompt(config)
+
+## Building LLM
+print("Building LLM")
+groq_model_name = config.get("groq_model_name", "")
+llm = get_llm_api(groq_model_name)
+
+## Loading BDDs
+print("Loading Databases")
+qdrants, df_qdrants = get_qdrants(config)
+
+dataframes_by_source = {
+    source: df_qdrants[df_qdrants['Source'] == source].drop(columns=['Source'])
+    for source in df_qdrants['Source'].unique()
+}
+
+for source, df in dataframes_by_source.items():
+    dataframes_by_source[source]['Filter'] = dataframes_by_source[source]['Filter'].fillna('Unknown')
+
+    unknown_percentage = df.apply(lambda x: (x == 'Unknown').mean())
+    columns_to_drop = unknown_percentage[unknown_percentage == 1.0].index
+    
+    if len(columns_to_drop) > 0:
+        print(f"Deleting following columns for {source}: {columns_to_drop.tolist()}")
+        dataframes_by_source[source] = df.drop(columns=columns_to_drop)
+
+## Loading Assets
+print("Loading assets")
+css, source_information_fr, source_information_en, about_contact_fr, about_contact_en = get_assets()
+theme = get_theme()
+init_prompt = get_init_prompt()
+
+## Updating TRANSLATIONS dictionnary
+list_tabs = list(config["tabs"])
+update_translation(list_tabs, config)
+
+def get_source_df(source_name):
+    return dataframes_by_source.get(source_name, pd.DataFrame())
+
+LANGUAGE_MAPPING = {
+    "fr": "french/français",
+    "en": "english/anglais"
+}
+
+def reformulate_questions(
+    lang_component,
+    question,
+    llm=llm,
+    chat_reformulation_prompts=chat_reformulation_prompts,
+    config=config,
+):
+    lang = lang_component.value if hasattr(lang_component, 'value') else lang_component
+    language = LANGUAGE_MAPPING.get(lang, "french/français")
+
+    for elt in zip_longest_fill(
+        *[
+            reformulate(language, llm, chat_reformulation_prompts, question, tab, config=config)
+            for tab in config["tabs"]
+        ]
+    ):
+        time.sleep(0.02)
+        yield elt
+
+def retrieve_sources(
+    *questions,
+    filters_dict,
+    qdrants=qdrants,
+    config=config,
+):
+    if filters_dict is None:
+        filters_dict = {}
+
+    formated_sources, text_sources = get_sources(
+        questions, filters_dict, qdrants, config
+    )
+
+    return (formated_sources, *text_sources)
+
+def retrieve_sources_wrapper(*args):
+    questions = list(args[:-1])
+    filters = args[-1]
+
+    return retrieve_sources(
+        questions,
+        filters_dict=filters
+    )
+
+def answer_questions(
+    lang_component, 
+    *questions_sources, 
+    llm=llm, 
+    chat_qa_prompts=chat_qa_prompts, 
+    config=config
+):
+    lang = lang_component.value if hasattr(lang_component, 'value') else lang_component
+    language = LANGUAGE_MAPPING.get(lang, "french/français")
+
+    questions = [elt for elt in questions_sources[: len(questions_sources) // 2]]
+    sources = [elt for elt in questions_sources[len(questions_sources) // 2 :]]
+
+    for elt in zip_longest_fill(
+        *[
+            answer(language, llm, chat_qa_prompts, question, source, tab, config)
+            for question, source, tab in zip(questions, sources, config["tabs"])
+        ]
+    ):
+        time.sleep(0.02)
+        yield [
+            [(question, parse_output_llm_with_sources(ans))]
+            for question, ans in zip(questions, elt)
+        ]
+
+def get_synthesis(
+    lang_component,
+    question,
+    *answers,
+    llm=llm,
+    synthesis_prompt_template=synthesis_prompt_template,
+    config=config,
+):
+    lang = lang_component.value if hasattr(lang_component, 'value') else lang_component
+    language = LANGUAGE_MAPPING.get(lang, "french/français")
+
+    answer = []
+    for i, tab in enumerate(config["tabs"]):
+        if len(str(answers[i])) >= 100:
+            answer.append(
+                f"{tab}\n{answers[i]}".replace("<p>", "").replace("</p>\n", "")
+            )
+
+    if len(answer) == 0:
+        return "Aucune source n'a pu être identifiée pour répondre, veuillez modifier votre question"
+    else:
+        for elt in llm.stream(
+            synthesis_prompt_template,
+            {
+                "question": question.replace("<p>", "").replace("</p>\n", ""),
+                "answers": "\n\n".join(answer),
+                "language": language
+            },
+        ):
+            time.sleep(0.01)
+            yield [(question, parse_output_llm_with_sources(elt))]
+
+def get_unique_values_filters(df):
+    filters_values = sorted([
+        str(x) for x in df['Filter'].unique() 
+        if pd.notna(x) and str(x).strip() != ''
+    ])
+
+    return filters_values
+
+def filter_data(filter, source): 
+    if source not in dataframes_by_source:
+        raise ValueError(f"'{source}' not found withing the sources availible")
+    
+    df = dataframes_by_source[source]
+
+    if filter:
+        df = df[df['Filter'].fillna('').astype(str).isin(filter)]
+
+    return df.values.tolist()
+              
+def update_filters(filters_dict, agent, values):
+    field = "file_filtering_modality"
+    if filters_dict is None:
+        filters_dict = {}
+    new_filters = dict(filters_dict)
+
+    if agent not in new_filters:
+        new_filters[agent] = {}
+
+    if not values or isinstance(values, list):
+        if field in new_filters[agent]:
+            del new_filters[agent][field]
+        if not new_filters[agent]:
+            del new_filters[agent]
+    else:
+        new_filters[agent][field] = values
+
+    return new_filters, new_filters
+
+with gr.Blocks(
+    title=f"🔍 Spinoza",
+    css=css,
+    js=update_footer(),
+    theme=theme,
+) as demo:
+    accordions_qa = {}
+    accordions_filters = {}
+    current_language = gr.State(value="fr")
+    chatbots = {}
+    question = gr.State("")
+    agt_input_flt = {}
+    agt_desc = {}
+    agt_input_dsp = gr.State({})
+    docs_textbox = gr.State([""])
+    agent_questions = {elt: gr.State("") for elt in config["tabs"]}
+    component_sources = {elt: gr.State("") for elt in config["tabs"]}
+    text_sources = {elt: gr.State("") for elt in config["tabs"]}
+    tab_states = {elt: gr.State(elt) for elt in config["tabs"]}
+    filters_state = gr.State({})
+    filters_display = gr.JSON(
+        label="Filtres sélectionnés",
+        value={},
+        visible=False
+    )
+
+    with gr.Row(elem_classes="header-row"):
+        button_fr = gr.Button("", elem_id="fr-button", elem_classes="lang-button", icon='./assets/logos/france_round.png') 
+        button_en = gr.Button("", elem_id="en-button", elem_classes="lang-button", icon='./assets/logos/us_round.png')
+
+    with gr.Row(elem_classes="main-row"):
+        with gr.Tab("Q&A", elem_id="main-component"):
+            with gr.Row(elem_id="chatbot-row"):
+                with gr.Column(scale=2, elem_id="center-panel"):
+                    with gr.Row(elem_id="input-message"):
+                        ask = gr.Textbox(
+                            placeholder=get_text("ask_placeholder", current_language.value),
+                            show_label=False,
+                            scale=7,
+                            lines=1,
+                            interactive=True,
+                            elem_id="input-textbox",
+                        )
+
+                    with gr.Group(elem_id="chatbot-group"):
+                        for tab in list(config["tabs"].keys()):
+                            agent_name = get_text(f"agent_{config['source_mapping'][tab]}_qa", current_language.value)
+                            elem_id = f"accordion-{config['source_mapping'][tab]}"
+                            elem_classes = "accordion accordion-agent"
+
+                            with gr.Accordion(
+                                label=agent_name,
+                                open=False,
+                                elem_id=elem_id,
+                                elem_classes=elem_classes,
+                            ) as accordions_qa[config['source_mapping'][tab]]:
+                                # chatbot_key = agent_name.lower().replace(" ", "_")
+                                chatbots[tab] = gr.Chatbot(
+                                    value=None,
+                                    show_copy_button=True,
+                                    show_share_button=False,
+                                    show_label=False,
+                                    elem_id=f"chatbot-{agent_name.lower().replace(' ', '-')}",
+                                    layout="panel",
+                                    avatar_images=(
+                                        "./assets/logos/help.png",
+                                        (
+                                            "./assets/logos/spinoza.png"
+                                            if agent_name == "Spinoza"
+                                            else None
+                                        ),
+                                    )
+                                )
+
+                        agent_name = "Spinoza"
+                        with gr.Accordion(
+                            label=agent_name,
+                            open=True,
+                            elem_id="accordion-Spinoza",
+                            elem_classes="accordion accordion-agent spinoza-agent",
+                        ) as accordion_spinoza:
+                            # chatbot_key = agent_name.lower().replace(" ", "_")
+                            chatbots["Spinoza"] = gr.Chatbot(
+                                value=([(None, get_text("init_prompt", current_language.value))]),
+                                show_copy_button=True,
+                                show_share_button=False,
+                                show_label=False,
+                                elem_id=f"chatbot-{agent_name.lower().replace(' ', '-')}",
+                                layout="panel",
+                                avatar_images=(
+                                    "./assets/logos/help.png",
+                                    "./assets/logos/spinoza.png",
+                                ),
+                            )
+
+                with gr.Column(scale=1, variant="panel", elem_id="right-panel"):
+                    with gr.TabItem("Sources", elem_id="tab-sources", id=0):
+                        sources_textbox = gr.HTML(
+                            show_label=False, elem_id="sources-textbox"
+                        )
+
+        with gr.Tab(label=get_text("source_filter_label", current_language.value), elem_id="filter-component") as source_filter_tab:
+            source_filter_title= gr.Markdown(value=get_text("source_filter_title", current_language.value))
+            source_filter_subtitle = gr.Markdown(value=get_text("source_filter_subtitle", current_language.value))
+
+            with gr.Row(elem_id="filter-row"):
+                with gr.Column(scale=2, elem_id="filter-center-panel"):
+                    with gr.Group(elem_id="filter-group"):
+                        for tab in list(config["tabs"].keys()):
+                            agent_name = get_text(f"agent_{config['source_mapping'][tab]}_flt", current_language.value)
+                            elem_id = f"accordion-filter-{config['source_mapping'][tab]}"
+                            elem_classes = "accordion accordion-source"
+                            
+                            with gr.Accordion(
+                                label=agent_name,
+                                open=False,
+                                elem_id=elem_id,
+                                elem_classes=elem_classes,
+                            ) as accordions_filters[config['source_mapping'][tab]]:
+                                question_filter = gr.Markdown(value=get_text("question_filter", current_language.value))
+                                with gr.Tabs():
+                                    df = get_source_df(config['source_mapping'][tab])
+                                    if not df.empty and 'Filter' in df.columns:                             
+                                        filters = get_unique_values_filters(df)
+                                        
+                                        with gr.Row():
+                                            var_name = f"{config['source_mapping'][tab]}_input_flt"            
+                                            agt_input_flt[var_name] = gr.CheckboxGroup(
+                                                [filter for filter in filters],
+                                                label="Filter(s):"
+                                            )
+
+                                            agt_input_flt[var_name].change(
+                                                fn=update_filters,
+                                                inputs=[filters_state, gr.State(config['source_mapping'][tab]), agt_input_flt[var_name]],
+                                                outputs=[filters_state, filters_display]
+                                            )
+
+                                    else:
+                                        gr.Markdown("**Error:** No data / 'Filter' column doesn't exist...")                                         
+
+        with gr.Tab(label=get_text("source_informatation_label", current_language.value), elem_id="source-component") as source_information_tab:
+            with gr.Row():
+                with gr.Column(scale=1):
+                    display_info_desc = gr.Markdown(value=get_text("display_info_desc", current_language.value))
+                    accordions_inf = {}
+                    with gr.Tabs(elem_id="main-tab-disp"):
+                            for tab in list(config["tabs"].keys()):
+                                agent_name = get_text(f"agent_{config['source_mapping'][tab]}_tab", current_language.value)
+                                elem_id = f"accordion-{config['source_mapping'][tab]}-tab"
+                                elem_classes = "disp-tabs"
+
+                                with gr.Tab(
+                                    label=agent_name,
+                                    elem_id=elem_id,
+                                    elem_classes=elem_classes
+                                ) as accordions_inf[config['source_mapping'][tab]]:
+                                    var_name = f"{config['source_mapping'][tab]}_desc"
+                                    agt_desc[var_name] = gr.Markdown(value=get_text(f"{config['source_mapping'][tab]}_desc", current_language.value))
+                                    df = get_source_df(config['source_mapping'][tab])
+                                    if not df.empty and 'Filter' in df.columns:
+                                        filters = get_unique_values_filters(df)
+                                        
+                                        with gr.Row():
+                                            var_name = f"{config['source_mapping'][tab]}_input_dsp"                                                                          
+                                            agt_input_dsp.value[var_name] = gr.CheckboxGroup(
+                                                [filter for filter in filters],
+                                                label="Filter(s):"
+                                            )
+
+                                        output_df = gr.Dataframe(
+                                            headers=['Title', 'Pages', 'Filter Category', 'Publishing Date'],
+                                            datatype=['str', 'number', 'str', 'number'],
+                                            value=df.values.tolist(),
+                                            column_widths=[300, 100, 100, 150],
+                                            wrap=True
+                                        )
+                
+                                        agt_input_dsp.value[var_name].change(
+                                            filter_data,
+                                            inputs=[agt_input_dsp.value[var_name]]+[gr.State(config['source_mapping'][tab])],
+                                            outputs=[output_df]
+                                        )
+                                    
+                                    else:
+                                        gr.Markdown("**Error:** No data / 'Filter' column doesn't exist...")  
+
+        with gr.Tab(label=get_text("contact_label", current_language.value), elem_id="contact-component") as contact_label:
+            with gr.Row():
+                with gr.Column(scale=1):
+                    contact_info = gr.Markdown(value=about_contact_fr)
+
+    ask.submit(
+        start_agents, inputs=[current_language], outputs=[chatbots["Spinoza"]] + [source_filter_tab], js=accordion_trigger()
+    ).then(
+        fn=reformulate_questions,
+        inputs=[current_language]+
+        [ask],
+        outputs=[agent_questions[tab] for tab in config["tabs"]],
+    ).then(
+        fn=retrieve_sources_wrapper,
+        inputs=[agent_questions[tab] for tab in config["tabs"]] + [filters_state],
+        outputs=[sources_textbox] + [text_sources[tab] for tab in config["tabs"]],
+    ).then(
+        fn=answer_questions,
+        inputs=[current_language]
+        + [agent_questions[tab] for tab in config["tabs"]]
+        + [text_sources[tab] for tab in config["tabs"]],
+        outputs=[chatbots[tab] for tab in config["tabs"]],
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_end()
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_spinoza()
+    ).then(
+        fn=get_synthesis,
+        inputs=[current_language]
+        + [ask]
+        + [chatbots[tab] for tab in config["tabs"]],
+        outputs=[chatbots["Spinoza"]],
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_spinoza_end()
+    ).then(
+        fn=end_agents, inputs=[current_language], outputs=[source_filter_tab]
+    )
+
+    def reset_app(language):
+       
+        chatbot_updates = {}
+        for tab in config["tabs"]:
+            chatbot_updates[tab] = gr.update(value=None)
+        chatbot_updates["Spinoza"] = gr.update(value=[(None, get_text("init_prompt", language))])
+        
+        empty_checkbox = gr.update(value=None)
+        checkbox_components = list(agt_input_flt.keys()) + list(agt_input_dsp.value.keys())
+        checkbox_updates = {component: empty_checkbox for component in checkbox_components}
+        
+        return {
+            "chatbots": chatbot_updates,
+            "filters_state": gr.update(value={}),
+            "filters_display": gr.update(value={}),
+            "ask": gr.update(value="", placeholder=get_text("ask_placeholder", language)),
+            "sources_textbox": gr.update(value=""),
+            "checkbox_updates": checkbox_updates
+        }
+    
+    def toggle_language_fr():
+        reset_state = reset_app("fr")
+        return [
+            "fr",
+            reset_state["ask"],
+            reset_state["chatbots"]["Spinoza"],
+            *[reset_state["chatbots"][tab] for tab in config["tabs"]],
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_qa", "fr"),
+                    open=False,
+                    elem_id=f"accordion-{config['source_mapping'][tab]}",
+                    elem_classes="accordion accordion-agent"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(label=get_text("source_filter_label", "fr"), elem_id="filter-component"),
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_flt", "fr"),
+                    elem_id=f"accordion-filter-{config['source_mapping'][tab]}",
+                    elem_classes="accordion accordion-source"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(value=get_text("source_filter_title", 'fr')),
+            gr.update(value=get_text("source_filter_subtitle", 'fr')),
+            gr.update(value=get_text("question_filter", 'fr')),
+            gr.update(label=get_text("source_informatation_label", "fr"), elem_id="source-component"),
+            gr.update(value=get_text("display_info_desc", "fr")),
+            *[
+                gr.update(value=get_text(f"{config['source_mapping'][tab]}_desc", "fr"))
+                for tab in list(config["tabs"].keys())
+            ],
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_tab", "fr"),
+                    elem_id=f"accordion-{config['source_mapping'][tab]}-tab",
+                    elem_classes="disp-tabs"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(label=get_text("contact_label", "fr")),
+            gr.update(value=about_contact_fr),
+            gr.update(value=""),
+            gr.update(value={}),
+            gr.update(value={}),
+            *[
+                gr.update(value=None) for _ in range(len(agt_input_flt))
+            ]    
+        ]
+
+    def toggle_language_en():
+        reset_state = reset_app("en")
+        return [
+            "en",
+            reset_state["ask"],
+            reset_state["chatbots"]["Spinoza"],
+            *[reset_state["chatbots"][tab] for tab in config["tabs"]],
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_qa", "en"),
+                    open=False,
+                    elem_id=f"accordion-{config['source_mapping'][tab]}",
+                    elem_classes="accordion accordion-agent"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(label=get_text("source_filter_label", "en"), elem_id="filter-component"),
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_flt", "en"),
+                    elem_id=f"accordion-filter-{config['source_mapping'][tab]}",
+                    elem_classes="accordion accordion-source"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(value=get_text("source_filter_title", 'en')),
+            gr.update(value=get_text("source_filter_subtitle", 'en')),
+            gr.update(value=get_text("question_filter", 'en')),
+            gr.update(label=get_text("source_informatation_label", "en"), elem_id="source-component"),
+            gr.update(value=get_text("display_info_desc", "en")),
+            *[
+                gr.update(value=get_text(f"{config['source_mapping'][tab]}_desc", "en"))
+                for tab in list(config["tabs"].keys())
+            ],
+            *[
+                gr.update(
+                    label=get_text(f"agent_{config['source_mapping'][tab]}_tab", "en"),
+                    elem_id=f"accordion-{config['source_mapping'][tab]}-tab",
+                    elem_classes="disp-tabs"
+                )
+                for tab in list(config["tabs"].keys())
+            ],
+            gr.update(label=get_text("contact_label", "en")),
+            gr.update(value=about_contact_en),
+            gr.update(value=""),
+            gr.update(value={}),
+            gr.update(value={}),
+            *[
+                gr.update(value=None) for _ in range(len(agt_input_flt))
+            ]    
+        ]
+
+    button_fr.click(
+        fn=toggle_language_fr,
+        inputs=[],
+        outputs=[
+            current_language, 
+            ask, 
+            chatbots["Spinoza"],
+            *[chatbots[tab] for tab in config["tabs"]],
+            *[accordions_qa[key] for key in accordions_qa.keys()],
+            source_filter_tab,
+            *[accordions_filters[key] for key in accordions_filters.keys()],
+            source_filter_title, 
+            source_filter_subtitle,
+            question_filter,
+            source_information_tab,
+            display_info_desc, 
+            *[agt_desc[key] for key in agt_desc.keys()],
+            *[accordions_inf[key] for key in accordions_inf.keys()],
+            contact_label,
+            contact_info,
+            sources_textbox,
+            filters_state,
+            filters_display,
+            *[agt_input_flt[key] for key in agt_input_flt.keys()]
+        ]
+    )
+
+    button_en.click(
+        fn=toggle_language_en,
+        inputs=[],
+        outputs=[
+            current_language, 
+            ask, 
+            chatbots["Spinoza"],
+            *[chatbots[tab] for tab in config["tabs"]],
+            *[accordions_qa[key] for key in accordions_qa.keys()],
+            source_filter_tab,
+            *[accordions_filters[key] for key in accordions_filters.keys()],
+            source_filter_title, 
+            source_filter_subtitle,
+            question_filter,
+            source_information_tab,
+            display_info_desc, 
+            *[agt_desc[key] for key in agt_desc.keys()],
+            *[accordions_inf[key] for key in accordions_inf.keys()],
+            contact_label,
+            contact_info,
+            sources_textbox,
+            filters_state,
+            filters_display,
+            *[agt_input_flt[key] for key in agt_input_flt.keys()]
+        ]
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch(debug=True, share=True)

assets/about_contact_en.md

@@ -0,0 +1,78 @@
+# The Spinoza Project
+
+ 
+
+Reporters Without Borders and the Alliance of General Information Press have collaborated to create an AI tool tailored to journalism. The project aims to develop a collaborative prototype to assist journalists in their work while ensuring the protection of media data.
+
+ 
+
+The ultimate goal is to provide reliable, pluralistic, and high-quality information. Ekimetrics acts as the technical partner in this project, responsible for identifying appropriate solutions to meet the needs and implementing the prototype.
+
+ 
+
+If you encounter any bugs to report, need additional information, or wish to get in touch with us, feel free to write to us at the following address: [[email protected]](mailto:[email protected]).
+
+ 
+
+---
+
+ 
+
+## How it Works
+
+ 
+
+### How Sources are Retrieved
+
+ 
+
+Here are some details about the **relevance score**. The relevance score is a metric used to evaluate the relevance of retrieved documents to a given query in a *vectorstore*. When a document is stored as a vector, the relevance score indicates how close that document is to the query in terms of vector similarity.
+
+ 
+
+Here’s how it generally works:
+
+ 
+
+**Vector Representation**: Documents and the query are converted into vectors in a vector space.
+
+**Similarity Calculation**: A similarity measure (such as dot product or cosine distance) is used to compare the vectors of the documents with the query vector.
+
+**Relevance Score**: The result of this comparison is the relevance score, which indicates how relevant each document is to the query.
+
+ 
+
+A higher score means the document is more relevant to the query. This allows the retrieved documents to be ranked by their relevance.
+
+ 
+
+### How Summaries are Generated
+
+ 
+
+Summaries are generated using advanced natural language processing (NLP) algorithms, which rely on the following steps:
+
+ 
+
+**Analyzing Source Content**: Relevant documents identified through the relevance score are analyzed.
+
+**Extracting Key Information**: The essential points of the documents are extracted, taking into account their importance and complementarity.
+
+**Automatic Writing**: A coherent and readable summary is generated, following a format tailored to journalists’ needs.
+
+ 
+
+These steps ensure that the proposed summaries are faithful to the original content while being easily usable by the end-users.
+
+ 
+
+---
+
+ 
+
+## Contact Us
+
+
+For any questions or suggestions, feel free to contact us : Spinoza Support" <[email protected]>!
+
+

assets/about_contact_fr.md

@@ -0,0 +1,78 @@
+# Le projet Spinoza
+
+ 
+
+Reporters sans frontières et l’Alliance de la presse d’information générale ont collaboré pour créer un outil d’IA adapté au journalisme. Le projet vise à développer un prototype collaboratif pour aider les journalistes dans leur travail tout en garantissant le respect des données des médias.
+
+ 
+
+L’objectif final est de fournir une information fiable, pluraliste et de haute qualité. Ekimetrics joue dans ce projet le rôle de partenaire technique, en charge de rechercher des solutions adaptées aux besoins identifiés et d’implémenter le prototype.
+
+ 
+
+En cas de bug à reporter, de besoin d’informations supplémentaires ou si vous souhaitez nous contacter, n’hésitez pas à nous écrire à l’adresse suivante : [[email protected]](mailto:[email protected]).
+
+ 
+
+---
+
+ 
+
+## Le fonctionnement
+
+ 
+
+### Comment sont retrouvées les sources
+
+ 
+
+Voici quelques informations sur ce qu’est le **score de pertinence**. Le score de pertinence est une mesure utilisée pour évaluer la pertinence des documents récupérés par rapport à une requête donnée dans un *vectorstore*. Lorsqu’un document est stocké sous forme de vecteur, le score de pertinence indique à quel point ce document est proche de la requête en termes de similarité vectorielle.
+
+ 
+
+Voici comment cela fonctionne généralement :
+
+ 
+
+**Représentation vectorielle** : Les documents et la requête sont convertis en vecteurs dans un espace vectoriel.
+
+**Calcul de similarité** : Une mesure de similarité (comme le produit scalaire ou la distance cosinus) est utilisée pour comparer les vecteurs des documents avec celui de la requête.
+
+**Score de pertinence** : Le résultat de cette comparaison est le score de pertinence, qui indique à quel point chaque document est pertinent par rapport à la requête.
+
+ 
+
+Un score plus élevé signifie que le document est plus pertinent pour la requête. Cela permet de classer les documents récupérés en fonction de leur pertinence.
+
+ 
+
+### Comment sont générées les synthèses
+
+ 
+
+Les synthèses sont générées grâce à l’utilisation d’algorithmes avancés de traitement du langage naturel (NLP) qui s’appuient sur les informations suivantes :
+
+ 
+
+**Analyse des contenus sources** : Les documents pertinents identifiés grâce au score de pertinence sont analysés.
+
+**Extraction des informations clés** : Les points essentiels des documents sont extraits, en tenant compte de leur importance et de leur complémentarité.
+
+**Rédaction automatique** : Une synthèse cohérente et lisible est produite, en respectant un format adapté aux besoins des journalistes.
+
+ 
+
+Ces étapes garantissent que les synthèses proposées soient fidèles aux contenus originaux tout en étant facilement exploitables par les utilisateurs.
+
+ 
+
+---
+
+ 
+
+## Contactez-nous
+
+
+Pour toute question ou suggestion, n’hésitez pas à nous contacter : Spinoza Support" <[email protected]> !
+
+

assets/source_information_en.md

@@ -0,0 +1,34 @@
+Here's a brief introduction to the data sources accessible by the different agents.
+
+1. **Science**: This tool consists of IPCC and IPBES reports.
+
+2. **Law**: This tool is based on French law and includes 21 "codes" that were modified by the 2021 "Climate Law".
+
+3. **Public Organizations**: This tool queries the French national low-carbon strategy (SNBC).
+
+4. **ADEME**: This tool is dedicated to ADEME data, and we have selected different categories of reports:
+
+   - Guides made available to the general public
+   - Experience reports on new technologies
+   - Studies and research on local impacts, institutional documents (analyses commissioned by France & activity reports)
+   - Sectoral transition plans for the most emitting industrial sectors (glass, paper, cement, steel, aluminum, chemistry, sugar)
+
+5. **Press**: In 2023, hundreds of thousands of articles from 212 press titles were analyzed to identify those dedicated to Ecological Transition. A documentary query of more than 300 keywords helped select articles mentioning these terms in the title, header, subheadings, or multiple times in the text. The chosen articles were specifically focused on ecological transition and not mere mentions. Once deduplicated and proportionally distributed among media groups, articles were randomly selected, without relying on criteria of size, format, or content, reaching a total of 28,450 articles.
+
+6. **AFP**: More than 700 AFP documents were also collected:
+    - References and boxes: These educational formats contain an average of 400 to 600 words. Structured in 3 to 5 sub-sections, their objective is to clearly and concisely explain a current event.
+    - Dispatches: These articles are written by AFP and cover real-time news, following an inverted pyramid approach (essential information first). Their length varies from a few words ("alert") to about 600 to 700 words for more detailed articles ("general paper").
+    - Fact-checking: Verification of facts related to current events.
+    - General papers
+
+<br>
+
+Here is some information about what a relevance score is. The relevance score is a metric used to evaluate the relevance of documents retrieved in relation to a given query within a **vectorstore**. When a document is stored as a vector, the relevance score indicates how closely that document aligns with the query in terms of vector similarity.
+
+Here's how it generally works:
+
+   - **Vector Representation**: Documents and the query are converted into vectors in a vector space.
+   - **Similarity Calculation**: A similarity measure (such as dot product or cosine distance) is used to compare document vectors with the query vector.
+   - **Relevance Score**: The result of this comparison is the relevance score, which indicates how relevant each document is to the query.
+
+A higher score means the document is more relevant to the query. This allows ranking retrieved documents based on their relevance. 

assets/source_information_fr.md

@@ -0,0 +1,35 @@
+Voici une brève introduction aux sources de données accessibles par les différents agents.
+
+1. **Science** : cet outil est composé des rapports du GIEC et de l'IPBES.
+
+2. **Loi** : cet outil est basé sur le droit français et regroupe 21 des "codes" qui ont été modifiés par la "loi climat" de 2021.
+
+3. **Organismes Publics** : cet outil interroge la politique nationale française de la stratégie bas carbone (SNBC).
+
+4. **ADEME** : cet outil est dédié aux données de l'ADEME, et nous avons sélectionné différentes catégories de rapports :
+
+   - Guides mis à disposition du grand public
+   - Rapports d’expériences sur les nouvelles technologies
+   - Études et recherches sur les impacts locaux, documents institutionnels (analyses commandées par la France & rapports d'activité)
+   - Plans de transition sectoriels pour les secteurs industriels les plus émetteurs (verre, papier, ciment, acier, aluminium, chimie, sucre)
+
+5. **Presse** : En 2023, des centaines de milliers d'articles provenant de 212 titres de presse ont été analysés pour repérer ceux consacrés à la Transition Écologique. Une requête documentaire de plus de 300 mots-clés a permis de sélectionner les articles mentionnant ces termes dans le titre, le chapo, les intertitres ou plusieurs fois dans le texte. Les articles choisis étaient spécifiquement axés sur la transition écologique et non de simples mentions. Une fois dédupliqués et répartis proportionnellement entre les groupes de médias, des articles ont été tirés aléatoirement, sans se baser sur des critères de taille, de format ou de contenu pour arriver à un total de 28450 artivles 
+
+6. **AFP** : Plus de 700 documents de l'AFP ont aussi été collectées : 
+    - Repères et encadrés : Ces formats pédagogiques contiennent en moyenne entre 400 et 600 mots. Structurés en 3 à 5 sous-parties, leur objectif est d’expliquer de manière claire et concise un fait d’actualité.
+    - Dépêches : Ces articles sont rédigés par l’AFP et traitent de l’actualité en temps réel, selon une approche de pyramide inversée (les informations essentielles en premier). Leur longueur varie de quelques mots ("alerte") à environ 600 à 700 mots pour les articles plus détaillés ("papier général").
+    - Fact-checking : Vérification des faits en lien avec l’actualité.
+    - Papiers généraux 
+
+
+<br>
+
+Voici quelques informations sur ce qu’est le score de pertinence. Le score de pertinence est une mesure utilisée pour évaluer la pertinence des documents récupérés par rapport à une requête donnée dans un **vectorstore**. Lorsqu’un document est stocké sous forme de vecteur, le score de pertinence indique à quel point ce document est proche de la requête en termes de similarité vectorielle.
+
+Voici comment cela fonctionne généralement :
+
+   - **Représentation vectorielle** : Les documents et la requête sont convertis en vecteurs dans un espace vectoriel.
+   - **Calcul de similarité** : Une mesure de similarité (comme le produit scalaire ou la distance cosinus) est utilisée pour comparer les vecteurs des documents avec celui de la requête.
+   - **Score de pertinence** : Le résultat de cette comparaison est le score de pertinence, qui indique à quel point chaque document est pertinent par rapport à la requête.
+
+Un score plus élevé signifie que le document est plus pertinent pour la requête. Cela permet de classer les documents récupérés en fonction de leur pertinence.

assets/style.css

@@ -0,0 +1,326 @@
+.message {
+    font-size: 14px !important;
+}
+
+a {
+    text-decoration: none;
+    color: inherit;
+}
+
+.card {
+    background-color: white;
+    border-radius: 10px;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    overflow: hidden;
+    display: flex;
+    flex-direction: column;
+    margin: 20px;
+}
+
+.card-content {
+    padding: 20px;
+}
+
+.card-content h2 {
+    font-size: 14px !important;
+    font-weight: bold;
+    margin-bottom: 10px;
+    margin-top: 0px !important;
+    color: #dc2626 !important;
+    ;
+}
+
+.card-content p {
+    font-size: 12px;
+    margin-bottom: 0;
+}
+
+.card-footer {
+    background-color: #f4f4f4;
+    font-size: 10px;
+    padding: 10px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.card-footer span {
+    flex-grow: 1;
+    text-align: left;
+    color: #999 !important;
+}
+
+.message.user {
+    /* background-color:#7494b0 !important; */
+    border: none;
+    /* color:white!important; */
+}
+
+.message.bot {
+    /* background-color:#f2f2f7 !important; */
+    border: none;
+}
+
+.message.svelte-x4qvqz .prose {
+  font-size: small;
+}
+
+.message-row.panel.svelte-x4qvqz.svelte-x4qvqz {
+  margin:0;
+  padding:calc(var(--spacing-xl) * 2) calc(var(--spacing-xxl) * 1);
+}
+
+@media screen {
+    div#sources-textbox {
+        height: calc(95vh - 160px) !important;
+        overflow-y: auto !important;
+        scrollbar-width: none;
+        -ms-overflow-style: none;
+    }
+
+    div#sources-textbox::-webkit-scrollbar {
+        width: 0;
+        height: 0;
+    }
+
+    div.svelte-iyf88w {
+        scrollbar-width: none;
+        height: calc(-180px + 95vh) !important;
+        overflow-y: auto;
+    }
+
+    div.svelte-iyf88w::-webkit-scrollbar {
+        width: 0;
+        height: 0;
+    }
+
+    div#chatbot-row {
+        height: calc(95vh - 100px) !important;
+    }
+
+    .form {
+        position: relative;
+        top: 10px;
+    }
+
+    #center-panel {
+        flex-grow: 2;
+        min-width: min(320px, 100%);
+        height: calc(-100px + 95vh) !important;
+    }
+
+    .compact.svelte-vt1mxs,
+    .panel.svelte-vt1mxs {
+        border:solid var(--panel-border-width) var(--panel-border-color);
+        border-radius:var(--container-radius);
+        background:var(--panel-background-fill);
+        padding:var(--spacing-lg);
+        height: calc(-100px + 95vh) !important;
+    }
+
+    .accordion-agent.spinoza-agent {
+        height: 15cm;
+    }
+
+    .accordion-agent.spinoza-agent > button > span {
+        color: #000000;
+        font-size: large;
+        font-weight: bold;
+    }
+
+    .accordion-agent > button > span {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+    .accordion-source > button > span {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+}
+
+textarea.scroll-hide {
+    max-height: 42px;
+}
+
+footer {
+    position: fixed;
+    left: 0;
+    bottom: 0;
+    width: 100%;
+    text-align: center;
+}
+
+.footer-ekimetrics {
+    align-items: center;
+    margin-left: var(--size-2);
+    font-size: small;
+}
+
+@media screen and (max-width: 767px) {
+    /* Your mobile-specific styles go here */
+    div#chatbot-row{
+        height:500px !important;
+    }
+
+    #submit-button{
+        padding:0px !important;
+        min-width: 80px;
+    }
+
+    /* This will hide all list items */
+    div.tab-nav button {
+        display: none !important;
+    }
+
+    /* This will show only the first list item */
+    div.tab-nav button:first-child {
+        display: block !important;
+    }
+    
+    /* This will show only the first list item */
+    div.tab-nav button:nth-child(2) {
+        display: block !important;
+    }
+    
+    #right-panel button{
+        display: block !important;
+    }
+}
+
+.tabitem {
+    border: none !important;
+}
+
+.other-tabs>div {
+    padding-left: 40px;
+    padding-right: 40px;
+    padding-top: 10px;
+}
+
+.tab-nav {
+    border: none !important;
+}
+
+.tab-nav>button.selected {
+    color: #4b8ec3;
+    font-weight: bold;
+    border: none;
+}
+
+#science-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+#law-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+#org-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+#ademe-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+#press-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+#afp-tab>button.selected {
+    color: #4b8ec3 !important;
+    font-weight: bold !important;
+    border: 2px solid #4b8ec3 !important;
+    border-radius: 5px !important; 
+    padding: 5px 15px !important;
+}
+
+.header-row {
+    position: fixed;
+    top: 10px;
+    right: 10px;
+    z-index: 1000;
+    display: flex;
+    justify-content: flex-end;
+    gap: 10px;
+}
+
+.header-row button {
+    flex: none !important;
+    width: auto !important;
+    min-width: 0 !important;
+}
+
+.header-row .lang-button {
+    flex-grow: 0 !important;
+    flex-shrink: 0 !important;
+    flex-basis: 35px !important;
+    width: 35px !important;
+    height: 35px !important;
+    padding: 0 !important;
+    align-items: center !important;
+    justify-content: center !important;
+    background: none !important;
+    border: none !important;
+    box-shadow: none !important;
+}
+
+.lang-button.selected {
+    background-color: #2196F3 !important;
+    color: white !important;
+    border-color: #1976D2 !important;
+}
+
+.header-row .lang-button:hover {
+    background: none !important;
+    border: none !important;
+    box-shadow: none !important;
+}
+.main-row {
+    margin-top: 50px !important; 
+}
+
+#input-textbox>label>textarea {
+    border-radius: 40px;
+    padding-left: 30px;
+    resize: none;
+}
+
+#input-message>div {
+    border: none;
+}
+
+.loader {
+  animation: blink 2s linear infinite;
+}
+
+@keyframes blink {
+        0% {
+          opacity: 0;
+        }
+        100% {
+          opacity: 1;
+        }
+      }

assets/utils_javascript.py

@@ -0,0 +1,72 @@
+def update_footer():
+    return """
+    function update_footer() {
+        var footer = document.querySelector('footer')
+        footer.innerHTML = "<a href='https://rsf.org/fr' class='footer-ekimetrics' target='_blank' rel='noreferrer'>À l'initiative de RSF</a>";
+        footer.innerHTML += "<a href='https://www.alliancepresse.fr/' class='footer-ekimetrics' target='_blank' rel='noreferrer'>et l'Alliance Presse</a>";
+        footer.innerHTML += "<div class='footer-ekimetrics'> · </div>";
+        footer.innerHTML += "<a href='https://www.culture.gouv.fr/' class='footer-ekimetrics' target='_blank' rel='noreferrer'>Avec le soutien du Ministère de la Culture</a>";
+        footer.innerHTML += "<div class='footer-ekimetrics'> · </div>";
+        footer.innerHTML += "<a href='https://ekimetrics.com/' class='footer-ekimetrics' target='_blank' rel='noreferrer'>Conçu par Ekimetrics</a>";             
+    }
+    """
+
+
+def accordion_trigger():
+    return """
+    function accordion_trigger() {
+        var input_textbox = document.getElementById("input-textbox");
+        input_textbox.addEventListener('keyup', function (e) {
+            if (e.key === 'Enter' || e.keyCode === 13) {
+                document.querySelectorAll(".loader, .loader-helper").forEach(el => el.remove());
+                var accordions = document.querySelectorAll('.accordion-agent');
+                accordions.forEach(function (accordion) {
+                    var agentName = "Agent " + accordion.id.split('-')[1];
+                    var buttonSpan = accordion.querySelector('button > span');                    
+                    if (!accordion.classList.contains('spinoza-agent')) {
+                        buttonSpan.textContent = agentName;
+                        buttonSpan.innerHTML += "<span class='loader-helper'> - </span><span class='loader'>loading</span>";
+                    }
+                });
+            }
+        });
+    }
+    """
+
+
+def accordion_trigger_end():
+    return """
+    function accordion_trigger_end() {
+        var accordions = document.querySelectorAll('.accordion-agent');
+
+        accordions.forEach(function (accordion) {
+            if (!accordion.classList.contains('spinoza-agent')) {
+                var agentName = "Agent " + accordion.id.split('-')[1];
+                var buttonSpan = accordion.querySelector('button > span');
+                buttonSpan.textContent = agentName + " - ready";
+            }
+        });
+    }
+    """
+
+
+def accordion_trigger_spinoza():
+    return """
+    function accordion_trigger_spinoza() {
+        var accordion_spinoza = document.querySelector('.spinoza-agent');
+        document.querySelectorAll(".loader, .loader-helper").forEach(el => el.remove());
+        var buttonSpan = accordion_spinoza.querySelector('button > span');
+        buttonSpan.textContent = "Spinoza";
+        buttonSpan.innerHTML += "<span class='loader-helper'> - </span><span class='loader'>generating</span>";
+    }
+    """
+
+
+def accordion_trigger_spinoza_end():
+    return """
+    function accordion_trigger_spinoza_end() {
+        var accordion_spinoza = document.querySelector('.spinoza-agent');
+        var buttonSpan = accordion_spinoza.querySelector('button > span');
+        buttonSpan.textContent = "Spinoza - ready";
+    }
+    """

pyproject.toml

@@ -0,0 +1,32 @@
+[tool.poetry]
+name = "spinoza_project"
+version = "0.1.0"
+description = ""
+authors = ["Miguel Omenaca Muro <[email protected]>"]
+readme = "README.md"
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.10"
+gradio = "4.37.2"
+sentence-transformers = "2.2.2"
+msal = "^1.28.1"
+qdrant-client = "^1.9.1"
+loadenv = "^0.1.1"
+datasets = "^2.20.0"
+transformers = "4.39.0"
+azure-search-documents = "^11.4.0"
+azure-identity = "^1.17.1"
+load-dotenv = "^0.1.0"
+python-dotenv = "^1.0.1"
+langchain-groq = "^0.2.1"
+langchain-openai = "^0.2.6"
+langchain-community = "^0.3.5"
+langchain = "^0.3.7"
+huggingface-hub = "< 0.26"
+fastapi = "0.111.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

spinoza_project/config.yaml

@@ -0,0 +1,87 @@
+demo_name: Spinoza Q&A
+
+default_databases: "SpinozaProject/spinoza-database"
+## Veuillez préciser le chemin du répertoire dans lequel est stocké les fichiers pickle des bases de données à ajouter
+database_hf: ""
+# Exemple >>> database_hf: "user_repo/database_name" 
+
+databases_pickle_files:
+  ## Veuillez spécifier le nom EXACT des fichiers pickles ajoutés dans les répertoires ci-dessus correspondant à chaque base de données
+  science_agent: "database_Science.pickle"
+  law_agent: "database_Loi.pickle"
+  pub_institutions_agent: "database_Organismes_publics.pickle"
+  ademe_agent: "database_ADEME.pickle"
+  # Exemple >>> new_agent: "database_Nouvelle_bdd.pickle"
+
+prompts_path_list:
+  ## Veuillez spécifier le chemin des prompts associés à chaque agent au sein du répertoire du projet Spinoza 
+  science_agent: "./spinoza_project/prompt_Science.yaml"
+  law_agent: "./spinoza_project/prompt_Loi.yaml"
+  pub_institutions_agent: "./spinoza_project/prompt_Organismes_publics.yaml"
+  ademe_agent: "./spinoza_project/prompt_ADEME.yaml"
+  # Exemple >>> new_agent: "./spinoza_project/prompt_Template.yaml"
+
+tabs:
+  ## Veuillez associer ici le nom précis des bases de données que vous souhaitez utiliser/afficher, ainsi que leur description
+  science_agent: "Cet outil s’appuie sur les rapports du **GIEC**, de l'**IPBES** et de l'**UNESCO**. Nous avons sélectionné ces organismes de sciences consensuelles afin de garantir la fiabilité maximale des réponses obtenues."
+  law_agent: "Cet outil regroupe 21 des codes juridiques français modifiés par la **'loi climat' de 2021**. Il est important de souligner qu’il offre une vision limitée de la réglementation française, ne prenant pas en compte, par exemple, les réglementations régionales, les jurisprudences ou les lois européennes. Son objectif est de mettre en perspective des questions avec des articles spécifiques, sans prétendre fournir de réponses légales définitives."
+  pub_institutions_agent: "Cet outil interroge les documents publics de certains organismes publics. Il ne s’agit pas d’une vision exhaustive de leurs données. Si vous estimez que certains ajouts sont nécessaires, n’hésitez pas à nous les transmettre à l’adresse suivante : **[[email protected]](mailto:[email protected])"
+  ademe_agent:
+    "Cet outil est dédié aux données de l’ADEME et évoluera en intégrant directement une base de données administrée par l’ADEME dans le cadre de son projet SofIA. Les documents actuellement disponibles couvrent des typologies variées, notamment :
+
+    Guides à destination du grand public ;
+
+    - Rapports d’expériences sur les nouvelles technologies ;
+
+    - Études et recherches sur les impacts locaux, ainsi que des documents institutionnels (analyses commandées par la France et rapports d’activité) ;
+
+    - Plans de transition sectoriels pour les industries les plus émettrices (verre, papier, ciment, acier, aluminium, chimie, sucre)."
+  # Exemple >>> new_agent: "*Description de la base de données*"
+
+en_description:
+  ## Veuillez renseigner ici la description anglaise des mêmes bases de données 
+  science_agent: "This tool relies on reports from internationally recognized scientific organizations, including the **IPCC (Intergovernmental Panel on Climate Change)**, **IPBES (Intergovernmental Science-Policy Platform on Biodiversity and Ecosystem Services)**, and **UNESCO**. We have selected these sources for their rigorous, consensus-based scientific approaches, ensuring the highest reliability of the answers provided."
+  law_agent: "This tool integrates information from 21 French legal codes amended by the **2021 'Climate and Resilience Law'**, a major legislative effort to address climate change in France. It is important to note that this tool provides a limited view of French legal frameworks, as it does not include regional regulations, case law, or European Union laws. Its primary objective is to contextualize user questions with relevant legal articles rather than offering definitive legal advice."
+  pub_institutions_agent: "This tool queries publicly available documents from selected French public institutions. While the coverage is not exhaustive, it provides valuable insights into these organizations’ policies and actions. If you believe additional sources should be included, please feel free to send your suggestions to: **[email protected]**."
+  ademe_agent:
+   "This tool is dedicated to data from ADEME (French Agency for Ecological Transition), a key institution supporting France’s environmental and energy transition policies. It will soon evolve to incorporate a database directly managed by ADEME as part of their SofIA project. Currently, the documents available cover a broad range of topics, including:
+
+    - **Public guides**: Educational materials aimed at the general public.
+
+    - **Technological pilot projects**: Reports on experimental applications of emerging technologies.
+
+    - **Research and studies**: Analyses of local environmental impacts and institutional reports commissioned by the French government.
+
+    - **Sector-specific transition plans**: Detailed strategies for decarbonizing high-emission industries such as glass, paper, cement, steel, aluminum, chemicals, and sugar production."
+  # Exemple >>> new_agent: "*New database's english description*"
+  
+source_mapping:
+  ## Veuillez renseigner ici le nom que vous souhaiteriez donner à chaque base de données dans l'outil Spinoza
+  science_agent: "Science"
+  law_agent: "Loi"
+  pub_institutions_agent: "Organismes publics"
+  ademe_agent: "ADEME"
+  # Exemple >>> new_agent: "Nom de la base de données"
+
+en_names:
+  ## Veuillez renseigner ici les équivalents anglais du nom des bases de données 
+  science_agent: "Science"
+  law_agent: "Law"
+  pub_institutions_agent: "Public Institutions"
+  ademe_agent: "ADEME"
+  # Exemple >>> new_agent: "Database's name"
+
+## Veuillez renseigner le modèle Groq que devront utiliser les agents pour répondre aux questions
+groq_model_name: "llama-3.3-70b-versatile" # llama-3.1-8b-instant / llama-3.3-70b-versatile/ llama-3.1-70b-versatile / llama-3.2-3b-preview
+
+query_preprompt: "query: "
+passage_preprompt: "passage: "
+embedding_model: "intfloat/multilingual-e5-base"
+num_document_retrieved: 5
+min_similarity: 0.05
+
+## Chat API
+user_token: "user"
+assistant_token: "assistant"
+system_token: "system"
+stop_token: "" ## useless in chat mode

spinoza_project/database_building/Data.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76da75de415d4452746e75e3748617f6be00d3175ff078324a6749c0bcd51f9c
+size 571471

spinoza_project/hackathon/Data.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3450f9f6135001d44470f81de741d331f29aa0509695b257637b8c82c5df40a3
+size 570556

spinoza_project/prompt_ADEME.yaml

@@ -0,0 +1,79 @@
+role_instruction:
+  prompt:
+    [
+      "You are Spinoza Fact Checker, an AI Assistant developed by Ekimetrics.",
+      "Your role is to answer questions factually based on the documents provided to you, which may contain opinions, recommendations, or analyses.",
+      "You act as a sustainble journalist, providing structured, factual, and concise responses while citing your sources and mentioning linked articles.",
+      "If a question is not related to climate, do not answer it and state that the question falls outside your expertise."
+    ]
+  type: "system"
+
+source_prompt:
+  prompt:
+    [
+      "Below are several documents formatted as: Doc X \n textual content.",
+      "<documents>",
+      "{sources}",
+      "</documents>",
+      "",
+      "Treat the textual content as providing relevant opinions, recommendations, or analyses.",
+      "For each fact or analysis used in your response, reference the source clearly (e.g., [Doc 2]: some analysis from Doc 2).",
+      "Incorporate all the relevant content from the documents to provide a well-rounded response.",
+      "Disregard any information that is irrelevant to the question at hand.",
+      "If you do not have relevant documents or they lack context, state that you don't have enough context to answer.",
+      "If the question is not related to climate, explain that it falls outside your scope of expertise."
+    ]
+  type: "instruction"
+
+question_answering_prompt:
+  prompt:
+    [
+      "Répondez à la question suivante : {question}.",
+      "Si votre réponse est basée sur un article spécifique, formulez-la de la manière suivante : 'Selon l'article [nom de l'article], [réponse]'.",
+      "Si la réponse s'appuie sur plusieurs articles, utilisez un point par article.",
+      "Citez les passages pertinents des sources lorsque cela est nécessaire.",
+      "Si la question n'est pas liée à des questions environnementales, dites que vous ne pouvez pas y répondre en raison de l'irrélevance des sources fournies.",
+      "Si la question n'est pas liée à l'environnement, dites explicitement que la question ne relève pas de votre domaine d'expertise.",
+      "Répondez impérativement en {language}."
+    ]
+  type: "prompt"
+
+
+reformulation_prompt:
+  prompt:
+    [
+      "Reformulez le message de l'utilisateur en une question autonome et concise.",
+      "Reformulez strictement en {language}.",
+      "La question reformulée doit être claire et suffisamment précise pour interroger des textes publics provenant d'analyses.",
+      "Si pertinent, utilisez le résumé de la conversation pour ajouter du contexte.",
+      "Si la question est trop vague, reformulez-la telle qu'elle est sans faire d'hypothèses supplémentaires.",
+      "Si la question n'est pas liée au climat ou à la réglementation environnementale, indiquez qu'elle est hors de votre domaine d'expertise.",
+      "",
+      "Exemples:",
+      "---",
+      "user:",
+      "Quels sont les avis sur la taxe carbone?",
+      "",
+      "assistant:",
+      "Quels avis et recommandations sont formulés au sujet de l'application de la taxe carbone en France?",
+      "---",
+      "user:",
+      "Quelles recommandations pour l'indice de réparabilité?",
+      "",
+      "assistant:",
+      "Quelles recommandations les publications fournissent-elles au sujet de l'indice de réparabilité des produits?",
+      "---",
+      "user:",
+      "Quels enjeux autour de l'eau?",
+      "",
+      "assistant:",
+      "Quelles analyses ou avis sont formulés au sujet de la gestion de l'eau dans les publications disponibles?",
+      "---",
+      "user:",
+      "{question}",
+      "---",
+      "Si la question n'est pas liée au climat ou à la réglementation environnementale:",
+      "assistant:",
+      "La question posée ne relève d'enjeux environnementaux, je ne peux donc pas y répondre."
+    ]
+  type: "prompt"

spinoza_project/prompt_Loi.yaml

@@ -0,0 +1,85 @@
+role_instruction:
+  prompt:
+    [
+      "You are Spinoza law analyst, an AI Assistant developed by Ekimetrics.",
+      "Your primary role is to provide factually accurate responses about climate change, based solely on the documents provided.",
+      "You act as a legal expert, delivering structured, factual, concise, and sourced responses.",
+      "Always quote your sources by mentioning document titles and linked articles when applicable.",
+      "Your only focus is climate change"
+    ]
+  type: "system"
+
+source_prompt:
+  prompt:
+    [
+      "Below are several documents formatted as: Doc X \n textual content.",
+      "<documents>",
+      "{sources}",
+      "</documents>",
+      "",
+      "Treat the content of the provided documents as authoritative.",
+      "For each fact presented in your answer, reference the source explicitly (e.g., [Doc 2]: some fact from Doc 2).",
+      "Incorporate all relevant information from the documents to form a complete response.",
+      "Disregard any irrelevant facts or information that do not directly pertain to the question.",
+      "If no relevant documents are provided or if they lack sufficient context, try to display what could be relevant from a climate perspective.",
+      "If you do so, disclose at the begining of your awnser that you are trying to open on another subject, and remind the user that you are using only data linked to climate and that your awnser can be incomplete",
+    ]
+  type: "instruction"
+
+question_answering_prompt:
+  prompt:
+    [
+      "Respond to the following question: {question}.",
+      "If your response is based on a specific article, phrase it as follows: 'Selon l'article [name of the article], [answer].'",
+      "When the answer references multiple articles, structure the response with bullet points, each citing the corresponding article.",
+      "Quote text from the sources when relevant.",
+      "If the question is unrelated to legal matters, explicitly state that you cannot provide an answer based on the given sources.",
+      "Answer imperatively in {language}.",
+    ]
+  type: "prompt"
+  
+reformulation_prompt:
+  prompt:
+    [
+      "Reformulez le message de l'utilisateur en une question autonome et concise en {language}",
+      "La question reformulée doit être claire et suffisamment précise pour interroger des textes juridiques concernant la réglementation environnementale.",
+      "Si pertinent, utilisez le résumé de la conversation pour ajouter du contexte.",
+      "Si la question est trop éloignée du changement, reformule-la de manière à ce qu'elle concerne également le changement climatique",
+      "Si la question est trop vague, reformulez-la telle qu'elle est sans faire d'hypothèses supplémentaires.",
+      "",
+      "Exemples:",
+      "---",
+      "user:",
+      "Applique-t-on une taxe carbone?",
+      "",
+      "assistant:",
+      "La taxe carbone est-elle appliquée en France?",
+      "---",
+      "user:",
+      "Quelle obligation produit l'indice de réparabilité des produits?",
+      "",
+      "assistant:",
+      "Quelles sont les exigences légales concernant l'indice de réparabilité des produits?",
+      "---",
+      "user:",
+      "Quelles obligations de faire un bilan carbone?",
+      "",
+      "assistant:",
+      "Quand doit-on réaliser un bilan des émissions de gaz à effet de serre?",
+      "---",
+      "user:",
+      "Quels enjeux autour de l'eau?",
+      "",
+      "assistant:",
+      "Quels articles réglementent la consommation d'eau et que stipulent-ils?",
+      "---",
+      "user:",
+      "Peut on se baigner dans la gironde ?",
+      "",
+      "assistant:",
+      "Quelles sont les réglementations concernant la baignade en Gironde?",
+      "---",
+      "user:",
+      "{question}",
+    ]
+  type: "prompt"

spinoza_project/prompt_Organismes_publics.yaml

@@ -0,0 +1,78 @@
+role_instruction:
+  prompt:
+    [
+      "You are Spinoza Fact Checker, an AI Assistant developed by Ekimetrics.",
+      "Your role is to answer questions factually based on the documents provided to you, which may contain opinions, recommendations, or analyses.",
+      "You act as a journalist, providing structured, factual, and concise responses while citing your sources and mentioning linked articles.",
+      "If a question is not related to climate, do not answer it and state that the question falls outside your expertise."
+    ]
+  type: "system"
+
+source_prompt:
+  prompt:
+    [
+      "Below are several documents formatted as: Doc X \n textual content.",
+      "<documents>",
+      "{sources}",
+      "</documents>",
+      "",
+      "Treat the textual content as providing relevant opinions, recommendations, or analyses.",
+      "For each fact or analysis used in your response, reference the source clearly (e.g., [Doc 2]: some analysis from Doc 2).",
+      "Incorporate all the relevant content from the documents to provide a well-rounded response.",
+      "Disregard any information that is irrelevant to the question at hand.",
+      "If you do not have relevant documents or they lack context, state that you don't have enough context to answer.",
+      "If the question is not related to climate, explain that it falls outside your scope of expertise."
+    ]
+  type: "instruction"
+
+question_answering_prompt:
+  prompt:
+    [
+      "Répondez à la question suivante : {question}.",
+      "Si votre réponse est basée sur un article spécifique, formulez-la de la manière suivante : 'Selon l'article [nom de l'article], [réponse]'.",
+      "Si la réponse s'appuie sur plusieurs articles, utilisez un point par article.",
+      "Citez les passages pertinents des sources lorsque cela est nécessaire.",
+      "Si la question n'est pas liée à des questions environnementales, dites que vous ne pouvez pas y répondre en raison de l'irrélevance des sources fournies.",
+      "Si la question n'est pas liée au climat ou à la réglementation environnementale, dites explicitement que la question ne relève pas de votre domaine d'expertise.",
+      "Répondez impérativement en {language}."
+    ]
+  type: "prompt"
+
+
+reformulation_prompt:
+  prompt:
+    [
+      "Reformulez le message de l'utilisateur en une question autonome et concise en {language}",
+      "La question reformulée doit être claire et suffisamment précise pour interroger des textes publics provenant d'analyses.",
+      "Si pertinent, utilisez le résumé de la conversation pour ajouter du contexte.",
+      "Si la question est trop vague, reformulez-la telle qu'elle est sans faire d'hypothèses supplémentaires.",
+      "Si la question n'est pas liée au climat ou à la réglementation environnementale, indiquez qu'elle est hors de votre domaine d'expertise.",
+      "",
+      "Exemples:",
+      "---",
+      "user:",
+      "Quels sont les avis sur la taxe carbone?",
+      "",
+      "assistant:",
+      "Quels avis et recommandations sont formulés au sujet de l'application de la taxe carbone en France?",
+      "---",
+      "user:",
+      "Quelles recommandations pour l'indice de réparabilité?",
+      "",
+      "assistant:",
+      "Quelles recommandations les publications fournissent-elles au sujet de l'indice de réparabilité des produits?",
+      "---",
+      "user:",
+      "Quels enjeux autour de l'eau?",
+      "",
+      "assistant:",
+      "Quelles analyses ou avis sont formulés au sujet de la gestion de l'eau dans les publications disponibles?",
+      "---",
+      "user:",
+      "{question}",
+      "---",
+      "Si la question n'est pas liée au climat ou à la réglementation environnementale:",
+      "assistant:",
+      "La question posée ne relève d'enjeux environnementaux, je ne peux donc pas y répondre."
+    ]
+  type: "prompt"

spinoza_project/prompt_Science.yaml

@@ -0,0 +1,71 @@
+role_instruction:
+  prompt:
+    [
+      "You are Spinoza Fact Checker, an AI Assistant developed by Ekimetrics.",
+      "Your role is to answer questions factually based on the sources provided to you.",
+      "You act as a scientific expert, providing structured, factual, and concise responses while citing your sources.",
+      "If a question is not related to science, state that it is outside your expertise and do not provide an answer."
+    ]
+  type: "system"
+
+source_prompt:
+  prompt:
+    [
+      "Here are some documents formatted as: Doc X \n textual content.",
+      "<documents>",
+      "{sources}",
+      "</documents>",
+      "",
+      "Treat the textual content as authoritative.",
+      "Reference the source of each fact before stating it (e.g., [Doc 2]: some fact from Doc 2).",
+      "Incorporate all the relevant information from the documents to form your response.",
+      "Ignore irrelevant information.",
+      "If you have no documents or if they are irrelevant, state that you don't have enough context to answer.",
+      "If the question is not related to science, state that it is outside your expertise."
+    ]
+  type: "instruction"
+
+question_answering_prompt:
+  prompt:
+    [
+      "Answer the following question: {question}.",
+      "Use bullet points to organize your answer.",      
+      "Translate your answer in {language} and return only that translated answer.",
+      "Avoid returning any text that is not in {language}.",
+    ]
+  type: "prompt"
+
+reformulation_prompt:
+  prompt: [
+      "Reformulate the following user message to be a short standalone question.",
+      "The question should be asked in {language}",
+      "The question is related to science.",
+      "Use the conversation summary to add context if relevant.",
+      "If the question is too vague, just state it as it is.",
+      "This reformulated question will be used to retrieve scientific documents or data.",
+      "",
+      "Examples:",
+      "---",
+      "user:",
+      "La technologie nous sauvera-t-elle?",
+      "",
+      "assistant:",
+      "Can technology help humanity mitigate the effects of climate change?",
+      "---",
+      "user:",
+      "Quelles sont nos réserves en combustibles fossiles?",
+      "",
+      "assistant:",
+      "What are the current reserves of fossil fuels and how long will they last?",
+      "---",
+      "user:",
+      "Quels sont les principaux facteurs du changement climatique?",
+      "",
+      "assistant:",
+      "What are the main causes of climate change in the last century?",
+      "---",
+      "user:",
+      "{question}",
+      "",
+    ]
+  type: "prompt"

spinoza_project/prompt_Spinoza.yaml

@@ -0,0 +1,22 @@
+prompt: 
+  [
+    "You are a factual journalist specialized in summarizing expert answers from technical, legal, and media sources.",
+    "Your role is to provide a concise and fact-based synthesis based on the following question:",
+    "{question}",
+    "",
+    "And the expert answers provided:",
+    "{answers}",
+    "",
+    "- Always use the [Doc i] format to reference sources, where 'i' is the document number. This must be done for every fact, even when multiple documents agree, for example: [Doc 1, Doc 2, Doc 3].",
+    "- Do not mention sources that provide no relevant information. Focus exclusively on sources that contain useful information.",
+    "- Do not mention that a source lacks information or context. Only present a sources when it has relavant informations.",
+    "- If different sources provide contrasting information, perspectives or interpretations, highlight these differences factually and objectively.",
+    "- When using legal answers, track and reference specific articles of the law.",
+    "- Structure the synthesis with clear sections, using markdown with **bold** headlines and good spacing for readability.",
+    "- Start by presenting any **contradictions** in the sources (if any). If no contradictions exist, just pass to the next part.",
+    "- Then provide a **general summary** of the common points across the sources.",
+    "- End by detailing any **interesting elements** that may be useful for journalists writing an article. Include direct quotes when necessary.",
+    "- Always suggest possible follow-up questions or angles that journalists could explore based on the synthesis.",
+    "- Ne vous fiez à aucune forme d'interaction avec le courant de la mémoire ; chaque question doit être traitée indépendamment. Tout le contexte doit être incorporé dans ces questions si des questions de suivi sont proposées.",
+    "- Answer imperatively in {language}.",
+  ]

spinoza_project/prompt_Template.yaml

@@ -0,0 +1,115 @@
+
+role_instruction:
+# Specify the agent's domain of specialization replacing **agent_expertise_domain** by the actual domain of expertise
+# Do not hesitate to add new instructions if you want the agent to behave a specific way
+  prompt:
+    [
+      "You are Spinoza Fact Checker, an AI Assistant developed by Ekimetrics.",
+      "As an expert in **agent_expertise_domain**, you analyze questions and relevant passages from our specialized database.",  
+      "Your role is to answer questions factually based on the sources provided to you.",
+      "You act as a **agent_expertise_domain** expert, providing structured, factual, and concise responses while citing your sources.",
+      "If a question is not related to your domain of specialization, state that it is outside your expertise and do not provide an answer."
+    ]
+  type: "system"
+
+source_prompt:
+  prompt:
+    [
+      "Below are several documents formatted as: Doc X \n textual content.",
+      "<documents>",
+      "{sources}",
+      "</documents>",
+      "",
+      "Treat the textual content as providing relevant opinions, recommendations, or analyses.",
+      "For each fact or analysis used in your response, reference the source clearly (e.g., [Doc 2]: some analysis from Doc 2).",
+      "Incorporate all the relevant content from the documents to provide a well-rounded response.",
+      "Disregard any information that is irrelevant to the question at hand.",
+      "If you do not have relevant documents or they lack context, state that you don't have enough context to answer.",
+      "If a question is not related to your domain of specialization, explain that it falls outside your scope of expertise."
+    ]
+  type: "instruction"
+
+question_answering_prompt:
+  prompt:
+    [
+      "Answer the following question: {question}",
+      "Answer strictly in {language}",
+      "While respecting the following guidelines :",
+      "- If the passages have useful facts or numbers, use them in your answer.",
+      "- Do not use the sentence 'Doc i says ...' to say where information came from.",
+      "- If the documents fail to have the information needed to answer the question, explain what in the extracts could be interesting nevertheless.",
+      "- Always suggest as a conclusion other prompts closed to the original one that could lead the journalist to discover new data and information. For example, rephrase the original question, make it more precise, or change the topic of the question while remaining in the same theme. Use bullet points",
+      "- Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.",
+      "- If it makes sense, use bullet points and lists to make your answers easier to understand.",
+      "- You do not need to use every passage. Only use the ones that help answer the question.",
+      "- If a specific location is mentioned in the question, make it the core of your answer and follow the //specific guidelines//",
+      "",
+      "//specific guidelines//",
+      "if [the question is open and broad] then [:",
+      "- If the documents do not have the information needed to answer the question, say that you don't have enough information to answer directly to this question - it must be at the beginning of the text.",
+      "- If the documents fail to have the information needed to answer the question, explain what in the extracts could be interesting nevertheless.",
+      "- Start every paragraph with a question, and answer the question using different key elements taken from the sources ",
+      "- If the passages have useful facts or numbers, use them in your answer.",
+      "- When you use information from a passage, mention where it came from by using [Doc i] at the end of the sentence. i stands for the number of the document.",
+      "- Do not use the sentence 'Doc i says ...' to say where information came from.",
+      "- If the same thing is said in more than one document, you can mention all of them like this: [Doc i, Doc j, Doc k]",
+      "- Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.",
+      "- If it makes sense, use bullet points and lists to make your answers easier to understand.",
+      "- You do not need to use every passage. Only use the ones that help answer the question.",
+      "- If the documents do not have the information needed to answer the question, just say you do not have enough information.",
+      "- Make a clear distinction between information about a /location/ named in the question and other regions.",
+      "  - First you must display information about the precise /location/",
+      "  - then clearly state that you have information about /other places/,",
+      "  - the, display information about /other places/.",
+      "- Always suggest as a conclusion other prompts closed to the original one that could lead the journalist to discover new data and information. For example, rephrase the original question, make it more precise, or change the topic of the question while remaining in the same theme. Use bullet points]",
+      "",
+      "if [the question is factual and precise] then [",
+      "- If the documents do not have the information needed to answer the question, say that you don't have enough information to answer directly to this question - it must be at the beginning of the text.",
+      "- If the documents fail to have the information needed to answer the question, explain what in the extracts could be interesting nevertheless.",
+      "- Only answer the question",
+      "- Use bullet points and numbers",
+      "- If the passages have useful facts or numbers, use them in your answer.",
+      "- When you use information from a passage, mention where it came from by using [Doc i] at the end of the sentence. i stands for the number of the document.",
+      "- Do not use the sentence 'Doc i says ...' to say where information came from.",
+      "- If the same thing is said in more than one document, you can mention all of them like this: [Doc i, Doc j, Doc k]",
+      "- Do not just summarize each passage one by one. Group your summaries to highlight the key parts in the explanation.",
+      "- If it makes sense, use bullet points and lists to make your answers easier to understand.",
+      "- You do not need to use every passage. Only use the ones that help answer the question.",
+      "- If the documents do not have the information needed to answer the question, just say you do not have enough information.",
+      "- Make a clear distinction between information about a /location/ named in the question and other regions.",
+      "  - First you must display information about the precise /location/",
+      "  - then clearly state that you have information about /other places/,",
+      "  - the, display information about /other places/",
+      "- Always suggest as a conclusion other prompts closed to the original one that could lead the journalist to discover new data and information. For example, rephrase the original question, make it more precise, or change the topic of the question while remaining in the same theme. Use bullet points]",
+      "-Awnser in French"
+    ]
+  type: "prompt"
+
+reformulation_prompt:
+  prompt: [
+      "Reformulez le message de l'utilisateur en une question autonome et concise en français, en tenant compte du contexte fourni par la question initiale.",
+      "Reformule la question initiale strictement en {language}",
+      "Cette question servira à rechercher des documents pertinents dans une liste d'articles de presse.",
+      "Si la question est trop vague ou ambiguë, reformulez-la pour la rendre plus précise et ainsi augmenter les chances de trouver des documents pertinents dans ce corpus.",
+      "Ajoutez des éléments contextuels si nécessaire, tout en conservant la pertinence du sujet principal.",
+      "Si la question est déjà claire, reformulez-la simplement en gardant son essence.",
+      "",
+      "Exemples:",
+      "---",
+      "user:",
+      "Quels enjeux autour de l'eau?",
+      "",
+      "assistant:",
+      "Quels articles abordent les enjeux liés à l'eau et sous quels aspects sont-ils traités?",
+      "---",
+      "user:",
+      "Quelles obligations de faire un bilan carbone?",
+      "",
+      "assistant:",
+      "Quelles sont les obligations légales liées au bilan carbone et comment ces obligations sont-elles traitées dans les articles?",
+      "---",
+      "user:",
+      "{question}",
+      "",
+    ]
+  type: "prompt"

spinoza_project/source/backend/document_store.py

@@ -0,0 +1,45 @@
+from qdrant_client.http import models
+import pickle as pickle
+import torch
+import io
+
+device_str = "cuda:0" if torch.cuda.is_available() else "cpu"
+device = torch.device(device_str)
+
+
+class Device_Unpickler(pickle.Unpickler):
+
+    def find_class(self, module, name):
+        if module == "torch.storage" and name == "_load_from_bytes":
+            return lambda b: torch.load(io.BytesIO(b), map_location=device_str)
+        else:
+            return super().find_class(module, name)
+
+
+def pickle_to_document_store(path):
+    with open(path, "rb") as f:
+        document_store = Device_Unpickler(f).load()
+    document_store.embeddings.encode_kwargs["device"] = device_str
+    return document_store
+
+
+def get_qdrant_filters(filter_dict: dict):
+    """Build a Qdrant filter based on a filter dict.
+
+    Filter dict must use metadata fields and be formated like:
+
+    filter_dict = {'file_name':['file1', 'file2'],'sub_type':['text']}
+    """
+    return models.Filter(
+        must=[
+            models.FieldCondition(
+                key=f"metadata.{field}",
+                match=models.MatchAny(any=filter_dict[field]),
+            )
+            for field in filter_dict
+        ]
+    )
+
+def escape_value(value):
+    return value.replace("'", "''")
+

spinoza_project/source/backend/get_prompts.py

@@ -0,0 +1,33 @@
+from spinoza_project.source.backend.prompt_utils import SpecialTokens, make_chat_prompt
+from langchain.prompts.chat import ChatPromptTemplate
+
+
+def get_qa_prompts(config, prompts):
+    special_tokens = SpecialTokens(config)
+    role_instruction = make_chat_prompt(prompts["role_instruction"], special_tokens)
+    source_prompt = make_chat_prompt(prompts["source_prompt"], special_tokens)
+    # memory_prompt=make_chat_prompt(prompts['memory_prompt'], special_tokens)
+    question_answering_prompt = make_chat_prompt(
+        prompts["question_answering_prompt"], special_tokens
+    )
+    reformulation_prompt = make_chat_prompt(
+        prompts["reformulation_prompt"], special_tokens
+    )
+    # summarize_memory_prompt = make_chat_prompt(
+    #     prompts["summarize_memory_prompt"], special_tokens
+    # )
+
+    chat_qa_prompt = ChatPromptTemplate.from_messages(
+        [
+            role_instruction,
+            source_prompt,
+            # memory_prompt,
+            question_answering_prompt,
+        ]
+    )
+    chat_reformulation_prompt = ChatPromptTemplate.from_messages([reformulation_prompt])
+    # chat_summarize_memory_prompt = ChatPromptTemplate.from_messages([summarize_memory_prompt])
+    return (
+        chat_qa_prompt,
+        chat_reformulation_prompt,
+    )  # , chat_summarize_memory_prompt

spinoza_project/source/backend/llm_utils.py

@@ -0,0 +1,56 @@
+from msal import ConfidentialClientApplication
+from langchain_openai import AzureOpenAIEmbeddings, AzureChatOpenAI
+from langchain_groq import ChatGroq
+from langchain.vectorstores.azuresearch import AzureSearch
+import os
+import pandas as pd
+from tqdm import tqdm
+from collections import defaultdict
+import json
+
+class LLM:
+    def __init__(self, llm):
+        self.llm = llm
+        self.callbacks = []
+
+    def stream(self, prompt, prompt_arguments):
+        self.llm.streaming = True
+        streamed_content = self.llm.stream(prompt.format_messages(**prompt_arguments))
+        output = ""
+        for op in streamed_content:
+            output += op.content
+            yield output
+
+    def get_prediction(self, prompt, prompt_arguments):
+        self.llm.callbacks = self.callbacks
+        return self.llm.predict_messages(
+            prompt.format_messages(**prompt_arguments)
+        ).content
+
+    async def get_aprediction(self, prompt, prompt_arguments):
+        self.llm.callbacks = self.callbacks
+        prediction = await self.llm.apredict_messages(
+            prompt.format_messages(**prompt_arguments)
+        )
+        return prediction
+
+    async def get_apredictions(self, prompts, prompts_arguments):
+        self.llm.callbacks = self.callbacks
+        predictions = []
+        for prompt_, prompt_args_ in zip(prompts.keys(), prompts_arguments):
+            prediction = await self.llm.apredict_messages(
+                prompts[prompt_].format_messages(**prompt_args_)
+            )
+            predictions.append(prediction.content)
+        return predictions
+
+
+def get_llm_api(groq_model_name):
+    print("Using GROQ API")
+    return LLM(
+        ChatGroq(
+            model=groq_model_name,
+            temperature=0,
+            max_tokens=2048,
+        )
+    )

spinoza_project/source/backend/prompt_utils.py

@@ -0,0 +1,68 @@
+from langchain.prompts.chat import ChatMessagePromptTemplate
+
+
+class SpecialTokens:
+    def __init__(self, config):
+        self.user_token = config["user_token"]
+        self.assistant_token = config["assistant_token"]
+        self.system_token = config["system_token"]
+        self.stop_token = config["stop_token"]
+
+
+def to_instruction(query, special_tokens):
+    return special_tokens.user_token + query + special_tokens.stop_token
+
+
+def to_prompt(query, special_tokens):
+    return (
+        special_tokens.user_token
+        + query
+        + special_tokens.stop_token
+        + special_tokens.assistant_token
+    )
+
+
+def to_system(query, special_tokens):
+    return special_tokens.system_token + query + special_tokens.stop_token
+
+
+def make_prompt(prompt, special_tokens):
+    prompt_type = prompt["type"]
+    if prompt_type == "system":
+        return to_system("\n".join(prompt["prompt"]), special_tokens)
+    elif prompt_type == "instruction":
+        return to_instruction("\n".join(prompt["prompt"]), special_tokens)
+    elif prompt_type == "prompt":
+        return to_prompt("\n".join(prompt["prompt"]), special_tokens)
+    else:
+        return "Invalid prompt type, please check your config"
+
+
+def to_chat_instruction(query, special_tokens):
+    return ChatMessagePromptTemplate.from_template(
+        query, role=special_tokens.user_token
+    )
+
+
+def to_chat_system(query, special_tokens):
+    return ChatMessagePromptTemplate.from_template(
+        query, role=special_tokens.system_token
+    )
+
+
+def to_chat_prompt(query, special_tokens):
+    return ChatMessagePromptTemplate.from_template(
+        query, role=special_tokens.user_token
+    )
+
+
+def make_chat_prompt(prompt, special_tokens):
+    prompt_type = prompt["type"]
+    if prompt_type == "system":
+        return to_chat_system("\n".join(prompt["prompt"]), special_tokens)
+    elif prompt_type == "instruction":
+        return to_chat_instruction("\n".join(prompt["prompt"]), special_tokens)
+    elif prompt_type == "prompt":
+        return to_chat_prompt("\n".join(prompt["prompt"]), special_tokens)
+    else:
+        return "Invalid prompt type, please check your config"

spinoza_project/source/frontend/gradio_utils.py

@@ -0,0 +1,499 @@
+import gradio as gr
+import os
+import yaml
+import pandas as pd
+from tqdm import tqdm
+from collections import defaultdict
+from langchain.prompts.chat import ChatPromptTemplate
+from huggingface_hub import hf_hub_download
+from spinoza_project.source.frontend.utils import make_html_source
+from spinoza_project.source.backend.prompt_utils import (
+    to_chat_instruction,
+    SpecialTokens,
+)
+from spinoza_project.source.backend.get_prompts import get_qa_prompts
+from spinoza_project.source.backend.document_store import pickle_to_document_store
+from spinoza_project.source.backend.document_store import get_qdrant_filters
+
+def get_config():
+    if os.getenv("EKI_OPENAI_LLM_DEPLOYMENT_NAME"):
+        with open("./spinoza_project/config.yaml") as f:
+            return yaml.full_load(f)
+
+    else:
+        with open("./spinoza_project/config_public.yaml") as f:
+            return yaml.full_load(f)
+
+
+def get_prompts(config):
+    prompts = {}
+    for key, prompt_path in config["prompts_path_list"].items():
+        with open(prompt_path) as f:
+            source = config["source_mapping"][key]
+            prompts[source] = yaml.full_load(f)
+    return prompts
+
+
+def set_prompts(prompts, config):
+    chat_qa_prompts, chat_reformulation_prompts = ({}, {})
+    for source, prompt in prompts.items():
+        chat_qa_prompt, chat_reformulation_prompt = get_qa_prompts(config, prompt)
+        chat_qa_prompts[source] = chat_qa_prompt
+        chat_reformulation_prompts[source] = chat_reformulation_prompt
+
+    return chat_qa_prompts, chat_reformulation_prompts
+
+
+def get_assets():
+    with open("./assets/style.css", "r") as f:
+        css = f.read()
+    with open("./assets/source_information_fr.md", "r") as f:
+        source_information_fr = f.read()
+    with open("./assets/source_information_en.md", "r") as f:
+        source_information_en = f.read()
+    with open("./assets/about_contact_fr.md", "r") as f:
+        about_contact_fr = f.read()
+    with open("./assets/about_contact_en.md", "r") as f:
+        about_contact_en = f.read()
+    return (
+        css,
+        source_information_fr,
+        source_information_en,
+        about_contact_fr,
+        about_contact_en,
+    )
+
+
+def get_qdrants(config):
+    qdrants = {}
+    tabs_to_load = [
+        tab
+        for tab in config["source_mapping"].keys()
+    ]
+
+    print("------ Default Qdrants")
+    for tab in tqdm(tabs_to_load, desc="Downloading qdrants from HF..."):
+        tab_name = config["source_mapping"][tab]
+        try:
+            if tab_name in ["Science", "Loi", "Organismes publics", "ADEME"]:
+                file_path = hf_hub_download(
+                    repo_id=config["default_databases"],
+                    filename=config["databases_pickle_files"][tab],
+                    repo_type="dataset",
+                )
+            elif tab_name not in ["Science", "Loi", "Organismes publics", "ADEME"]:
+                print(f"------ Customed Qdrants: {tab_name}")
+                file_path = hf_hub_download(
+                    repo_id=config["database_hf"],
+                    filename=config["databases_pickle_files"][tab],
+                    repo_type="dataset",
+                )
+            else:
+                continue
+
+            qdrants[tab_name] = pickle_to_document_store(file_path)
+
+        except Exception as e:
+            print(f"Error occured while loading {tab_name}: {str(e)}")
+            raise
+    
+
+    documents = defaultdict(
+        lambda: {
+            "database": "",
+            "file_source_sub_type": "",
+            "file_author": "",
+            "file_title": "",
+            "file_date_publishing": "",
+            "chunk_count": 0,
+            "page_numbers": set(),
+            "file_filtering_modality": "",
+        }
+    )
+
+    for db_name, qdrant in tqdm(qdrants.items(), desc="Processing databases"):
+        print(f"Database: {db_name}")
+        try:
+            offset = None
+            total_processed = 0
+            batch_size = 10000
+
+            while True:
+                scroll_result = qdrant.client.scroll(
+                    collection_name=qdrant.collection_name,
+                    limit=batch_size,
+                    offset=offset,
+                    with_payload=True,
+                    with_vectors=False,
+                )
+
+                if not scroll_result[0]:
+                    break
+
+                for point in scroll_result[0]:
+                    metadata = point.payload.get("metadata", {})
+
+                    unique_fields = [
+                        metadata.get("file_title", ""),
+                        str(metadata.get("file_date_publishing", "")),
+                        metadata.get("file_name", ""),
+                        metadata.get("file_url", ""),
+                    ]
+
+                    doc_key = "_".join(field for field in unique_fields if field)
+                    if not doc_key:
+                        doc_key = f"unknown_doc_{total_processed}"
+
+                    doc_info = documents[doc_key]
+                    doc_info["database"] = db_name
+                    doc_info["file_title"] = metadata.get("file_title", "Unknown")
+                    date_str = str(metadata.get("file_date_publishing", "Unknown"))
+                    doc_info["file_date_publishing"] = (
+                        date_str[:4]
+                        if date_str[:4].isdigit() and len(date_str) >= 4
+                        else "Unknown"
+                    )
+                    doc_info["file_filtering_modality"] = metadata.get(
+                        "file_filtering_modality", "Unknown"
+                    )
+
+                    if "content_page_number" in metadata:
+                        doc_info["page_numbers"].add(metadata["content_page_number"])
+
+                    total_processed += 1
+
+                offset = scroll_result[1]
+
+                print(
+                    f"\rTraitement de {db_name}: {total_processed} chunks traités",
+                    end="",
+                )
+
+                if offset is None:
+                    break
+
+        except Exception as e:
+            print(f"\nErreur lors du traitement de {db_name}: {str(e)}")
+            continue
+
+    df_documents = pd.DataFrame(
+        [
+            {
+                "Source": info["database"],
+                "Title": info["file_title"],
+                "Page range": len(info["page_numbers"]),
+                "Filter": info["file_filtering_modality"],
+                "Publishing date": info["file_date_publishing"],
+            }
+            for info in documents.values()
+        ]
+    )
+
+    return qdrants, df_documents
+
+
+def get_theme():
+    return gr.themes.Base(
+        primary_hue="blue",
+        secondary_hue="red",
+        font=[
+            gr.themes.GoogleFont("Poppins"),
+            "ui-sans-serif",
+            "system-ui",
+            "sans-serif",
+        ],
+    )
+
+
+def get_init_prompt(lang="en"):
+    prompt = """
+        Hello, I am Spinoza, a conversational assistant specialized in climate-related topics, designed to support you in your journalistic endeavors. I will answer your climate-related questions **based on the provided sources**.
+
+        ⚠️ Limitations
+        *Please note that this questioning system is in its early stages, and it may occasionally provide irrelevant answers. If you are not satisfied with the response, please ask a more specific question or share your feedback to help us improve the system.*
+
+        What would you like to learn?
+        """
+    if lang == "fr":
+        prompt = """
+        Bonjour, je suis Spinoza, un assistant conversationnel expert sur le climat conçu pour vous aider dans votre parcours journalistique. Je répondrai à vos questions en lien avec le climat en me basant **sur les sources fournies**.
+
+        ⚠️ Limitations
+        *Veuillez noter que ce système de questionnement est à un stade précoce, il n'est pas parfait et peut parfois donner des réponses non pertinentes. Si vous n'êtes pas satisfait de la réponse, veuillez poser une question plus spécifique ou signaler vos commentaires pour nous aider à améliorer le système.*
+
+        Que voulez-vous apprendre ?
+        """
+
+    return prompt.strip()
+
+
+def get_synthesis_prompt(config):
+    special_tokens = SpecialTokens(config)
+    with open(f"./spinoza_project/prompt_Spinoza.yaml", "r") as f:
+        synthesis_template = f.read()
+
+    synthesis_prompt = to_chat_instruction(synthesis_template, special_tokens)
+    synthesis_prompt_template = ChatPromptTemplate.from_messages([synthesis_prompt])
+
+    return synthesis_prompt_template
+
+
+def zip_longest_fill(*args, fillvalue=None):
+    # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-
+    iterators = [iter(it) for it in args]
+    num_active = len(iterators)
+    if not num_active:
+        return
+
+    cond = True
+    fillvalues = [fillvalue] * len(iterators)
+    while cond:
+        values = []
+        for i, it in enumerate(iterators):
+            try:
+                value = next(it)
+                if not value:
+                    value = next(it)
+            except StopIteration:
+                value = fillvalues[i]
+            values.append(value)
+
+        new_cond = False
+        for i, elt in enumerate(values):
+            if elt != fillvalues[i]:
+                new_cond = True
+        cond = new_cond
+
+        fillvalues = values.copy()
+        yield tuple(values)
+
+
+def start_agents(lang_component):
+    lang = lang_component.value if hasattr(lang_component, "value") else lang_component
+    if lang == "fr":
+        gr.Info(message="Les agents et Spinoza démarent leurs analyses...", duration=3)
+        return [
+            (
+                None,
+                "J'attends que tous les agents aient terminé pour générer une réponse...",
+            )
+        ], gr.update(
+            label=get_text("source_filter_label", lang),
+            elem_id="filter-component",
+            interactive=False,
+        )
+
+    elif lang == "en":
+        gr.Info(
+            message="The agents and Spinoza are starting their analyses...", duration=3
+        )
+        return [
+            (
+                None,
+                "I am waiting for all the agents to finish before generating a response...",
+            )
+        ], gr.update(
+            label=get_text("source_filter_label", lang),
+            elem_id="filter-component",
+            interactive=False,
+        )
+
+
+def end_agents(lang_component):
+    lang = lang_component.value if hasattr(lang_component, "value") else lang_component
+    if lang == "fr":
+        gr.Info(
+            message="Les agents et Spinoza ont fini de répondre à votre question",
+            duration=3,
+        )
+        return gr.update(
+            label=get_text("source_filter_label", lang),
+            elem_id="filter-component",
+            interactive=True,
+        )
+
+    elif lang == "en":
+        gr.Info(
+            message="The agents and Spinoza have finished answering your question.",
+            duration=3,
+        )
+        return gr.update(
+            label=get_text("source_filter_label", lang),
+            elem_id="filter-component",
+            interactive=True,
+        )
+
+
+def next_call():
+    return
+
+
+def format_question(question):
+    return f"{question}"
+
+
+def parse_question(question):
+    x = question.replace("<p>", "").replace("</p>\n", "")
+    if "### " in x:
+        return x.split("### ")[1]
+    return x
+
+
+def reformulate(language, llm, chat_reformulation_prompts, question, tab, config):
+    if tab in list(config["tabs"].keys()):
+        return llm.stream(
+            chat_reformulation_prompts[config["source_mapping"][tab]],
+            {
+                "question": parse_question(question),
+                "language": language,
+             },
+        )
+    else:
+        return iter([None] * 5)
+
+
+def add_question(question):
+    return question
+
+
+def answer(language, llm, chat_qa_prompts, question, source, tab, config):
+    if tab in list(config["tabs"].keys()):
+        if len(source) < 10:
+            return iter(["Aucune source trouvée, veuillez reformuler votre question"])
+        else:
+
+            return llm.stream(
+                chat_qa_prompts[config["source_mapping"][tab]],
+                {
+                    "question": parse_question(question),
+                    "sources": source.replace("<p>", "").replace("</p>\n", ""),
+                    "language": language,
+                },
+            )
+    else:
+        return iter([None] * 5)
+
+def perform_search(search_object, question, k, filter_query, config):
+    if search_object is None:
+        return []
+
+    cleaned_question = question.replace("<p>", "").replace("</p>\n", "")
+    if config['query_preprompt']:
+        search_query = (f"{config['query_preprompt']}{cleaned_question}")
+    else: 
+        search_query = cleaned_question
+
+    # For qdrants objects
+    try:
+        results = search_object.similarity_search_with_relevance_scores(
+            search_query, k=k, filter=filter_query if filter_query else {}
+        )
+        return results
+
+    except Exception as e:
+        print(f"Error during search for {search_object}: {str(e)}")
+        return []
+
+
+def get_sources(questions, filters, qdrants, config):
+    k = config["num_document_retrieved"]
+    min_similarity = config["min_similarity"]
+    formated = []
+    text = [""] * len(config["tabs"])
+
+    for i, (question, tab) in enumerate(zip(questions[0], list(config["tabs"].keys()))):
+        try:
+            search_object = qdrants[config['source_mapping'][tab]]
+
+            if tab in filters:
+                # if qdrant object
+                filter_query = get_qdrant_filters(filters[tab])
+            else:
+                filter_query = None
+
+            sources = perform_search(
+                search_object, question, k, filter_query, config
+            )
+
+            sources = [(doc, score) for doc, score in sources if score >= min_similarity]
+
+            formated.extend(
+                [
+                    make_html_source(source[0], j, source[1], config)
+                    for j, source in zip(range(k * i + 1, k * (i + 1) + 1), sources)
+                ]
+            )
+            text[i] = "\n\n".join(
+                [
+                    f"Doc {str(j)} with source type {source[0].metadata.get('file_source_type')}:\n"
+                    + source[0].page_content
+                    for j, source in zip(range(k * i + 1, k * (i + 1) + 1), sources)
+                ]
+            )
+
+        except KeyError:
+            continue
+
+    formated = "".join(formated)
+    return formated, text
+
+
+TRANSLATIONS = {
+    "fr": {
+        "ask_placeholder": "Posez votre question ici !",
+        "init_prompt": get_init_prompt("fr"),
+        "source_filter_label": "Paramètres des Agents",
+        "source_filter_title": "Filtrage des sources par agent.",
+        "source_filter_subtitle": "Configurez les filtres à appliquer sur les documents afin de raffiner les bases de données utilisées indépendamment par les différents agents.",
+        "question_filter": "Veuillez choisir les éléments sur lesquels appliquer le filtrage?",
+        "source_informatation_label": "Détails des Données",
+        "acc_info_label": "Informations détaillées",
+        "acc_info_desc": "Vue d'ensemble sur les sources utilisées par la solution.",
+        "display_info_desc": "Vue d'ensemble sur toutes les bases de données utilisées par la solution (à titre informatif).",
+        "contact_label": "À propos & Contact",
+    },
+    "en": {
+        "ask_placeholder": "Ask me anything here!",
+        "init_prompt": get_init_prompt("en"),
+        "source_filter_label": "Agent Data Settings",
+        "source_filter_title": "Sources filtering by agent.",
+        "source_filter_subtitle": "Configure document filters for each agent to refine their search scope.",
+        "question_filter": "Choose the items you would like to filter on...",
+        "source_informatation_label": "Data Details Overview",
+        "acc_info_label": "Detailed informations",
+        "acc_info_desc": "Brief overview of sources used by the solution.",
+        "display_info_desc": "Overview of all databases used by the solution for informational purposes.",
+        "contact_label": "About & Contact",
+    },
+}
+
+def update_translation(list_tabs, config):
+    for elt in list_tabs:
+        qa_key = f"agent_{config['source_mapping'][elt]}_qa"
+        flt_key = f"agent_{config['source_mapping'][elt]}_flt"
+        db_key = f"agent_{config['source_mapping'][elt]}_tab"
+        desc_key = f"{config['source_mapping'][elt]}_desc"
+
+        if qa_key not in TRANSLATIONS["fr"]:
+            qa_translations = {
+                "fr": f"Agent {config['source_mapping'][elt]}",
+                "en": f"{config['en_names'][elt]} Agent"
+            }
+            db_translations = {
+                "fr": f"Rubrique {config['source_mapping'][elt]}",
+                "en": f"{config['en_names'][elt]} Database"
+            }
+            desc_translations = {
+                "fr": config["tabs"][elt],
+                "en": config['en_description'][elt]
+            }
+            for lang in TRANSLATIONS:
+                TRANSLATIONS[lang][qa_key] = qa_translations.get(lang, qa_translations["fr"])
+                TRANSLATIONS[lang][flt_key] = qa_translations.get(lang, qa_translations["fr"])
+                TRANSLATIONS[lang][db_key] = db_translations.get(lang, db_translations["fr"])
+                TRANSLATIONS[lang][desc_key] = desc_translations.get(lang, desc_translations["fr"])
+
+
+
+def get_text(key, language):
+    return TRANSLATIONS[language][key]

spinoza_project/source/frontend/utils.py

@@ -0,0 +1,172 @@
+from queue import SimpleQueue
+from dotenv import load_dotenv
+import re
+from langchain.callbacks.base import BaseCallbackHandler
+
+job_done = object()  # signals the processing is done
+
+
+class StreamingGradioCallbackHandler(BaseCallbackHandler):
+    """Callback handler for streaming. Only works with LLMs that support streaming."""
+
+    def __init__(self, q):
+        self.q = q
+
+    def on_llm_start(self, serialized, prompts, **kwargs) -> None:
+        """Run when LLM starts running."""
+        while not self.q.empty():
+            try:
+                self.q.get(block=False)
+            except SimpleQueue.empty:
+                continue
+
+    def on_llm_new_token(self, token, **kwargs) -> None:
+        """Run on new LLM token. Only available when streaming is enabled."""
+        self.q.put(token)
+
+    def on_llm_end(self, response, **kwargs) -> None:
+        """Run when LLM ends running."""
+        self.q.put(job_done)
+
+    def on_llm_error(self, error, **kwargs) -> None:
+        """Run when LLM errors."""
+        self.q.put(job_done)
+
+
+def add_gradio_streaming(llm):
+    q = SimpleQueue()
+    job_done = object()  # signals the processing is done
+    llm.callbacks = [StreamingGradioCallbackHandler(q)]
+    return llm, q
+
+
+def gradio_stream(llm, prompt):
+    thread = Thread(target=llm.predict, kwargs={"text": prompt})
+    thread.start()
+    text = ""
+    while True:
+        next_token = q.get(block=True)  # Blocks until an input is available
+        if next_token is job_done:
+            break
+        text += next_token
+        time.sleep(0.03)
+        yield text
+    thread.join()
+
+
+def get_source_link(metadata):
+    return metadata["file_url"] + f"#page={metadata['content_page_number'] + 1}"
+
+
+def make_html_source(source, i, score, config):
+    meta = source.metadata
+    if meta["file_source_type"] == "AFP":
+        return f"""
+        <div class="card" id="doc{i}">
+            <div class="card-content">
+                <h2>Doc {i} - {meta['file_title']} - {meta['file_type']} AFP</h2>
+                <p>{source.page_content}</p>
+            </div>
+            <div class="card-footer">
+                <span>{meta['file_source_type']}</span>
+                <span>Relevance Score : {round(100*score,1)}%</span>
+            </div>
+        </div>
+        """
+
+    if meta["file_source_type"] == "Presse":
+        if meta["file_url"] != "none":
+            return f"""
+            <div class="card" id="doc{i}">
+                <div class="card-content">
+                    <h2>Doc {i} - {meta['file_title']} - {meta['file_publisher']}</h2>
+                    <p>{source.page_content}</p>
+                </div>
+                <div class="card-footer">
+                    <span>{meta['file_source_type']}</span>
+                    <span>Relevance Score : {round(100*score,1)}%</span>
+                    <a href={meta['file_url']} target="_blank">
+                        <span role="img" aria-label="Open PDF">🔗</span>
+                    </a>
+                </div>
+            </div>
+            """
+        else:
+            return f"""
+            <div class="card" id="doc{i}">
+                <div class="card-content">
+                    <h2>Doc {i} - {meta['file_title']} - {meta['file_publisher']}</h2>
+                    <p>{source.page_content}</p>
+                </div>
+                <div class="card-footer">
+                    <span>{meta['file_source_type']}</span>
+                    <span>Relevance Score : {round(100*score,1)}%</span>
+                </div>
+            </div>
+            """
+
+    if meta["file_url"]:
+        return f"""
+        <div class="card" id="doc{i}">
+            <div class="card-content">
+                <h2>Doc {i} - {meta['file_title']} - Page {meta['content_page_number'] + 1}</h2>
+                <p>{source.page_content.replace(config["passage_preprompt"], "")}</p>
+            </div>
+            <div class="card-footer">
+                <span>{meta['file_source_type']}</span>
+                <span>Relevance Score : {round(100*score,1)}%</span>
+                <a href="{get_source_link(meta)}" target="_blank">
+                    <span role="img" aria-label="Open PDF">🔗</span>
+                </a>
+            </div>
+        </div>
+        """
+    else:
+        return f"""
+        <div class="card" id="doc{i}">
+            <div class="card-content">
+                <h2>Doc {i} - {meta['file_title']} - Page {meta['content_page_number'] + 1}</h2>
+                <p>{source.page_content.replace(config["passage_preprompt"], "")}</p>
+            </div>
+            <div class="card-footer">
+                <span>{meta['file_source_type']}</span>
+                <span>Relevance Score : {round(100*score,1)}%</span>
+            </div>
+        </div>
+        """
+
+
+def parse_output_llm_with_sources(output):
+    content_parts = re.split(
+        r"([\[(]?(?:(?:Doc|doc|Document|document)\s*\d+(?:\s*,\s*(?:Doc|doc|Document|document)\s*\d+)*)[\])]?)",  # r"[\[(]?(Doc\s?\d+(?:,\s?Doc\s?\d+)*|doc\s?\d+(?:,\s?doc\s?\d+)*|Doc\s\d+)[\])?]",
+        output,
+    )
+    # print(content_parts)
+    parts = []
+    for part in content_parts:
+        if part.lower().strip("[]()").startswith("doc"):
+            subpart = part.strip("[]()")
+            subpart = subpart.lower().replace("document", "").replace("doc", "").strip()
+            subpart = f"""<a href="#doc{subpart}" class="a-doc-ref" target="_self"><span class='doc-ref'><sup>{subpart}</sup></span></a>"""
+            parts.append(subpart)
+        else:
+            parts.append(part)
+    content_parts = "".join(parts)
+
+    return content_parts
+
+
+def clear_text_box(textbox):
+    return ""
+
+
+def add_text(chatbot, text):
+    chatbot = chatbot + [(text, None)]
+    return chatbot, text
+
+
+def init_env():
+    try:
+        load_dotenv()
+    except:
+        pass