more details content

app.py

@@ -1,20 +1,11 @@
 import streamlit as st
 import pymupdf as fitz
 import pyperclip
-from utils.audit.audit_doc import audit_descriptif
+from utils.audit.audit_doc import audit_descriptif_pdf,audit_text,audit_descriptif_word
 import dotenv
 from utils.audit.audit_audio import evaluate_audio_quality
-
-
-# Function to extract text from PDF
-def extract_text_from_pdf(file):
-    document = fitz.open(stream=file.read(), filetype="pdf")
-    full_text = ""
-    for page_num in range(len(document)):
-        page = document.load_page(page_num)
-        text = page.get_text("text")
-        full_text += text
-    return full_text
+from PIL import Image
+from io import BytesIO
 
 # Function to classify file type
 def classify_file(file):
@@ -30,6 +21,95 @@ def classify_file(file):
         return "text"
     else:
         return "unknown"
+    
+#display content
+def display_content_doc(content:dict):
+
+    number_of_pages = len(content)
+    st.info("si vous choisissez 0, vous verrez le contenu de toutes les pages")
+    
+    number = st.number_input("Numéro de page", min_value=0, max_value=number_of_pages, value=0,key="number_page_content")
+    #0 means all pages
+    if number > 0:
+        page : dict = content[f"page_{number-1}"]
+    option = st.radio("Type de contenu",list(content[f"page_0"].keys()), index=0,horizontal=True)
+    if option == "images":
+        if number == 0:
+            images = [img for page in content.values() for img in page["images"]]
+        else:
+            images = page["images"]
+        col1,col2,col3 = st.columns(3)
+        for i, (img_bytes, img_width, img_height) in enumerate(images):
+            if i%3 == 0:
+                col1.image(Image.open(BytesIO(img_bytes)), caption=f'Image {i + 1}', width=img_width)
+            elif i%3 == 1:
+                col2.image(Image.open(BytesIO(img_bytes)), caption=f'Image {i + 1}', width=img_width)
+            else:
+                col3.image(Image.open(BytesIO(img_bytes)), caption=f'Image {i + 1}', width=img_width)
+
+    elif option == "texte":
+        if number == 0:
+            text = "-------------------\n".join([page["texte"] for page in content.values()])
+        else:
+            text = page["texte"]
+        
+        st.text_area("Texte",text,height=200)
+    
+    elif option == "liens":
+        if number == 0:
+            links = [link for page in content.values() for link in page["liens"]]
+        else:
+            links = page["liens"]
+        for i, link in enumerate(links):
+            st.markdown(f"- {i+1}: {link['uri']} (page {link['page']})")
+        
+
+    
+    
+def display_audit_pdf(uploaded_file):
+    if st.session_state.name_file != uploaded_file.name:
+        st.session_state.name_file = uploaded_file.name
+        with st.spinner("Analyse du document..."):
+            st.session_state.audit = audit_descriptif_pdf(uploaded_file,200)
+    audit = st.session_state.audit["audit"]
+    content = st.session_state.audit["content"]
+    #global audit
+    audit_simplified = {
+        "Nombre de pages": audit["number_of_pages"],
+        "Nombre d'images": audit["number_of_images"],
+        "Nombre de liens": audit["number_of_links"],
+        "Nombre de tableaux": audit["number_of_tables"],
+        "Nombre de tokens": audit["number_of_tokens"],
+        "Nombre de mots": audit["number_of_words"],
+        "Mots clés": audit["key_words"]
+    }
+
+    well_formatted_audit = "Contenus audités\n"
+    for key, value in audit_simplified.items():
+        well_formatted_audit += f"- {key}: {value}\n"
+
+    st.code(well_formatted_audit)
+
+    #audit par page
+    with st.expander("Audit par page"):
+        number = st.number_input("Numéro de page", min_value=1, max_value=audit["number_of_pages"], value=1,key="number_page_audit")
+        audit_page = audit[f"page_{number-1}"]
+        audit_page = {
+            "Nombre d'images": audit_page["number_of_images"],
+            "Nombre de liens": audit_page["number_of_links"],
+            "Nombre de tableaux": audit_page["number_of_tables"],
+            "Nombre de tokens": audit_page["number_of_tokens"],
+            "Nombre de mots": audit_page["number_of_words"],
+        }
+        well_formatted_audit_page = "Audit descriptif\n"
+        for key, value in audit_page.items():
+            well_formatted_audit_page += f"- {key}: {value}\n"
+        
+        st.code(well_formatted_audit_page)
+    
+    with st.expander("Cliquer ici pour voir le contenu du document"):
+        display_content_doc(content)
+
 
 def main():
     dotenv.load_dotenv()
@@ -42,25 +122,53 @@ def main():
         st.session_state.name_file = ""
 
     # File uploader
-    uploaded_file = st.file_uploader("Télécharger un documents")
+    uploaded_file = st.file_uploader("Télécharger un ou plusieurs des documents")
 
     if uploaded_file is not None:
         type = classify_file(uploaded_file)
+
         st.write(f"Type de fichier: {type}")
+
+        st.write("### Audit de tout le document")
+        
         if type == "pdf":
+            display_audit_pdf(uploaded_file)
 
+        
+        elif type == "audio":
             if st.session_state.name_file != uploaded_file.name:
                 st.session_state.name_file = uploaded_file.name
-                with st.spinner("Analyse du document..."):
-                    st.session_state.audit = audit_descriptif(uploaded_file)
+                with st.spinner("Analyse de l'audio..."):
+                    st.session_state.audit = evaluate_audio_quality(uploaded_file)
             audit = st.session_state.audit
 
-            #global audit
+            #audit global simplifié
+            audit_simplified = {
+                "Volume": f"{audit['volume']:0.2f} dBFS",
+                "SNR": f"{max(audit['SNR'],0):0.2f} dB",
+                "Durée": f"{audit['duration']:0.2f} minutes",
+                "Nombre de tokens": audit["number_of_tokens"]
+            }
+
+            well_formatted_audit = "Contenus audités\n"
+            for key, value in audit_simplified.items():
+                well_formatted_audit += f"- {key}: {value}\n"
+            
+            st.code(well_formatted_audit)
+            
+            with st.expander("Transcription"):
+                st.write(audit["transcription"])
+        
+        elif type == "text":
+            text = uploaded_file.read().decode("utf-8")
+            if st.session_state.name_file != uploaded_file.name:
+                st.session_state.name_file = uploaded_file.name
+                with st.spinner("Analyse du texte..."):
+                    st.session_state.audit = audit_text(text)
+            audit = st.session_state.audit
+
+            #audit global simplifié
             audit_simplified = {
-                "Nombre de pages": audit["number_of_pages"],
-                "Nombre d'images": audit["number_of_images"],
-                "Nombre de liens": audit["number_of_links"],
-                "Nombre de tableaux": audit["number_of_tables"],
                 "Nombre de tokens": audit["number_of_tokens"],
                 "Nombre de mots": audit["number_of_words"]
             }
@@ -69,56 +177,31 @@ def main():
             for key, value in audit_simplified.items():
                 well_formatted_audit += f"- {key}: {value}\n"
 
-            st.write("### Audit de tout le document")
             st.code(well_formatted_audit)
-
-            #audit par page
-            with st.expander("Audit par page"):
-                number = st.number_input("Numéro de page", min_value=1, max_value=audit["number_of_pages"], value=1)
-                audit_page = audit[f"page_{number-1}"]
-                audit_page = {
-                    "Nombre d'images": audit_page["number_of_images"],
-                    "Nombre de liens": audit_page["number_of_links"],
-                    "Nombre de tableaux": audit_page["number_of_tables"],
-                    "Nombre de tokens": audit_page["number_of_tokens"],
-                    "Nombre de mots": audit_page["number_of_words"]
-                }
-                well_formatted_audit_page = "Audit descriptif\n"
-                for key, value in audit_page.items():
-                    well_formatted_audit_page += f"- {key}: {value}\n"
-                
-                st.code(well_formatted_audit_page)
-
-            # # Button to copy text to clipboard
-            # if st.button("Copy to Clipboard"):
-            #     pyperclip.copy(audit)
-            #     st.success("Text copied to clipboard successfully!")
-            # else:
-            #     st.info("Please upload a PDF file to extract text.")
         
-        elif type == "audio":
+        elif type == "word":
             if st.session_state.name_file != uploaded_file.name:
                 st.session_state.name_file = uploaded_file.name
-                with st.spinner("Analyse de l'audio..."):
-                    st.session_state.audit = evaluate_audio_quality(uploaded_file)
+                with st.spinner("Analyse du document..."):
+                    st.session_state.audit = audit_descriptif_word(uploaded_file)
             audit = st.session_state.audit
 
-            #audit global simplifié
+            #global audit
             audit_simplified = {
-                "Volume": f"{audit['volume']:0.2f} dBFS",
-                "SNR": f"{max(audit['SNR'],0):0.2f} dB",
-                "Durée": f"{audit['duration']:0.2f} minutes",
-                "Nombre de tokens": audit["number_of_tokens"]
+                "Nombre de pages": audit["number_of_paragraphs"],
+                "Nombre d'images": audit["number_of_images"],
+                "Nombre de liens": audit["number_of_links"],
+                "Nombre de tableaux": audit["number_of_tables"],
+                "Nombre de tokens": audit["number_of_tokens"],
+                "Nombre de mots": audit["number_of_words"]
             }
 
-            well_formatted_audit = "Audit descriptif\n"
+            well_formatted_audit = "Contenus audités\n"
             for key, value in audit_simplified.items():
                 well_formatted_audit += f"- {key}: {value}\n"
-            
+
             st.code(well_formatted_audit)
-            
-            with st.expander("Transcription"):
-                st.write(audit["transcription"])
+
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -6,4 +6,9 @@ numpy
 scipy
 textstat
 pymupdf
-openai
+openai
+nltk
+rake_nltk
+python-docx
+io
+pillow

utils/audit/audit_doc.py

@@ -2,6 +2,15 @@
 import pymupdf
 import tiktoken
 import textstat
+from docx import Document
+import io
+from rake_nltk import Rake
+import nltk
+from nltk.corpus import stopwords
+
+# Download NLTK stopwords
+nltk.download('stopwords')
+nltk.download('punkt')
 
 
 def evaluate_text_quality(text: str) -> dict:
@@ -44,12 +53,17 @@ def evaluate_text_quality(text: str) -> dict:
     # Scale the global score to 0-5
     global_score_0_5 = global_score * 5
 
+def extract_keywords(text):
+    rake = Rake(stopwords.words('english'))
+    rake.extract_keywords_from_text(text)
+    return rake.get_ranked_phrases_with_scores()
+
 def count_tokens(input_string: str) -> int:
     tokenizer = tiktoken.get_encoding("cl100k_base")
     tokens = tokenizer.encode(input_string)
     return len(tokens)
 
-def audit_descriptif(file) -> dict:
+def audit_descriptif_pdf(file,max_img_width) -> dict:
     document = pymupdf.open(stream=file.read())
 
     audit_dict_doc = {
@@ -58,20 +72,56 @@ def audit_descriptif(file) -> dict:
         "number_of_links": 0,
         "number_of_tables": 0,
         "number_of_tokens": 0,
-        "number_of_words": 0
+        "number_of_words": 0,
+        "key_words": []
     }
 
+    doc_content = dict()
+
     for page in document:
 
         audit_dict_page = {}
+        page_content = {
+            "images": [],
+            "texte": "",
+            "liens": []
+        }
+
         #number of images
-        number_images = len(page.get_images())
+        images = page.get_images()
+        number_images = len(images)
         audit_dict_page["number_of_images"] = number_images
         audit_dict_doc["number_of_images"] += number_images
-
+        
+        #get images
+        for _, img in enumerate(images):
+            xref = img[0]
+            base_image = document.extract_image(xref)
+
+            image_bytes = base_image["image"]
+            image_width = base_image["width"]
+            image_height = base_image["height"]
+            
+            # Adjust image size if it exceeds the maximum width
+            if image_width > max_img_width:
+                ratio = max_img_width / image_width
+                image_width = max_img_width
+                image_height = int(image_height * ratio)
+            
+            page_content["images"].append((image_bytes, image_width, image_height))
+
+
+        
+        #get links with uri
+        links = []
+        for link in page.get_links():
+            if link['kind'] == pymupdf.LINK_URI and 'uri' in link:
+                links.append({"uri": link["uri"], "page": page.number})
+        
+        page_content["liens"] = links
 
         #number of links
-        number_links = len(page.get_links())
+        number_links = len(links)
         audit_dict_page["number_of_links"] = number_links
         audit_dict_doc["number_of_links"] += number_links
 
@@ -85,12 +135,80 @@ def audit_descriptif(file) -> dict:
         number_tokens = count_tokens(text)
         number_words = len(text.split())
 
-        audit_dict_page["number_of_tokens"] = count_tokens(text)
-        audit_dict_page["number_of_words"] = len(text.split())
+        audit_dict_page["number_of_tokens"] = number_tokens
+        audit_dict_page["number_of_words"] = number_words
+
+        #get text
+        page_content["texte"] = text
 
         audit_dict_doc["number_of_tokens"] += number_tokens
         audit_dict_doc["number_of_words"] += number_words
 
         audit_dict_doc[f"page_{page.number}"] = audit_dict_page
 
-    return audit_dict_doc
+        doc_content[f"page_{page.number}"] = page_content
+    
+    # Extract key words from the document
+    text = " ".join([page["texte"] for page in doc_content.values()])
+    key_words = extract_keywords(text)
+    audit_dict_doc["key_words"] = key_words[:5]
+
+    #merge 2 dicts
+    global_audit = {
+        "audit": audit_dict_doc,
+        "content": doc_content
+    }
+
+    return global_audit
+
+def audit_text(text: str) -> dict:
+    audit_dict = {
+        "number_of_tokens": count_tokens(text),
+        "number_of_words": len(text.split())
+    }
+
+    return audit_dict
+
+
+
+def count_tokens(text):
+    # Implement a token counting method. Here, we assume tokens are words.
+    return len(text.split())
+
+def audit_descriptif_word(file) -> dict:
+    document = Document(io.BytesIO(file.read()))
+
+    audit_dict_doc = {
+        "number_of_paragraphs": 0,
+        "number_of_images": 0,
+        "number_of_links": 0,
+        "number_of_tables": 0,
+        "number_of_tokens": 0,
+        "number_of_words": 0
+    }
+
+    for para in document.paragraphs:
+        audit_dict_doc["number_of_paragraphs"] += 1
+        text = para.text
+
+        # Count tokens and words in the paragraph
+        number_tokens = count_tokens(text)
+        number_words = len(text.split())
+
+        audit_dict_doc["number_of_tokens"] += number_tokens
+        audit_dict_doc["number_of_words"] += number_words
+
+        # Count links (assuming they are hyperlinks)
+        for run in para.runs:
+            if run.link:
+                audit_dict_doc["number_of_links"] += 1
+
+    for table in document.tables:
+        audit_dict_doc["number_of_tables"] += 1
+
+    # Counting images (inline shapes and pictures)
+    for shape in document.inline_shapes:
+        audit_dict_doc["number_of_images"] += 1
+
+    return audit_dict_doc
+