Adds doc 2 speech node to keyword search and plays audio in UI

.gitignore

@@ -128,4 +128,6 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-.vscode/
+.vscode/
+
+data/audio/

core/pipelines.py

@@ -2,14 +2,19 @@
 Haystack Pipelines
 """
 
+from pathlib import Path
 from haystack import Pipeline
 from haystack.document_stores import InMemoryDocumentStore
 from haystack.nodes.retriever import DensePassageRetriever, TfidfRetriever
 from haystack.nodes.preprocessor import PreProcessor
 from haystack.nodes.ranker import SentenceTransformersRanker
+from haystack.nodes.audio.document_to_speech import DocumentToSpeech
+import os
 
+data_path = 'data/'
+os.makedirs(data_path, exist_ok=True)
 
-def keyword_search(index="documents", split_word_length=100):
+def keyword_search(index="documents", split_word_length=100, audio_output=False):
     """
     **Keyword Search Pipeline**
 
@@ -44,6 +49,13 @@ def keyword_search(index="documents", split_word_length=100):
     index_pipeline.add_node(
         document_store, name="DocumentStore", inputs=["TfidfRetriever"]
     )
+    
+    if audio_output:
+        doc2speech = DocumentToSpeech(
+            model_name_or_path="espnet/kan-bayashi_ljspeech_vits",
+            generated_audio_dir=Path(data_path + 'audio'),
+        )
+        search_pipeline.add_node(doc2speech, name='DocumentToSpeech', inputs=['TfidfRetriever'])
 
     return search_pipeline, index_pipeline
 

core/search_index.py

@@ -32,13 +32,17 @@ def search(queries, pipeline):
         for res in matches:
             if not score_is_empty:
                 score_is_empty = True if res.score is None else False
+            match = {
+                "text": res.content,
+                "id": res.meta["id"],
+                "fragment_id": res.id,
+            }
+            if not score_is_empty:
+                match.update({'score': res.score})
+            if hasattr(res, 'content_audio'):
+                match.update({'content_audio': res.content_audio})
             query_results.append(
-                {
-                    "text": res.content,
-                    "score": res.score,
-                    "id": res.meta["id"],
-                    "fragment_id": res.id,
-                }
+                match
             )
         if not score_is_empty:
             query_results = sorted(

interface/components.py

@@ -59,8 +59,10 @@ def component_show_search_result(container, results):
             st.markdown(f"### Match {idx+1}")
             st.markdown(f"**Text**: {document['text']}")
             st.markdown(f"**Document**: {document['id']}")
-            if document["score"] is not None:
+            if 'score' in document:
                 st.markdown(f"**Score**: {document['score']:.3f}")
+            if 'content_audio' in document:
+                st.audio(str(document['content_audio']))
             st.markdown("---")
 
 

requirements.txt

@@ -5,4 +5,8 @@ black==22.8.0
 plotly==5.10.0
 newspaper3k==0.2.8
 PyPDF2==2.10.7
-pytesseract==0.3.10
+pytesseract==0.3.10
+soundfile==0.10.3.post1
+espnet
+pydub==0.25.1
+espnet_model_zoo==0.1.7