Update app.py

app.py

@@ -2,60 +2,105 @@ import torch
 from transformers import AutoFeatureExtractor, AutoModelForImageClassification
 from einops import rearrange
 import gradio
+import scipy.io.wavfile
+import time
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
 import call_labels
 
 # define the feature extractor
-extractor = AutoFeatureExtractor.from_pretrained("vincentclaes/mit-indoor-scenes")
+ViT_extractor = AutoFeatureExtractor.from_pretrained('vincentclaes/mit-indoor-scenes')
 
-# define the pretrained model
-model = AutoModelForImageClassification.from_pretrained("vincentclaes/mit-indoor-scenes")
+# define the pretrained ViT model
+ViT_model = AutoModelForImageClassification.from_pretrained('vincentclaes/mit-indoor-scenes')
 
-# retrieve the labels provided from MIT Indoor Scenes dataset (https://www.kaggle.com/itsahmad/indoor-scenes-cvpr-2019)
-labels = call_labels.call_labels()
+# call eval() to change the forward() behaviour of the module it is called upon
+ViT_model.eval()
 
-# call model.eval() to assert that we are evaluating the model and not updating the weights
-model.eval()
-
-# define the function used for model inference
-def classify(image):
-  # disable gradient calculation
+# define the function used for the ViT model inference
+def ViT_inference(image):
+  # disable gradient calculation/backpropagation
   with torch.no_grad():
     # extract features from the image input
-    inputs = extractor(images=image, return_tensors='pt')
+    inputs = ViT_extractor(images=image, return_tensors='pt')
     # call the logits parameter only (object: SequenceClassifierOutput)
-    outputs = model(**inputs).logits
+    outputs = ViT_model(**inputs).logits
     # remove the batch dimension
     outputs = rearrange(outputs, '1 j->j')
     # use the softmax function to convert the logits into probabilities
-    outputs = torch.nn.functional.softmax(outputs)
+    outputs = torch.nn.functional.softmax(outputs, dim=0)
     # convert the data type from tensor to a numpy array
     outputs = outputs.cpu().numpy()
-    # returns a key-value pair of the id labels and its corresponding probabilities
-    return {labels[str(i)]: float(outputs[i]) for i in range(len(labels))}
-
-# define the gradio interface
-gradio.Interface(fn=classify,
-                 inputs=gradio.inputs.Image(shape=(224,224),
-                                            image_mode='RGB',
-                                            source='upload',
-                                            tool='editor',
-                                            type='pil',
-                                            label=None,
-                                            optional=False),
-                 outputs=gradio.outputs.Label(num_top_classes=5,
-                                              type='auto'),
-                 theme='grass',
-                 examples=[['bedroom.jpg'],
-                           ['bathroom.jpg'],
-                           ['samsung_room.jpg']],
-                 live=True,
-                 layout='horizontal',
-                 title='Indoor Scene Recognition',
-                 description='A smart and easy-to-use indoor scene classifier. Start by uploading an input image of an indoor scene. The outputs are the top five indoor scene classes that best describe your input image.',
-                 article='''<h2>Additional Information</h2><p style='text-align: justify'>This indoor scene classifier employs the <b><a href='https://huggingface.co/google/vit-base-patch16-224-in21k' target='_blank'>google/vit-base-patch16-224-in21k</a></b>, a <b>Vision Transformer (ViT)</b> model pre-trained on the <b><a href='https://github.com/Alibaba-MIIL/ImageNet21K' target='_blank'>ImageNet-21k</a></b> (14 million images, 21,843 classes) at a resolution of 224 pixels by 224 pixels and was first introduced in the paper <b><a href='https://arxiv.org/abs/2010.11929' target='_blank'>An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale</a></b> by Dosovitskiy et al. It was then fine-tuned on the <b><a href='https://www.kaggle.com/itsahmad/indoor-scenes-cvpr-2019' target='_blank'>MIT Indoor Scenes</a></b> data set from Kaggle. The source model used in this space is from <b><a href='https://huggingface.co/vincentclaes/mit-indoor-scenes' target='_blank'>vincentclaes/mit-indoor-scenes</a></b>.</p>
-                 <p style='text-align: justify'>For further research on the Vision Transformer, the original GitHub repository is found in <b><a href='https://github.com/google-research/vision_transformer' target='_blank'>this link</a></b>.</p>
-                 <h2>Disclaimer</h2>
-                 <p style='text-align: justify'>The team releasing the Vision Transformer did not write a model card for it via Hugging Face. Hence, the Vision Transformer model card released in the Hugging Face Models library has been written by the Hugging Face team.</p>
-                 <h2>Limitations</h2>
-                 <p style='text-align: justify'>The model was trained only on 67 classes (indoor scenes). Hence, the model should perform better if the input indoor scene image belongs to one of the target classes it was trained on.</p>''',
-                 allow_flagging='never').launch()
+    # returns a key-value pair composed of id labels and its corresponding probabilities
+    # return {labels[str(i)]: float(outputs[i]) for i in range(len(labels))} '(Uncomment this for debugging purposes only.)'
+    # define a dictionary containing the key-value pair composed of id labels and its corresponding probabilities
+    logit_dict = {labels[str(i)]: float(outputs[i]) for i in range(len(labels))}
+    # retrieve the label with the maximum probability
+    max_key = max(logit_dict, key=logit_dict.get)
+    # format it as a string/text and pass it to a variable
+    tts_input = 'In front of you is the {}.'.format(str(max_key))
+    # returns a text format used as inputs to the text-to-speech model
+    return tts_input
+
+# define the ViT gradio interface
+ViT_interface = gradio.Interface(fn=ViT_inference,
+                                 inputs=gradio.inputs.Image(shape=(224,224),
+                                                            image_mode='RGB',
+                                                            source='upload',
+                                                            tool='editor',
+                                                            type='pil',
+                                                            label='Input: Indoor Scene Image'),
+                                 outputs='text')
+
+# define the pretrained TTS model
+TTS_model = Text2Speech.from_pretrained(
+    # call on the trained model
+    model_tag=str_or_none('kan-bayashi/ljspeech_vits'),
+    # set the vocoder
+    vocoder_tag=str_or_none('none'),
+    # set the device it should use
+    device='cpu',
+    # only for Tacotron 2 & Transformer
+    threshold=0.5,
+    # only for Tacotron 2
+    minlenratio=0.0,
+    maxlenratio=10.0,
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    # only for FastSpeech & FastSpeech2 & VITS
+    speed_control_alpha=1.0,
+    # only for VITS
+    noise_scale=0.333,
+    noise_scale_dur=0.333,
+)
+
+# define the function used for the TTS model inference
+def TTS_inference(text):
+  with torch.no_grad():
+    wav = TTS_model(text)['wav']
+    scipy.io.wavfile.write('out.wav',TTS_model.fs , wav.view(-1).cpu().numpy())
+  return 'out.wav'
+
+# define the TTS gradio interface
+TTS_interface = gradio.Interface(fn=TTS_inference,
+                                 inputs='text',
+                                 outputs=gradio.outputs.Audio(type='file', label='Output: Audio'))
+
+# Combine the two models using the gradio.mix.Series
+img2speech = gradio.mix.Series(ViT_interface, TTS_interface,
+                               theme='grass',
+                               live='True',
+                               layout='horizontal',
+                               title='''Hearing What's In Front of You: Indoor Scene Recognition-to-Speech''',
+                               description='For the blind and visually-impaired people. A smart and easy-to-use indoor scene classifier-to-speech. Start by uploading an input image of an indoor scene. The output is an audio file saying what you are externally facing in front of.',
+                               article='''<h2>Additional Information</h2><p style='text-align: justify'>This indoor scene classifier employs the <b><a href='https://huggingface.co/google/vit-base-patch16-224-in21k' target='_blank'>google/vit-base-patch16-224-in21k</a></b>, a <b>Vision Transformer (ViT)</b> model pre-trained on the <b><a href='https://github.com/Alibaba-MIIL/ImageNet21K' target='_blank'>ImageNet-21k</a></b> (14 million images, 21,843 classes) at a resolution of 224 pixels by 224 pixels and was first introduced in the paper <b><a href='https://arxiv.org/abs/2010.11929' target='_blank'>An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale</a></b> by Dosovitskiy et al. It was then fine-tuned on the <b><a href='https://www.kaggle.com/itsahmad/indoor-scenes-cvpr-2019' target='_blank'>MIT Indoor Scenes</a></b> data set from Kaggle. The source model used in this space is from <b><a href='https://huggingface.co/vincentclaes/mit-indoor-scenes' target='_blank'>vincentclaes/mit-indoor-scenes</a></b>.</p>
+                               <p style='text-align: justify'>For further research on the Vision Transformer, the original GitHub repository is found in <b><a href='https://github.com/google-research/vision_transformer' target='_blank'>this link</a></b>.</p>
+                               <p style='text-align: justify'>The Text-to-Speech model is from <b><a href='https://huggingface.co/espnet/kan-bayashi_ljspeech_vits' target='_blank'>espnet/kan-bayashi_ljspeech_vits</a></b>. It was imported from <b><a href='https://zenodo.org/record/5443814/' target='_blank'>this link</a></b> and was trained using the ljspeech/tts1 recipe in <b><a href='https://github.com/espnet/espnet/' target='_blank'>ESPnet: end-to-end speech processing toolkit</a></b>. The published work being referenced by this model is found in <b><a href='https://arxiv.org/pdf/1804.00015.pdf' target='_blank'>this link</a></b>.</p>
+                               <h2>Disclaimer</h2>
+                               <p style='text-align: justify'>The team releasing the Vision Transformer did not write a model card for it via Hugging Face. Hence, the Vision Transformer model card released in the Hugging Face Models library has been written by the Hugging Face team.</p>
+                               <h2>Limitations</h2>
+                               <p style='text-align: justify'>The model was trained only on 67 classes (indoor scenes). Hence, the model should perform better if the input indoor scene image belongs to one of the target classes it was trained on. For demonstration purposes, it temporarily accommodates English as its language but it is flexible and versatile to other common major languages.</p>
+                               <h2>Credits</h2>
+                               <p style='text-align: justify'>I would like to express my gratitude to <b><a href='https://github.com/vincentclaes' target='_blank'>Vincent Claes</a></b> and <b><a href='https://github.com/siddhu001' target='_blank'>Siddhant Arora</a></b> for uploading the ViT and TTS models in the Hugging Face Models library, respectively, purely for academic and research purposes. All credits go to these two brilliant people.''',
+                               allow_flagging='never').launch()