Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| from PIL import Image, ImageDraw, ImageFont | |
| import scipy.io.wavfile as wavfile | |
| from transformers import pipeline | |
| narrator = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs") | |
| object_detector = pipeline("object-detection", model="facebook/detr-resnet-50") | |
| def generate_audio(text): | |
| # Generate the narrated text | |
| narrated_text = narrator(text) | |
| # Save the audio to a WAV file | |
| wavfile.write("output.wav", rate=narrated_text["sampling_rate"], data=narrated_text["audio"][0]) | |
| return "output.wav" | |
| def read_objects(detection_objects): | |
| object_counts = {} | |
| for detection in detection_objects: | |
| label = detection['label'] | |
| object_counts[label] = object_counts.get(label, 0) + 1 | |
| response = "This picture contains" | |
| labels = list(object_counts.keys()) | |
| for i, label in enumerate(labels): | |
| response += f" {object_counts[label]} {label}" + ("s" if object_counts[label] > 1 else "") | |
| if i < len(labels) - 2: | |
| response += "," | |
| elif i == len(labels) - 2: | |
| response += " and" | |
| response += "." | |
| return response | |
| def draw_bounding_boxes(image, detections, font_path=None, font_size=20): | |
| draw_image = image.copy() | |
| draw = ImageDraw.Draw(draw_image) | |
| if font_path: | |
| font = ImageFont.truetype(font_path, font_size) | |
| else: | |
| font = ImageFont.load_default() | |
| for detection in detections: | |
| box = detection['box'] | |
| xmin, ymin, xmax, ymax = box['xmin'], box['ymin'], box['xmax'], box['ymax'] | |
| draw.rectangle([(xmin, ymin), (xmax, ymax)], outline="red", width=3) | |
| label = detection['label'] | |
| score = detection['score'] | |
| text = f"{label} {score:.2f}" | |
| if font_path: | |
| text_size = draw.textbbox((xmin, ymin), text, font=font) | |
| else: | |
| text_size = draw.textbbox((xmin, ymin), text) | |
| draw.rectangle([(text_size[0], text_size[1]), (text_size[2], text_size[3])], fill="red") | |
| draw.text((xmin, ymin), text, fill="white", font=font) | |
| return draw_image | |
| def detect_object(image): | |
| raw_image = image | |
| output = object_detector(raw_image) | |
| processed_image = draw_bounding_boxes(raw_image, output) | |
| natural_text = read_objects(output) | |
| processed_audio = generate_audio(natural_text) | |
| return processed_image, processed_audio | |
| examples = [ | |
| ["dogs.jpg"] | |
| ] | |
| demo = gr.Interface( | |
| fn=detect_object, | |
| inputs=[gr.Image(label="Select Image", type="pil")], | |
| outputs=[ | |
| gr.Image(label="Processed Image", type="pil"), | |
| gr.Audio(label="Generated Audio") | |
| ], | |
| title="Audio Described Object Detector", | |
| description="This application highlights objects in the provided image and generates an audio description.", | |
| examples=examples | |
| ) | |
| demo.launch() | |