Upload 28 files

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/common_object_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/common_object_vis2.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/dense_object_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/dense_object_vis2.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/edge_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/long_caption_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/long_caption_vis2.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/long_caption_vis3.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/longtail_object_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/short_caption_vis.png filter=lfs diff=lfs merge=lfs -text
+asset/qualitative_visualization/video_object_vis.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,77 @@
+import os 
+import argparse
+from typing import Dict, List
+from gdino import GroundingDINOAPIWrapper, visualize
+import gradio as gr
+import numpy as np
+import cv2
+def arg_parse():
+ parser = argparse.ArgumentParser(description="Gradio Demo for T-Rex2")
+ parser.add_argument(
+ "--token",
+ type=str,
+ help="This token is only for gradio space. Please do not take it away for your own purpose!",
+ )
+ args = parser.parse_args()
+ return args
+
+def resize_image_with_aspect_ratio(image: np.ndarray, min_size: int = 800, max_size: int = 1333) -> np.ndarray:
+ h, w = image.shape[:2]
+ aspect_ratio = w / h
+
+ # Determine the scaling factor based on the constraints
+ if h < w:
+ new_height = min_size
+ new_width = int(new_height * aspect_ratio)
+ if new_width > max_size:
+ new_width = max_size
+ new_height = int(new_width / aspect_ratio)
+ else:
+ new_width = min_size
+ new_height = int(new_width / aspect_ratio)
+ if new_height > max_size:
+ new_height = max_size
+ new_width = int(new_height * aspect_ratio)
+
+ # Resize the image
+ resized_image = cv2.resize(image, (new_width, new_height), interpolation=cv2.INTER_AREA)
+
+ return resized_image
+
+def inference(image, prompt: str, return_mask: bool = False, return_score: bool = False) -> gr.Image:
+ # shrink image first to save computation
+ if return_mask:
+ image = resize_image_with_aspect_ratio(image, min_size=600, max_size=1000)
+ prompts = dict(image=image, prompt=prompt)
+ results = gdino.inference(prompts, return_mask=return_mask)
+ image_pil = visualize(image, results, return_mask=return_mask, draw_score=return_score)
+ return image_pil
+
+args = arg_parse()
+gdino = GroundingDINOAPIWrapper(args.token)
+
+if __name__ == "__main__":
+ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue")) as demo:
+ with gr.Row():
+ with gr.Column():
+ input_image = gr.Image(label="Input Image")
+ with gr.Column():
+ output_image = gr.Image(label="Output Image")
+ with gr.Row():
+ return_mask = gr.Checkbox(label="Return Mask")
+ return_score = gr.Checkbox(label="Return Score")
+ prompt = gr.Textbox(label="Prompt", placeholder="e.g., person.pigeon.tree")
+ run = gr.Button(value="Run")
+ with gr.Row():
+ gr.Examples(
+ examples=[
+ ['asset/demo.jpg', 'person . pigeon . tree'],
+ ['asset/demo2.jpeg', 'wireless walkie-talkie . life jacket . atlantic cod . man . vehicle . accessory . cell phone .'],
+ ['asset/demo3.jpeg', 'wine rack . bottle . basket'],
+ ['asset/demo4.jpeg', 'Mosque. golden dome. smaller domes. minarets. arched windows. white facade. cars. electrical lines. streetlights. trees. pedestrians. blue sky. shadows'],
+ ['asset/demo5.jpeg', 'stately building. columns. sculptures. Spanish flag. clouds. blue sky. street. taxis. van. city bus. traffic lights. street lamps. road markings. pedestrians. sidewalk. traffic sign. palm trees']
+ ],
+ inputs=[input_image, prompt],
+ )
+ run.click(inference, inputs=[input_image, prompt, return_mask, return_score], outputs=output_image)
+ demo.launch(debug=True)

gdino/__init__.py

@@ -0,0 +1,4 @@
+from .model_wrapper import GroundingDINOAPIWrapper
+from .visualize import visualize
+
+__all__ = ["GroundingDINOAPIWrapper", "visualize"]

gdino/model_wrapper.py

@@ -0,0 +1,132 @@
+import tempfile
+from typing import Dict, List, Union
+import numpy as np
+from dds_cloudapi_sdk import (
+ DetectionTask,
+ Client,
+ Config,
+ TextPrompt,
+ DetectionModel,
+ DetectionTarget,
+)
+from PIL import Image
+import concurrent.futures
+
+class GroundingDINOAPIWrapper:
+ """API wrapper for Grounding DINO 1.5
+
+ Args:
+ token (str): The token for Grounding DINO 1.5 API. We are now opening free API access to Grounding DINO 1.5. For
+ educators, students, and researchers, we offer an API with extensive usage times to
+ support your educational and research endeavors. You can get free API token at here:
+ https://deepdataspace.com/request_api
+
+ """
+
+ def __init__(self, token: str):
+ self.client = Client(Config(token=token))
+
+ def inference(self, prompt: Dict, return_mask:bool=False):
+ """Main inference function of Grounding DINO 1.5. We take batch as input and
+ each image is a dict. N. We do not support batch inference for now.
+
+ Args:
+ prompts (dict): Annotations with the following keys:
+ - "image" (str): Path to image. E.g. "test1.jpg",
+ - "prompt" (str): Text prompt sepearted by '.' E.g. 'cate1 . cate2 . cate3'
+ return_mask (bool): Whether to return mask. Defaults to False.
+
+ Returns:
+ (Dict): Detection results in dict format with keys::
+ - "scores": (List[float]): A list of scores for each object in the batch
+ - "labels": (List[int]): A list of labels for each object in the batch
+ - "boxes": (List[List[int]]): A list of boxes for each object in the batch,
+ in format [xmin, ymin, xmax, ymax]
+ - "masks": (List[np.ndarray]): A list of segmentations for each object in the batch
+ """
+ # construct input prompts
+ image=self.get_image_url(prompt["image"]),
+ task=DetectionTask(
+ image_url=image[0],
+ prompts=[TextPrompt(text=prompt['prompt'])],
+ targets=[DetectionTarget.Mask, DetectionTarget.BBox] if return_mask else [DetectionTarget.BBox],
+ model=DetectionModel.GDino1_5_Pro,
+ )
+ self.client.run_task(task)
+ result = task.result
+ return self.postprocess(result, task, return_mask)
+
+
+ def postprocess(self, result, task, return_mask):
+ """Postprocess the result from the API call
+
+ Args:
+ result (TaskResult): Task result with the following keys:
+ - objects (List[DetectionObject]): Each DetectionObject has the following keys:
+ - bbox (List[float]): Box in xyxy format
+ - category (str): Detection category
+ - score (float): Detection score
+ - mask (DetectionObjectMask): Use mask.counts to parse RLE mask 
+ task (DetectionTask): The task object
+ return_mask (bool): Whether to return mask
+
+ Returns:
+ (Dict): Return dict in format:
+ {
+ "scores": (List[float]): A list of scores for each object
+ "categorys": (List[str]): A list of categorys for each object
+ "boxes": (List[List[int]]): A list of boxes for each object
+ "masks": (List[PIL.Image]): A list of masks in the format of PIL.Image
+ }
+ """
+ def process_object_with_mask(object):
+ box = object.bbox
+ score = object.score
+ category = object.category
+ mask = task.rle2rgba(object.mask)
+ return box, score, category, mask
+ 
+ def process_object_without_mask(object):
+ box = object.bbox
+ score = object.score
+ category = object.category
+ mask = None
+ return box, score, category, mask
+ 
+ boxes, scores, categorys, masks = [], [], [], []
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+ if return_mask:
+ process_object = process_object_with_mask
+ else:
+ process_object = process_object_without_mask
+ futures = [executor.submit(process_object, obj) for obj in result.objects]
+ for future in concurrent.futures.as_completed(futures):
+ box, score, category, mask = future.result()
+ boxes.append(box)
+ scores.append(score)
+ categorys.append(category)
+ if mask is not None:
+ masks.append(mask)
+
+ return dict(boxes=boxes, categorys=categorys, scores=scores, masks=masks)
+
+ def get_image_url(self, image: Union[str, np.ndarray]):
+ """Upload Image to server and return the url
+
+ Args:
+ image (Union[str, np.ndarray]): The image to upload. Can be a file path or np.ndarray.
+ If it is a np.ndarray, it will be saved to a temporary file.
+
+ Returns:
+ str: The url of the image
+ """
+ if isinstance(image, str):
+ url = self.client.upload_file(image)
+ else:
+ with tempfile.NamedTemporaryFile(delete=True, suffix=".png") as tmp_file:
+ # image is in numpy format, convert to PIL Image
+ image = Image.fromarray(image)
+ image.save(tmp_file, format="PNG")
+ tmp_file_path = tmp_file.name
+ url = self.client.upload_file(tmp_file_path)
+ return url

gdino/version.py

@@ -0,0 +1 @@
+__version__ = 'v1.5'

gdino/visualize.py

@@ -0,0 +1,108 @@
+from typing import Dict
+
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont, ImageOps
+import random
+
+
+def draw_mask(mask, draw, random_color=True):
+ """Draws a mask with a specified color on an image.
+
+ Args:
+ mask (np.array): Binary mask as a NumPy array.
+ draw (ImageDraw.Draw): ImageDraw object to draw on the image.
+ random_color (bool): Whether to use a random color for the mask.
+ """
+ if random_color:
+ color = (
+ random.randint(0, 255),
+ random.randint(0, 255),
+ random.randint(0, 255),
+ 153,
+ )
+ else:
+ color = (30, 144, 255, 153)
+
+ nonzero_coords = np.transpose(np.nonzero(mask))
+ 
+ for coord in nonzero_coords:
+ draw.point(coord[::-1], fill=color)
+
+def visualize(image_pil: Image,
+ result: Dict,
+ draw_width: float = 6.0,
+ return_mask=True,
+ draw_score=True) -> Image:
+ """Plot bounding boxes and labels on an image.
+
+ Args:
+ image_pil (PIL.Image): The input image as a PIL Image object.
+ result (Dict[str, Union[torch.Tensor, List[torch.Tensor]]]): The target dictionary containing
+ the bounding boxes and labels. The keys are:
+ - boxes (List[int]): A list of bounding boxes in shape (N, 4), [x1, y1, x2, y2] format.
+ - scores (List[float]): A list of scores for each bounding box. shape (N)
+ - categorys (List[str]): A list of categorys for each object
+ - masks (List[PIL.Image]): A list of masks in the format of PIL.Image
+ draw_score (bool): Draw score on the image. Defaults to False.
+
+ Returns:
+ PIL.Image: The input image with plotted bounding boxes, labels, and masks.
+ """
+ # Get the bounding boxes and labels from the target dictionary
+ boxes = result["boxes"]
+ scores = result["scores"]
+ categorys = result["categorys"]
+ masks = result.get("masks", [])
+
+ # Find all unique categories and build a cate2color dictionary
+ cate2color = {}
+ unique_categorys = set(categorys)
+ for cate in unique_categorys:
+ cate2color[cate] = tuple(np.random.randint(0, 255, size=3).tolist())
+
+ # Create a PIL ImageDraw object to draw on the input image
+ if isinstance(image_pil, np.ndarray):
+ image_pil = Image.fromarray(image_pil)
+ draw = ImageDraw.Draw(image_pil)
+ 
+ # Create a new binary mask image with the same size as the input image
+ mask = Image.new("L", image_pil.size, 0)
+ # Create a PIL ImageDraw object to draw on the mask image
+ mask_draw = ImageDraw.Draw(mask)
+
+ # Draw boxes, labels, and masks for each box and label in the target dictionary
+ for box, score, category in zip(boxes, scores, categorys):
+ # Extract the box coordinates
+ x0, y0, x1, y1 = box
+ x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)
+ color = cate2color[category]
+
+ # Draw the box outline on the input image
+ draw.rectangle([x0, y0, x1, y1], outline=color, width=int(draw_width))
+
+ # Draw the label and score on the input image
+ if draw_score:
+ text = f"{category} {score:.2f}"
+ else:
+ text = f"{category}"
+ 
+ font = ImageFont.load_default()
+ if hasattr(font, "getbbox"):
+ bbox = draw.textbbox((x0, y0), text, font)
+ else:
+ w, h = draw.textsize(text, font)
+ bbox = (x0, y0, w + x0, y0 + h)
+ draw.rectangle(bbox, fill=color)
+ draw.text((x0, y0), text, fill="white")
+
+ # Draw the mask on the input image if masks are provided
+ if len(masks) > 0 and return_mask:
+ size = image_pil.size
+ mask_image = Image.new("RGBA", size, color=(0, 0, 0, 0))
+ mask_draw = ImageDraw.Draw(mask_image)
+ for mask in masks:
+ mask = np.array(mask)[:, :, -1]
+ draw_mask(mask, mask_draw)
+
+ image_pil = Image.alpha_composite(image_pil.convert("RGBA"), mask_image).convert("RGB")
+ return image_pil

requirements.txt

@@ -0,0 +1,2 @@
+dds-cloudapi-sdk==0.2.1
+gradio==4.22.0

setup.py

@@ -0,0 +1,126 @@
+import glob
+import os
+import subprocess
+
+import torch
+from setuptools import find_packages, setup
+from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
+
+version = "v1.5"
+package_name = "gdino"
+cwd = os.path.dirname(os.path.abspath(__file__))
+
+sha = "Unknown"
+try:
+ sha = subprocess.check_output(["git", "rev-parse", "HEAD"],
+ cwd=cwd).decode("ascii").strip()
+except Exception:
+ pass
+
+
+def write_version_file():
+ version_path = os.path.join(cwd, "gdino/", "version.py")
+ with open(version_path, "w") as f:
+ f.write(f"__version__ = '{version}'\n")
+ # f.write(f"git_version = {repr(sha)}\n")
+
+
+def parse_requirements(fname="requirements.txt", with_version=True):
+ """Parse the package dependencies listed in a requirements file but strips
+ specific versioning information.
+
+ Args:
+ fname (str): path to requirements file
+ with_version (bool, default=False): if True include version specs
+
+ Returns:
+ List[str]: list of requirements items
+
+ CommandLine:
+ python -c "import setup; print(setup.parse_requirements())"
+ """
+ import re
+ import sys
+ from os.path import exists
+
+ require_fpath = fname
+
+ def parse_line(line):
+ """Parse information from a line in a requirements text file."""
+ if line.startswith("-r "):
+ # Allow specifying requirements in other files
+ target = line.split(" ")[1]
+ for info in parse_require_file(target):
+ yield info
+ else:
+ info = {"line": line}
+ if line.startswith("-e "):
+ info["package"] = line.split("#egg=")[1]
+ elif "@git+" in line:
+ info["package"] = line
+ else:
+ # Remove versioning from the package
+ pat = "(" + "|".join([">=", "==", ">"]) + ")"
+ parts = re.split(pat, line, maxsplit=1)
+ parts = [p.strip() for p in parts]
+
+ info["package"] = parts[0]
+ if len(parts) > 1:
+ op, rest = parts[1:]
+ if ";" in rest:
+ # Handle platform specific dependencies
+ # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
+ version, platform_deps = map(str.strip,
+ rest.split(";"))
+ info["platform_deps"] = platform_deps
+ else:
+ version = rest # NOQA
+ info["version"] = (op, version)
+ yield info
+
+ def parse_require_file(fpath):
+ with open(fpath, "r") as f:
+ for line in f.readlines():
+ line = line.strip()
+ if line and not line.startswith("#"):
+ for info in parse_line(line):
+ yield info
+
+ def gen_packages_items():
+ if exists(require_fpath):
+ for info in parse_require_file(require_fpath):
+ parts = [info["package"]]
+ if with_version and "version" in info:
+ parts.extend(info["version"])
+ if not sys.version.startswith("3.4"):
+ # apparently package_deps are broken in 3.4
+ platform_deps = info.get("platform_deps")
+ if platform_deps is not None:
+ parts.append(";" + platform_deps)
+ item = "".join(parts)
+ yield item
+
+ packages = list(gen_packages_items())
+ return packages
+
+
+if __name__ == "__main__":
+ print(f"Building wheel {package_name}-{version}")
+
+ with open("LICENSE", "r", encoding="utf-8") as f:
+ license = f.read()
+
+ write_version_file()
+
+ setup(
+ name="gdino",
+ version="v1.5",
+ author="International Digital Economy Academy, CVR",
+ url="https://github.com/IDEA-Research/Grounding-DINO-1.5-API",
+ description="Grounding DINO 1.5 API wrapper.",
+ license=license,
+ install_requires=parse_requirements("requirements.txt"),
+ packages=find_packages(exclude=("tests", )),
+ ext_modules=None,
+ cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
+ )