all

app.py

@@ -1,58 +1,29 @@
 
 import gradio as gr
 from pipeline.run_pipeline import *
+'''
+时间优化
+并发优化
+'''
+
 # from run import *
 # '''
 # 把一些文件移动到此文件路径下
 # '''
 # text = "A person is cutting a birthday cake with two red candles that spell out \"21\". The surface of the cake is round, and there is a balloon in the room. The person is using a silver knife to cut the cake."
 # image_path = "/newdisk3/wcx/val2014/COCO_val2014_000000297425.jpg"
+pipeline = Pipeline(type="image-to-text", api_key="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")
+# res,claim_list = pipeline.run(text=text, image_path=image_path,type="image-to-text")
+# print(res)
 
-def get_response(text, filepath):
-    pipeline = Pipeline()
-    res = pipeline.run(text=text, image_path=image_path)
-    return res
+def get_response(text, filepath, type):
+    res, claim_list = pipeline.run(text=text, image_path=filepath, type=type)
+    return claim_list, res
 
 demo = gr.Interface(
     fn=get_response,
-    inputs=["text", gr.Image(type="filepath")],
-    outputs=["text"],
+    inputs=[gr.Textbox(placeholder="Input I2T model's response or T2I model's prompt", label="text input"), gr.Image(type="filepath", label="image input"), gr.Radio(['image-to-text','text-to-image'], label='task type', value='image-to-text')],
+    outputs=[gr.Textbox(label="claim list"), gr.Textbox(label="detect results")],
 )
 
-demo.launch()
-
-# def generate_mutimodal(title, context, img):
-#     return f"Title:{title}\nContext:{context}\n...{img}"
-
-# server = gr.Interface(
-#     fn=generate_mutimodal, 
-#     inputs=[
-#         gr.Textbox(lines=1, placeholder="请输入标题"),
-#         gr.Textbox(lines=2, placeholder="请输入正文"),
-#         gr.Image(shape=(200, 200),  label="请上传图片(可选)")
-#     ], 
-#     outputs="text"
-# )
-
-# server.launch()
-
-# import numpy as np
-# import gradio as gr
-# def sepia(input_img):
-#     #处理图像
-#     sepia_filter = np.array([
-#         [0.393, 0.769, 0.189],
-#         [0.349, 0.686, 0.168],
-#         [0.272, 0.534, 0.131]
-#     ])
-#     sepia_img = input_img.dot(sepia_filter.T)
-#     sepia_img /= sepia_img.max()
-#     return sepia_img
-# #shape设置输入图像大小
-# demo = gr.Interface(sepia, gr.Image(), "image")
-# demo.launch()
-
-# Download human-readable labels for ImageNet.
-
-# gr.Interface(fn=sepia,inputs=gr.Image(type="pil"),outputs="image").launch()
-
+demo.queue().launch(share=True)

models/dbnetpp.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ecb39ac54bd954fa44fd25b419b7a51539ded0f79408eb2419ca78f9f0299c6
+size 113299489

models/groundingdino_swint_ogc.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b3ca2563c77c69f651d7bd133e97139c186df06231157a64c507099c52bc799
+size 693997677

models/maerec_b.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a98230f5421517bce9fdc197261ec6845e353ceb7fa970194c04403e0ba89b
+size 1764777173

pipeline/judge.py

@@ -2,9 +2,9 @@ import json
 import yaml
 import base64
 class Judger:
-    def __init__(self, prompt_path, chat, type):
+    def __init__(self, prompt_path, chat):
         with open(prompt_path,"r",encoding='utf-8') as file:
-            self.prompt = yaml.load(file, yaml.FullLoader)[type]
+            self.prompt = yaml.load(file, yaml.FullLoader)
         self.chat = chat
         
     
@@ -12,7 +12,7 @@ class Judger:
         with open(image_path, "rb") as image_file:
             return base64.b64encode(image_file.read()).decode('utf-8')
         
-    def get_response(self, object_res, attribue_res, text_res, fact_res, claim_list, image_path):
+    def get_response(self, type, object_res, attribue_res, text_res, fact_res, claim_list, image_path):
         input = '''
                 Here is the object detection expert model's result:
                 {object}
@@ -40,24 +40,28 @@ class Judger:
         else:
             text_det = "none information"
             
-            
-        img1 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/animal.jpg")
-        img2 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/ball.jpg")
+        if type == "image-to-text":
+            img1 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/sandbeach.jpg")
+            img2 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/football.jpg")
+        else:
+            img1 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/animal.jpg")
+            img2 = self.encode_image("/home/wcx/wcx/GroundingDINO/LVLM/cot/img_examples/ball.jpg")
         base64_source_image = self.encode_image(image_path)
         content = [
-                    {"type": "text", "text": self.prompt["user"]},
+                    {"type": "text", "text": self.prompt[type]["user"]},
                     {"type": "image_url","image_url": f"data:image/jpeg;base64,{img1}"},
-                    {"type": "text", "text": self.prompt["example1"]},
+                    {"type": "text", "text": self.prompt[type]["example1"]},
                     {"type": "image_url","image_url": f"data:image/jpeg;base64,{img2}"},
-                    {"type": "text", "text": self.prompt["example2"]},
+                    {"type": "text", "text": self.prompt[type]["example2"]},
                     {"type": "image_url","image_url": f"data:image/jpeg;base64,{base64_source_image}"},
                     {"type": "text", "text": input.format(object=object_det,text=text_det,fact=fact_res,claims=claim_list)}
                 ]
-            
+
+        
         message = [
                 {
                     'role': 'system',
-                    'content': self.prompt["system"]
+                    'content': self.prompt[type]["system"]
                 },
                 {
                     "role": "user",

pipeline/nltk/VERSION

@@ -1 +0,0 @@
-3.8.1

pipeline/nltk/__init__.py

@@ -1,209 +0,0 @@
-# Natural Language Toolkit (NLTK)
-#
-# Copyright (C) 2001-2023 NLTK Project
-# Authors: Steven Bird <[email protected]>
-#          Edward Loper <[email protected]>
-# URL: <https://www.nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-The Natural Language Toolkit (NLTK) is an open source Python library
-for Natural Language Processing.  A free online book is available.
-(If you use the library for academic research, please cite the book.)
-
-Steven Bird, Ewan Klein, and Edward Loper (2009).
-Natural Language Processing with Python.  O'Reilly Media Inc.
-https://www.nltk.org/book/
-
-isort:skip_file
-"""
-
-import os
-
-# //////////////////////////////////////////////////////
-# Metadata
-# //////////////////////////////////////////////////////
-
-# Version.  For each new release, the version number should be updated
-# in the file VERSION.
-try:
-    # If a VERSION file exists, use it!
-    version_file = os.path.join(os.path.dirname(__file__), "VERSION")
-    with open(version_file) as infile:
-        __version__ = infile.read().strip()
-except NameError:
-    __version__ = "unknown (running code interactively?)"
-except OSError as ex:
-    __version__ = "unknown (%s)" % ex
-
-if __doc__ is not None:  # fix for the ``python -OO``
-    __doc__ += "\n@version: " + __version__
-
-
-# Copyright notice
-__copyright__ = """\
-Copyright (C) 2001-2023 NLTK Project.
-
-Distributed and Licensed under the Apache License, Version 2.0,
-which is included by reference.
-"""
-
-__license__ = "Apache License, Version 2.0"
-# Description of the toolkit, keywords, and the project's primary URL.
-__longdescr__ = """\
-The Natural Language Toolkit (NLTK) is a Python package for
-natural language processing.  NLTK requires Python 3.7, 3.8, 3.9, 3.10 or 3.11."""
-__keywords__ = [
-    "NLP",
-    "CL",
-    "natural language processing",
-    "computational linguistics",
-    "parsing",
-    "tagging",
-    "tokenizing",
-    "syntax",
-    "linguistics",
-    "language",
-    "natural language",
-    "text analytics",
-]
-__url__ = "https://www.nltk.org/"
-
-# Maintainer, contributors, etc.
-__maintainer__ = "NLTK Team"
-__maintainer_email__ = "[email protected]"
-__author__ = __maintainer__
-__author_email__ = __maintainer_email__
-
-# "Trove" classifiers for Python Package Index.
-__classifiers__ = [
-    "Development Status :: 5 - Production/Stable",
-    "Intended Audience :: Developers",
-    "Intended Audience :: Education",
-    "Intended Audience :: Information Technology",
-    "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: Apache Software License",
-    "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Topic :: Scientific/Engineering",
-    "Topic :: Scientific/Engineering :: Artificial Intelligence",
-    "Topic :: Scientific/Engineering :: Human Machine Interfaces",
-    "Topic :: Scientific/Engineering :: Information Analysis",
-    "Topic :: Text Processing",
-    "Topic :: Text Processing :: Filters",
-    "Topic :: Text Processing :: General",
-    "Topic :: Text Processing :: Indexing",
-    "Topic :: Text Processing :: Linguistic",
-]
-
-from nltk.internals import config_java
-
-# support numpy from pypy
-try:
-    import numpypy
-except ImportError:
-    pass
-
-# Override missing methods on environments where it cannot be used like GAE.
-import subprocess
-
-if not hasattr(subprocess, "PIPE"):
-
-    def _fake_PIPE(*args, **kwargs):
-        raise NotImplementedError("subprocess.PIPE is not supported.")
-
-    subprocess.PIPE = _fake_PIPE
-if not hasattr(subprocess, "Popen"):
-
-    def _fake_Popen(*args, **kwargs):
-        raise NotImplementedError("subprocess.Popen is not supported.")
-
-    subprocess.Popen = _fake_Popen
-
-###########################################################
-# TOP-LEVEL MODULES
-###########################################################
-
-# Import top-level functionality into top-level namespace
-
-from nltk.collocations import *
-from nltk.decorators import decorator, memoize
-from nltk.featstruct import *
-from nltk.grammar import *
-from nltk.probability import *
-from nltk.text import *
-from nltk.util import *
-from nltk.jsontags import *
-
-###########################################################
-# PACKAGES
-###########################################################
-
-from nltk.chunk import *
-from nltk.classify import *
-from nltk.inference import *
-from nltk.metrics import *
-from nltk.parse import *
-from nltk.tag import *
-from nltk.tokenize import *
-from nltk.translate import *
-from nltk.tree import *
-from nltk.sem import *
-from nltk.stem import *
-
-# Packages which can be lazily imported
-# (a) we don't import *
-# (b) they're slow to import or have run-time dependencies
-#     that can safely fail at run time
-
-from nltk import lazyimport
-
-app = lazyimport.LazyModule("app", locals(), globals())
-chat = lazyimport.LazyModule("chat", locals(), globals())
-corpus = lazyimport.LazyModule("corpus", locals(), globals())
-draw = lazyimport.LazyModule("draw", locals(), globals())
-toolbox = lazyimport.LazyModule("toolbox", locals(), globals())
-
-# Optional loading
-
-try:
-    import numpy
-except ImportError:
-    pass
-else:
-    from nltk import cluster
-
-from nltk.downloader import download, download_shell
-
-try:
-    import tkinter
-except ImportError:
-    pass
-else:
-    try:
-        from nltk.downloader import download_gui
-    except RuntimeError as e:
-        import warnings
-
-        warnings.warn(
-            "Corpus downloader GUI not loaded "
-            "(RuntimeError during import: %s)" % str(e)
-        )
-
-# explicitly import all top-level modules (ensuring
-# they override the same names inadvertently imported
-# from a subpackage)
-
-from nltk import ccg, chunk, classify, collocations
-from nltk import data, featstruct, grammar, help, inference, metrics
-from nltk import misc, parse, probability, sem, stem, wsd
-from nltk import tag, tbl, text, tokenize, translate, tree, util
-
-
-# FIXME:  override any accidentally imported demo, see https://github.com/nltk/nltk/issues/2116
-def demo():
-    print("To run the demo code for a module, type nltk.module.demo()")

pipeline/nltk/app/__init__.py

@@ -1,47 +0,0 @@
-# Natural Language Toolkit: Applications package
-#
-# Copyright (C) 2001-2023 NLTK Project
-# Author: Edward Loper <[email protected]>
-#         Steven Bird <[email protected]>
-# URL: <https://www.nltk.org/>
-# For license information, see LICENSE.TXT
-
-"""
-Interactive NLTK Applications:
-
-chartparser:  Chart Parser
-chunkparser:  Regular-Expression Chunk Parser
-collocations: Find collocations in text
-concordance:  Part-of-speech concordancer
-nemo:         Finding (and Replacing) Nemo regular expression tool
-rdparser:     Recursive Descent Parser
-srparser:     Shift-Reduce Parser
-wordnet:      WordNet Browser
-"""
-
-
-# Import Tkinter-based modules if Tkinter is installed
-try:
-    import tkinter
-except ImportError:
-    import warnings
-
-    warnings.warn("nltk.app package not loaded (please install Tkinter library).")
-else:
-    from nltk.app.chartparser_app import app as chartparser
-    from nltk.app.chunkparser_app import app as chunkparser
-    from nltk.app.collocations_app import app as collocations
-    from nltk.app.concordance_app import app as concordance
-    from nltk.app.nemo_app import app as nemo
-    from nltk.app.rdparser_app import app as rdparser
-    from nltk.app.srparser_app import app as srparser
-    from nltk.app.wordnet_app import app as wordnet
-
-    try:
-        from matplotlib import pylab
-    except ImportError:
-        import warnings
-
-        warnings.warn("nltk.app.wordfreq not loaded (requires the matplotlib library).")
-    else:
-        from nltk.app.wordfreq_app import app as wordfreq