fix: update

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[server]
+maxUploadSize = 20

app.py

@@ -1,40 +1,93 @@
 import streamlit as st
-from fengxai.responsive import setModel, getModel, updateModel, applyBindingController
+from fengxai.render import setModel, getModel, updateModel, applyBindingController
+from fengxai.inference import imagePrediction
+from PIL import Image
+
 
 ## model
-setModel("distance", 30.07)        # (英尺/米)到物体的距离
-setModel("sensorHeight", 24)       # (毫米)传感器高度
-setModel("imageObjectHeight", 2724)# (像素)图像中物体的高度
-setModel("lensFocalLength", 60)    # (像素)镜头焦距
-setModel("imageHeight", 4912)      # (像素)图片的高度    
-## and controller     
-updateModel(
+setModel("distance", 30.07)                                                         # (英尺/米)到物体的距离
+setModel("sensorHeight", 24)                                                        # (毫米)传感器高度
+setModel("imageObjectHeight", 2724)                                                 # (像素)图像中物体的高度
+setModel("lensFocalLength", 60)                                                     # (像素)镜头焦距
+setModel("imageHeight", 4912)                                                       # (像素)图片的高度
+## and controller
+updateModel(                                                                        # 物体高度
     "objectHeight", 
-    getModel("sensorHeight") * getModel("imageObjectHeight") * getModel("distance") / (getModel("lensFocalLength") * getModel("imageHeight"))
-)                                  # 物体高度
+    str(getModel("sensorHeight") * getModel("imageObjectHeight") * getModel("distance") / (getModel("lensFocalLength") * getModel("imageHeight")))
+)
+## (default)set image info
+setModel("imageInfo")
+
+def changeUploadFile():
+    updateModel("imageInfo", {
+        "imageFile": getModel("imageUploadFile"),
+        "isUploadImageFile": True                                                   # 图片上传开关，控制ai模型在正确的事件下识别图像物体
+    })
+    
+def changeDelete():
+    updateModel("imageInfo", "")
 
 ## view: side 左侧边栏
 def render():
     st.set_page_config(layout="wide") 
-    
+
     with st.sidebar:
         st.title("example")
-        st.image("assets/e1.png")
-    
-    st.title("Rjxai v4_1: :blue[Get object height]") 
-       
+        st.image("assets/e2.jpg")
+        if st.button('use'):
+            updateModel("imageInfo", {
+                "imageFile": "assets/e2.jpg",
+                "isUploadImageFile": True
+            })
+
+    # page
+    st.title("Rjxai v4_1: :blue[Get object height]")
+    with st.container():
+        st.title("Step 1: Rjxai Image Identification")
+        st.file_uploader(
+            label = "Upload your image here: png, jpg, jpeg", 
+            type=['png','jpg','jpeg'],
+            on_change=changeUploadFile,
+            key="imageUploadFile",
+        )
+        if getModel('imageInfo') != "":
+            def callback(output):
+                updateModel("imageHeight", output.imageHeight)
+                updateModel("imageObjectHeight", output.objectHeight)
+                c1, c2 = st.columns([0.7, 0.3])
+                with c1.container():
+                    imageInput = Image.open(output.imageAnnotateFile)
+                    st.image(imageInput)
+                with c2.container():
+                    st.button("delete image", on_click=changeDelete)
+
+            if getModel("imageInfo")["isUploadImageFile"]:
+                def tmpCallback(output):
+                    updateModel("imageInfo", {
+                        "imageFile": getModel("imageInfo")["imageFile"],
+                        "tmpUploadImageInfo": output,                               # 存放图片的临时信息
+                        "isUploadImageFile": False
+                    })
+                    callback(getModel("imageInfo")["tmpUploadImageInfo"])
+                imagePrediction(getModel('imageFile')["imageFile"], tmpCallback)
+            else:
+                callback(getModel("imageInfo")["tmpUploadImageInfo"])
+
+    with st.container():
+        st.title("Step 2: Calculate real object height")
+
     col1, col2 = st.columns(2)
     with col1.container():
-        st.title("Input:")
-        st.number_input('(feet or meters) distance', key="distance")
+        st.subheader("Input:")
         st.number_input('(pixels) image height', key="imageHeight")
         st.number_input('(pixels) image object height', key="imageObjectHeight")
         st.number_input('(mm) sensor height', key="sensorHeight")
         st.number_input('(mm) lens focal length', key="lensFocalLength")
+        st.number_input('(feet or meters) distance', key="distance")
         
     with col2:
-        st.title("Output:")
+        st.subheader("Output:")
         st.text_area("(feet or meters) object height:", key="objectHeight", disabled=True)
-        
+
 # control 绑定
 applyBindingController(render)

fengx_launch.ipynb

@@ -18,12 +18,12 @@
    "outputs": [],
    "source": [
     "# 任务：安装依赖\n",
-    "%pip install streamlit"
+    "%pip install -r requirements.txt"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [

fengxai/SelfSupervised.py

@@ -0,0 +1,78 @@
+import torch
+import supervision as sv
+import cv2
+import numpy as np
+from segment_anything import SamPredictor, sam_model_registry
+from diffusers import StableDiffusionInpaintPipeline
+from groundingdino.util.inference import load_model, load_image, predict, annotate
+from torchvision.ops import box_convert
+from typing import List
+
+class SelfSupervised:
+    
+    def __init__(self):
+        # -----Set Image and CUDA
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        # ------SAM Parameters
+        self.model_type = "vit_h"
+        self.predictor = SamPredictor(sam_model_registry[self.model_type](
+            checkpoint="./weights/sam_vit_h_4b8939.pth").to(device=self.device))
+        # ------Stable Diffusion
+        self.pipe = StableDiffusionInpaintPipeline.from_pretrained(
+            "stabilityai/stable-diffusion-2-inpainting", torch_dtype=torch.float16,).to(self.device)
+        # ----Grounding DINO
+        self.groundingdino_model = load_model(
+            "fengxai/config/groundingdino/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
+
+    
+    def checkAnnotate(image_source: np.ndarray, boxes: torch.Tensor, logits: torch.Tensor, phrases: List[str]):
+        # 在原始图像中添加boxes
+        h, w, _ = image_source.shape
+        boxes = boxes * torch.Tensor([w, h, w, h])
+        # 参考：https://pytorch.org/vision/main/generated/torchvision.ops.box_convert.html
+        # xyxy: x1y1 为左上角，x2y2为右下角
+        # cxcywh: 通过盒子的中心，cxcy为盒子的中心，wh为宽度和高度
+        xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy()
+        detections = sv.Detections(xyxy=xyxy)
+        boxesHeight=int(xyxy[0][3]-xyxy[0][1])
+        boxesWidth=int(xyxy[0][2]-xyxy[0][0])
+        labels = [
+            f"{phrase} {logit:.2f} w:{boxesWidth} h:{boxesHeight}"
+            for phrase, logit in zip(phrases, logits)
+        ]
+        box_annotator = sv.BoxAnnotator()
+        annotated_frame = cv2.cvtColor(image_source, cv2.COLOR_RGB2BGR)
+        annotated_frame = box_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
+        return annotated_frame, xyxy
+
+    
+    # 预测图片
+    def imagePredict(self, imgFile, item="clothing", boxThreshold=0.3, textTreshold=0.25):
+        src, img = load_image(imgFile)
+        h, w, _ = src.shape
+        boxes, logits, phrases = predict(
+            model=self.groundingdino_model,
+            image=img,
+            caption=item,
+            box_threshold=boxThreshold,
+            text_threshold=textTreshold
+        )
+        # 查看annotate相关的信息
+        imgAnnnotated, xyxy = self.checkAnnotate(
+            image_source=src, boxes=boxes, logits=logits, phrases=phrases
+        )
+        imgAnnnotated = imgAnnnotated[..., ::-1]
+        
+        boxesHeight=int(xyxy[0][3]-xyxy[0][1])
+        boxesWidth=int(xyxy[0][2]-xyxy[0][0])
+        print("boxes shape=", boxes.shape)
+        cv2.imwrite("data/annotated_image.jpg", imgAnnnotated)
+        return {
+            "imageOutput": "data/annotated_image.jpg",
+            "imageHeight": h,
+            "imageWidth": w,
+            "objectHeight": boxesHeight,
+            "objectWidth": boxesWidth
+        }
+
+selfSupervised = SelfSupervised()

fengxai/inference.py

@@ -0,0 +1,17 @@
+import os
+from fengxai.SelfSupervised import selfSupervised
+
+def imagePrediction(imageFile, funcCallback):
+    if not os.path.exists("weights/sam_vit_h_4b8939.pth") \
+        and not os.path.exists("weights/groundingdino_swint_ogc.pth"):
+        cmd = """
+            mkdir weights \
+                && cd weights \
+                && wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth \
+                && wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth \
+                && cd ..
+        """
+        os.system(cmd)
+    
+    funcCallback(selfSupervised.imagePredict(imageFile))
+    

fengxai/{responsive.py → render.py}

@@ -11,7 +11,7 @@ def getModel(key):
 
 def updateModel(key, value=""):
     setModel(key, value)
-    st.session_state[key]=str(value) if value else ""
+    st.session_state[key]=value if value else 0 if type(value) == "int" else "" 
 
 # controller
 def applyBindingController(render):

requirements.txt

@@ -1 +1,8 @@
-streamlit
+streamlit
+torch 
+torchvision
+Pillow 
+segment_anything
+diffusers
+scipy
+git+https://github.com/IDEA-Research/GroundingDINO.git