Update app.py

app.py

@@ -137,6 +137,49 @@ def infer(f_in, interpolation, fps_output):
     return final_vid, files
 
 
+def remove_bg(frame):
+    b = 5
+    #subtract background (get scene with shadow) and correct hue against light
+    bg = cv2.medianBlur(frame, 255)
+    bg_gray = cv2.cvtColor(cv2.cvtColor(bg, cv2.COLOR_BGR2GRAY), cv2.COLOR_GRAY2BGR)
+    bg_diff = (bg-bg_gray).astype(np.int16)
+    frame_ = ((bg.astype(np.int16)-frame.astype(np.int16))+127).astype(np.uint8)
+    frame = (frame.astype(np.int16)-bg_diff).astype(np.uint8)
+    frame_ = cv2.bilateralFilter(frame_, b*4, b*8, b*2)
+    
+    #remove regions of low saturation (get scene without shadow)
+    mask = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
+    m = cv2.inRange(mask, np.array([0,0,0]), np.array([180,32,256]))
+    frame[m>0] = (127,127,127)
+    frame_ = cv2.medianBlur(frame_, b)
+    element = cv2.getStructuringElement(cv2.MORPH_RECT, (2*b+1, 2*b+1), (b,b))
+    frame_ = cv2.erode(cv2.dilate(frame_, element), element)
+    frame = cv2.medianBlur(frame, b)
+    
+    
+    m_ = frame_.reshape((-1,3))
+    # convert to np.float32
+    m_ = np.float32(m_)
+    # define criteria, number of clusters(K) and apply kmeans()
+    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 16, 1.0)
+    K = 3
+    ret,label,center=cv2.kmeans(m_,K,None,criteria,16,cv2.KMEANS_PP_CENTERS)
+    # Now convert back into uint8, and make original image
+    center = np.uint8(center)
+    res = center[label.flatten()]
+    frame_ = res.reshape((frame_.shape))
+    
+    
+    #remove shadows at edges
+    cv2.rectangle(frame_,(0,0),(frame_.shape[1]-1,frame_.shape[0]-1),(142,142,142),7)
+    mask = cv2.floodFill(frame_, None, (0, 0), 255, 0, 0, (4 | cv2.FLOODFILL_FIXED_RANGE))[2] #(4 | cv2.FLOODFILL_FIXED_RANGE | cv2.FLOODFILL_MASK_ONLY | 255 << 8)
+    # 255 << 8 tells to fill with the value 255)
+    mask = mask[1:mask.shape[0]-1, 1:mask.shape[1]-1]
+    frame_[mask>0] = (125,125,125)
+    frame_[frame_[:,:,0]>=142] = (255,255,255)
+
+    return frame#, frame_
+
 def logscale(linear):
     return int(math.pow(2, linear))
 
@@ -145,14 +188,35 @@ def linscale(linear):
 
 def sharpest(fl, i):
     break_vid = get_frames(fl, "vid_input_frame", "origin", i)
+
+    frames = []
     blur_s = []
     for jdx, fr in enumerate(break_vid[0]):
-        blur_s.append(cv2.Laplacian(cv2.cvtColor(cv2.imread(fr).astype(np.uint8), cv2.COLOR_BGR2GRAY), cv2.CV_64F).var())
+        frames.append(cv2.imread(fr).astype(np.uint8))
+        blur_s.append(cv2.Laplacian(cv2.cvtColor(frames[len(frames)-1], cv2.COLOR_BGR2GRAY), cv2.CV_64F).var())
         print(str(int(blur_s[jdx])))
 
-    mx = np.argmax(blur_s)
-    fl = break_vid[0][mx]
-    print(str(i) +'th file, sharpest frame: '+str(mx)+', name: '+fl)
+    indx = np.argmax(blur_s)
+    fl = break_vid[0][indx]
+
+    n = 25
+    half = int(n/2)
+    if indx-half < 0:
+        n = indx*2+1
+    elif indx+half >= len(frames):
+        n = (len(frames)-1-indx)*2+1
+        
+    #denoise
+    frame = cv2.fastNlMeansDenoisingColoredMulti(
+        srcImgs = frames,
+        imgToDenoiseIndex = indx,
+        temporalWindowSize = n,
+        hColor = 5,
+        templateWindowSize = 21,
+        searchWindowSize = 21)
+
+    cv2.imwrite(fl, frame)
+    print(str(i) +'th file, sharpest frame: '+str(indx)+', name: '+fl)
     return fl
 
 def sortFiles(e):
@@ -167,9 +231,10 @@ def loadf(f):
         for i, fl in enumerate(f):
             ftype = fl.split('/')
             if ftype[len(ftype)-1].split('.')[1] == 'mp4':
-                fnew.append(sharpest(fl, i))
+                fl = remove_bg(sharpest(fl, i))
             else:
-                fnew.append(fl)
+                fl = remove_bg(fl)
+            fnew.append(fl)
         return fnew, fnew
     else:
         return f, f