Upload 35 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+presentation.ipynb filter=lfs diff=lfs merge=lfs -text

CyclicGAN_Inference.ipynb

@@ -0,0 +1,208 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "10ee1bf4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "from tqdm import tqdm\n",
+    "from lib.gan_networks import define_G\n",
+    "import torchvision.transforms as transforms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "59797ab5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def __transforms2pil_resize(method):\n",
+    "    mapper = {\n",
+    "        transforms.InterpolationMode.BILINEAR: Image.BILINEAR,\n",
+    "        transforms.InterpolationMode.BICUBIC: Image.BICUBIC,\n",
+    "        transforms.InterpolationMode.NEAREST: Image.NEAREST,\n",
+    "        transforms.InterpolationMode.LANCZOS: Image.LANCZOS,\n",
+    "    }\n",
+    "    return mapper[method]\n",
+    "\n",
+    "\n",
+    "def __scale_width(\n",
+    "    img, target_size, crop_size, method=transforms.InterpolationMode.BICUBIC\n",
+    "):\n",
+    "    method = __transforms2pil_resize(method)\n",
+    "    ow, oh = img.size\n",
+    "    if ow == target_size and oh >= crop_size:\n",
+    "        return img\n",
+    "    w = target_size\n",
+    "    h = int(max(target_size * oh / ow, crop_size))\n",
+    "    return img.resize((w, h), method)\n",
+    "\n",
+    "\n",
+    "def get_transform(load_size, crop_size, method=transforms.InterpolationMode.BICUBIC):\n",
+    "    transform_list = [\n",
+    "        transforms.Lambda(lambda img: __scale_width(img, load_size, crop_size, method)),\n",
+    "        transforms.ToTensor(),\n",
+    "        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
+    "    ]\n",
+    "    return transforms.Compose(transform_list)\n",
+    "\n",
+    "\n",
+    "def tensor2im(input_image, imtype=np.uint8):\n",
+    "    \"\"\" \"Converts a Tensor array into a numpy image array.\n",
+    "\n",
+    "    Parameters:\n",
+    "        input_image (tensor) --  the input image tensor array\n",
+    "        imtype (type)        --  the desired type of the converted numpy array\n",
+    "    \"\"\"\n",
+    "    if not isinstance(input_image, np.ndarray):\n",
+    "        if isinstance(input_image, torch.Tensor):  # get the data from a variable\n",
+    "            image_tensor = input_image.data\n",
+    "        else:\n",
+    "            return input_image\n",
+    "        image_numpy = (\n",
+    "            image_tensor[0].cpu().float().numpy()\n",
+    "        )  # convert it into a numpy array\n",
+    "        if image_numpy.shape[0] == 1:  # grayscale to RGB\n",
+    "            image_numpy = np.tile(image_numpy, (3, 1, 1))\n",
+    "        image_numpy = (\n",
+    "            (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0\n",
+    "        )  # post-processing: tranpose and scaling\n",
+    "    else:  # if it is a numpy array, do nothing\n",
+    "        image_numpy = input_image\n",
+    "    return image_numpy.astype(imtype)\n",
+    "\n",
+    "\n",
+    "def create_model_and_transform(pretrained: str = None):\n",
+    "    # Creating model\n",
+    "    input_nc = 3\n",
+    "    output_nc = 3\n",
+    "    ngf = 64\n",
+    "    netG = \"resnet_9blocks\"\n",
+    "    norm = \"instance\"\n",
+    "    no_dropout = True\n",
+    "    init_type = \"normal\"\n",
+    "    init_gain = 0.02\n",
+    "    gpu_ids = []\n",
+    "\n",
+    "    netG_A = define_G(\n",
+    "        input_nc,\n",
+    "        output_nc,\n",
+    "        ngf,\n",
+    "        netG,\n",
+    "        norm,\n",
+    "        not no_dropout,\n",
+    "        init_type,\n",
+    "        init_gain,\n",
+    "        gpu_ids,\n",
+    "    )\n",
+    "    if pretrained:\n",
+    "        chkpntA = torch.load(pretrained)\n",
+    "        netG_A.load_state_dict(chkpntA)\n",
+    "    netG_A.eval()\n",
+    "\n",
+    "    netG_A = netG_A.cuda()\n",
+    "\n",
+    "    # Creating transform\n",
+    "    load_size = 1280\n",
+    "    crop_size = 224\n",
+    "    image_transforms = get_transform(load_size=load_size, crop_size=crop_size)\n",
+    "    return netG_A, image_transforms\n",
+    "\n",
+    "\n",
+    "def run_inference(img_path, model, transform):\n",
+    "    image = Image.open(img_path)\n",
+    "    inputs = image_transforms(image).unsqueeze(0).to(\"cuda\")\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        out = model(inputs)\n",
+    "    out = tensor2im(out)\n",
+    "    return Image.fromarray(out)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6fc20d26",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "initialize network with normal\n"
+     ]
+    },
+    {
+     "ename": "RuntimeError",
+     "evalue": "Error(s) in loading state_dict for UnetGenerator:\n\tMissing key(s) in state_dict: \"model.model.0.weight\", \"model.model.0.bias\", \"model.model.1.model.1.weight\", \"model.model.1.model.1.bias\", \"model.model.1.model.3.model.1.weight\", \"model.model.1.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.3.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.3.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.5.weight\", \"model.model.1.model.3.model.5.bias\", \"model.model.1.model.5.weight\", \"model.model.1.model.5.bias\", \"model.model.3.weight\", \"model.model.3.bias\". \n\tUnexpected key(s) in state_dict: \"model.1.weight\", \"model.1.bias\", \"model.4.weight\", \"model.4.bias\", \"model.7.weight\", \"model.7.bias\", \"model.10.conv_block.1.weight\", \"model.10.conv_block.1.bias\", \"model.10.conv_block.5.weight\", \"model.10.conv_block.5.bias\", \"model.11.conv_block.1.weight\", \"model.11.conv_block.1.bias\", \"model.11.conv_block.5.weight\", \"model.11.conv_block.5.bias\", \"model.12.conv_block.1.weight\", \"model.12.conv_block.1.bias\", \"model.12.conv_block.5.weight\", \"model.12.conv_block.5.bias\", \"model.13.conv_block.1.weight\", \"model.13.conv_block.1.bias\", \"model.13.conv_block.5.weight\", \"model.13.conv_block.5.bias\", \"model.14.conv_block.1.weight\", \"model.14.conv_block.1.bias\", \"model.14.conv_block.5.weight\", \"model.14.conv_block.5.bias\", \"model.15.conv_block.1.weight\", \"model.15.conv_block.1.bias\", \"model.15.conv_block.5.weight\", \"model.15.conv_block.5.bias\", \"model.16.conv_block.1.weight\", \"model.16.conv_block.1.bias\", \"model.16.conv_block.5.weight\", \"model.16.conv_block.5.bias\", \"model.17.conv_block.1.weight\", \"model.17.conv_block.1.bias\", \"model.17.conv_block.5.weight\", \"model.17.conv_block.5.bias\", \"model.18.conv_block.1.weight\", \"model.18.conv_block.1.bias\", \"model.18.conv_block.5.weight\", \"model.18.conv_block.5.bias\", \"model.19.weight\", \"model.19.bias\", \"model.22.weight\", \"model.22.bias\", \"model.26.weight\", \"model.26.bias\". ",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gan, image_transforms \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_model_and_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./checkpoints/clear2snowy.pth\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[12], line 82\u001b[0m, in \u001b[0;36mcreate_model_and_transform\u001b[0;34m(pretrained)\u001b[0m\n\u001b[1;32m     80\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pretrained:\n\u001b[1;32m     81\u001b[0m     chkpntA \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mload(pretrained)\n\u001b[0;32m---> 82\u001b[0m     \u001b[43mnetG_A\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_state_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchkpntA\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     83\u001b[0m netG_A\u001b[38;5;241m.\u001b[39meval()\n\u001b[1;32m     85\u001b[0m netG_A \u001b[38;5;241m=\u001b[39m netG_A\u001b[38;5;241m.\u001b[39mcuda()\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:2189\u001b[0m, in \u001b[0;36mModule.load_state_dict\u001b[0;34m(self, state_dict, strict, assign)\u001b[0m\n\u001b[1;32m   2184\u001b[0m         error_msgs\u001b[38;5;241m.\u001b[39minsert(\n\u001b[1;32m   2185\u001b[0m             \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMissing key(s) in state_dict: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m   2186\u001b[0m                 \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m missing_keys)))\n\u001b[1;32m   2188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_msgs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 2189\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mError(s) in loading state_dict for \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m   2190\u001b[0m                        \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(error_msgs)))\n\u001b[1;32m   2191\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _IncompatibleKeys(missing_keys, unexpected_keys)\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for UnetGenerator:\n\tMissing key(s) in state_dict: \"model.model.0.weight\", \"model.model.0.bias\", \"model.model.1.model.1.weight\", \"model.model.1.model.1.bias\", \"model.model.1.model.3.model.1.weight\", \"model.model.1.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.1.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.1.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.3.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.3.model.3.bias\", \"model.model.1.model.3.model.3.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.3.model.5.weight\", \"model.model.1.model.3.model.3.model.5.bias\", \"model.model.1.model.3.model.5.weight\", \"model.model.1.model.3.model.5.bias\", \"model.model.1.model.5.weight\", \"model.model.1.model.5.bias\", \"model.model.3.weight\", \"model.model.3.bias\". \n\tUnexpected key(s) in state_dict: \"model.1.weight\", \"model.1.bias\", \"model.4.weight\", \"model.4.bias\", \"model.7.weight\", \"model.7.bias\", \"model.10.conv_block.1.weight\", \"model.10.conv_block.1.bias\", \"model.10.conv_block.5.weight\", \"model.10.conv_block.5.bias\", \"model.11.conv_block.1.weight\", \"model.11.conv_block.1.bias\", \"model.11.conv_block.5.weight\", \"model.11.conv_block.5.bias\", \"model.12.conv_block.1.weight\", \"model.12.conv_block.1.bias\", \"model.12.conv_block.5.weight\", \"model.12.conv_block.5.bias\", \"model.13.conv_block.1.weight\", \"model.13.conv_block.1.bias\", \"model.13.conv_block.5.weight\", \"model.13.conv_block.5.bias\", \"model.14.conv_block.1.weight\", \"model.14.conv_block.1.bias\", \"model.14.conv_block.5.weight\", \"model.14.conv_block.5.bias\", \"model.15.conv_block.1.weight\", \"model.15.conv_block.1.bias\", \"model.15.conv_block.5.weight\", \"model.15.conv_block.5.bias\", \"model.16.conv_block.1.weight\", \"model.16.conv_block.1.bias\", \"model.16.conv_block.5.weight\", \"model.16.conv_block.5.bias\", \"model.17.conv_block.1.weight\", \"model.17.conv_block.1.bias\", \"model.17.conv_block.5.weight\", \"model.17.conv_block.5.bias\", \"model.18.conv_block.1.weight\", \"model.18.conv_block.1.bias\", \"model.18.conv_block.5.weight\", \"model.18.conv_block.5.bias\", \"model.19.weight\", \"model.19.bias\", \"model.22.weight\", \"model.22.bias\", \"model.26.weight\", \"model.26.bias\". "
+     ]
+    }
+   ],
+   "source": [
+    "gan, image_transforms = create_model_and_transform(\"./checkpoints/clear2snowy.pth\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d44ebf97",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 100/100 [00:39<00:00,  2.51it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "image_path = os.listdir(\"./data/images\")\n",
+    "save_folder = \"./data/gan/snow_images\"\n",
+    "\n",
+    "for img in tqdm(image_path):\n",
+    "    trg = os.path.join(\"./data/images\", img)\n",
+    "    src = os.path.join(f\"./data/gan/snow_images/\", img.split(\".\")[0] + \".jpg\")\n",
+    "    if not (os.path.exists(src)):\n",
+    "        out = run_inference(img_path=trg, model=gan, transform=image_transforms)\n",
+    "        out.save(src)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Fog_Effect_Generator.py

@@ -0,0 +1,113 @@
+import os
+import random
+import argparse
+import numpy as np
+from PIL import Image
+from pathlib import Path
+from skimage import color
+from tqdm.auto import tqdm
+
+from lib.lime import LIME
+from lib.fog_gen import fogAttenuation
+
+from lib.gen_utils import (
+                           illumination2opacity, 
+                           reduce_lightHSV, 
+                           scale_depth)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--clear_path", type=str, required=True, help="path to the file or the folder")
+    parser.add_argument("--depth_path", type=str, required=True, help="path to the file or the folder")
+    parser.add_argument("--save_folder", type=str, default="./generated/", help="path to the folder")
+    parser.add_argument("--txt_file", default=None, help="path to the folder")
+    parser.add_argument("--show", action="store_true")
+    return parser.parse_args()
+
+
+
+class FogEffectGenerator:
+    def __init__(self):
+        self._lime = LIME(iterations=25, alpha=1.0)
+        # self._illumination2darkness = {0: 1, 1: 0.75, 2: 0.65, 3:0.5}
+        self._illumination2darkness = {0: 1, 1: 0.9, 2: 0.8, 3: 0.7}
+        self._weather2visibility = (500, 2000)
+        # self._illumination2fogcolor = {0: (80, 120), 1: (120, 160), 2: (160, 200), 3: (200, 240)}
+        self._illumination2fogcolor = {0: (150, 180), 1: (180, 200), 2: (200, 240), 3: (200, 240)}
+    
+    def getIlluminationMap(self, img: np.ndarray) -> np.ndarray: 
+        self._lime.load(img)
+        T = self._lime.illumMap()
+        return T
+
+    def getIlluminationMapCheat(self, img: np.ndarray) -> np.ndarray: 
+        T = color.rgb2gray(img)
+        return T
+    
+    def genEffect(self, img_path: str, depth_path: str):
+        I = np.array(Image.open(img_path))
+        D = np.load(depth_path)
+
+        hI, wI, _ = I.shape
+        hD, wD = D.shape
+        
+        if hI!=hD or wI!=wD:
+            D = scale_depth(D, hI, wI)
+        
+        # T = self.getIlluminationMap(I)
+        T = self.getIlluminationMapCheat(I)
+        illumination_array = np.histogram(T, bins=4, range=(0,1))[0]/(T.size)
+        illumination = illumination_array.argmax()
+        
+        if illumination>0:
+            vmax = self._weather2visibility[1] if self._weather2visibility[1]<=D.max() else D.max()
+            if vmax<= self._weather2visibility[0]:
+                visibility = self._weather2visibility[0]
+            else:
+                visibility = random.randint(self._weather2visibility[0], int(vmax))
+            fog_color = random.randint(self._illumination2fogcolor[illumination][0], self._illumination2fogcolor[illumination][1])
+            I_dark = reduce_lightHSV(I, sat_red=self._illumination2darkness[illumination], val_red=self._illumination2darkness[illumination])
+            I_fog = fogAttenuation(I_dark, D, visibility=visibility, fog_color=fog_color)
+        else:
+            fog_color = 75
+            visibility = 150 #D.max()*0.75
+            I_fog = fogAttenuation(I, D, visibility=visibility, fog_color=fog_color)
+            
+        return I_fog
+
+def main():
+    args = parse_arguments()
+    foggen = FogEffectGenerator()
+    
+    clearP = Path(args.clear_path)
+    depthP = Path(args.depth_path)
+    if clearP.is_file() and (depthP.is_file() and depthP.suffix==".npy"):
+        snowy = foggen.genEffect(clearP, depthP)
+        if args.show:
+            Image.fromarray(snowy).show()
+
+    if clearP.is_dir() and depthP.is_dir():
+        if args.txt_file:
+            with open(args.txt_file, 'r') as f:
+                files = f.read().split('\n')
+            image_files = [clearP / f for f in files]
+        else:
+            image_files = sorted(Path(clearP).glob("*"))
+
+        depth_files = [Path(depthP) / ("-".join(imgf.name.split('-')[:2])+".npy") for imgf in image_files]
+        
+        valid_files = [idx for idx, f in enumerate(depth_files) if f.exists()]
+        image_files = [image_files[idx] for idx in valid_files] 
+        depth_files = [depth_files[idx] for idx in valid_files]
+        
+        save_folder = Path(args.save_folder)
+        if not save_folder.exists():
+            os.makedirs(str(save_folder))
+        
+        for imgp, depthp in tqdm(zip(image_files, depth_files), total=len(image_files)):
+            foggy = foggen.genEffect(imgp, depthp)
+            Image.fromarray(foggy).save(save_folder / (imgp.stem+"-fsyn.jpg"))
+
+if __name__=='__main__':
+    main()

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MiDaS_Depth_Estimation.py

@@ -0,0 +1,110 @@
+import os
+import torch
+import argparse
+import numpy as np
+import cv2
+from pathlib import Path
+from tqdm.auto import tqdm
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--img_path", type=str, required=True, help="path to the file or the folder"
+    )
+    parser.add_argument(
+        "--save_folder", type=str, default="./depth/", help="path to the folder"
+    )
+    parser.add_argument(
+        "--midas_model", type=str, default="DPT_Large", help="Midas model name"
+    )
+    parser.add_argument("--use_cuda", action="store_true")
+    parser.add_argument("--baseline", type=float, default=0.54)
+    parser.add_argument("--focal", type=float, default=721.09)
+    parser.add_argument("--img_scale", type=float, default=1)
+    return parser.parse_args()
+
+
+def get_depth_estimation_model(model_name: str, device="cpu"):
+    assert model_name in ["DPT_Large", "DPT_Hybrid", "MiDaS_small"]
+
+    midas = torch.hub.load("intel-isl/MiDaS", model_name)
+    midas.eval()
+    midas.to(device)
+
+    midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms")
+    if model_name in ["DPT_Large", "DPT_Hybrid"]:
+        transform = midas_transforms.dpt_transform
+    else:
+        transform = midas_transforms.small_transform
+    return midas, transform
+
+
+def getDisparityMap(model, transform, img_path):
+    img = cv2.imread(str(img_path))
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    input_batch = transform(img)
+
+    with torch.no_grad():
+        prediction = model(input_batch.cuda())
+
+        prediction = torch.nn.functional.interpolate(
+            prediction.unsqueeze(1),
+            size=img.shape[:2],
+            mode="bicubic",
+            align_corners=False,
+        ).squeeze()
+    return prediction.cpu().numpy()
+
+
+def get_depth_map(
+    midas, midas_transform, imgp, baseline=0.54, focal=721.09, img_scale=1
+):
+    disp = getDisparityMap(midas, midas_transform, imgp)
+    disp[disp < 0] = 0
+    disp = disp + 1e-3
+    depth = baseline * focal / (disp * img_scale)
+
+    return depth
+
+
+def get_depth_map_new(midas, midas_transform, imgp):
+    depth = getDisparityMap(midas, midas_transform, imgp)
+    depth[depth < 0] = 0
+    depth = depth + 1e-3
+    depth = depth
+    return depth.max() - depth
+
+
+def main():
+    args = parse_arguments()
+
+    device = torch.device("cpu")
+    if args.use_cuda:
+        device = torch.device("cuda")
+
+    ### kitti
+    baseline = args.baseline
+    focal = args.focal
+    img_scale = args.img_scale
+
+    imgP = Path(args.img_path)
+    save_folder = Path(args.save_folder)
+    if not save_folder.exists():
+        os.makedirs(str(save_folder))
+
+    midas, midas_transform = get_depth_estimation_model(
+        model_name=args.midas_model, device=device
+    )
+
+    if imgP.is_dir():
+        image_files = sorted(Path(imgP).glob("*"))
+        for imgp in tqdm(image_files):
+            depth = get_depth_map(
+                midas, midas_transform, imgp, baseline, focal, img_scale
+            )
+            np.save(save_folder / imgp.stem, depth)
+
+
+if __name__ == "__main__":
+    main()

Neural_Style_Transfer.py

@@ -0,0 +1,121 @@
+import os
+import copy
+import torch
+import random
+import argparse
+import numpy as np
+from PIL import Image
+from pathlib import Path
+from tqdm.auto import tqdm
+from lib.style_transfer_utils import (
+    tensor2pil,
+    load_style_transfer_model,
+    run_style_transfer,
+    style_content_image_loader,
+)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--content-imgs", type=str, help="Path to the content images.", required=True
+    )
+    parser.add_argument(
+        "--style-imgs", type=str, help="Path to the style images.", required=True
+    )
+    parser.add_argument(
+        "--save-folder",
+        type=str,
+        help="Path to the save the generated images.",
+        required=True,
+    )
+    parser.add_argument(
+        "--vgg", type=str, help="Path to the pretrained VGG model.", required=True
+    )
+
+    parser.add_argument("--cuda", action="store_true", help="use cuda.")
+    parser.add_argument(
+        "--ext", type=str, default="stl", help="extension for generated image."
+    )
+    parser.add_argument(
+        "--min-step", type=int, default=100, help="minimum iteration steps"
+    )
+    parser.add_argument(
+        "--max-step", type=int, default=200, help="maximum iteration steps"
+    )
+    parser.add_argument(
+        "--style-weight", type=float, default=100000, help="weight for style loss"
+    )
+    parser.add_argument(
+        "--content-weight", type=float, default=2, help="weight for content loss"
+    )
+
+    return parser.parse_args()
+
+
+def transfer_style(
+    cnn_path,
+    cimg,
+    simg,
+    min_step=100,
+    max_step=200,
+    style_weight=100000,
+    content_weight=2,
+    device="cpu",
+):
+    cnn = load_style_transfer_model(pretrained=cnn_path)
+
+    content_img, style_img = style_content_image_loader(cimg, simg)
+    input_img = copy.deepcopy(content_img).to(device, torch.float)
+
+    output = run_style_transfer(
+        cnn,
+        content_img,
+        style_img,
+        input_img,
+        num_steps=random.randint(min_step, max_step),
+        style_weight=style_weight,
+        content_weight=content_weight,
+        device=device,
+    )
+    return tensor2pil(output[0].detach().cpu())
+
+
+def main():
+    args = parse_arguments()
+    if args.cuda and torch.cuda.is_available():
+        device = torch.device("cuda:0")
+    else:
+        device = torch.device("cpu")
+
+    content_images = sorted(Path(args.content_imgs).glob("*"))
+    # with open(Path(args.content_imgs), "r") as f:
+    #     lines = f.read()
+    # content_images = lines.split("\n")
+    # content_images = [Path("./content_images") / f for f in content_images]
+    style_images = sorted(Path(args.style_imgs).glob("*"))
+
+    save_folder = Path(args.save_folder)
+    if not os.path.exists(args.save_folder):
+        print(f"Creating {args.save_folder}")
+        os.makedirs(str(save_folder))
+
+    for i, cimg in enumerate(content_images):
+        name, extension = cimg.name.split(".")
+        simg = style_images[i % len(style_images)]
+
+        output_img = transfer_style(
+            cnn_path=args.vgg,
+            cimg=cimg,
+            simg=simg,
+            min_step=args.min_step,
+            max_step=args.max_step,
+            style_weight=args.style_weight,
+            content_weight=args.content_weight,
+            device=device,
+        )
+        output_img.save(save_folder / f"{name}.{extension}")
+
+
+if __name__ == "__main__":
+    main()

Rain_Effect_Generator.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+import os
+import random
+import argparse
+import numpy as np
+from PIL import Image
+from pathlib import Path
+from skimage import color
+from tqdm.auto import tqdm
+
+from lib.lime import LIME
+from lib.fog_gen import fogAttenuation
+from lib.rain_gen import RainGenUsingNoise
+from lib.gen_utils import (
+    illumination2opacity,
+    layer_blend,
+    alpha_blend,
+    reduce_lightHSV,
+    scale_depth,
+)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--clear_path", type=str, required=True, help="path to the file or the folder"
+    )
+    parser.add_argument(
+        "--depth_path", type=str, required=True, help="path to the file or the folder"
+    )
+    parser.add_argument(
+        "--save_folder", type=str, default="./generated/", help="path to the folder"
+    )
+    parser.add_argument("--txt_file", default=None, help="path to the folder")
+    parser.add_argument("--show", action="store_true")
+    parser.add_argument("--fog", action="store_true")
+    return parser.parse_args()
+
+
+class RainEffectGenerator:
+    def __init__(self, fog=True):
+        self._lime = LIME(iterations=25, alpha=1.0)
+        # self._illumination2darkness = {0: 1, 1: 0.75, 2: 0.65, 3: 0.5}
+        self._illumination2darkness = {0: 1, 1: 0.95, 2: 0.85, 3: 0.8}
+        self._weather2visibility = (1000, 2000)
+        # self._weather2visibility = {'fog': (100,250), 'rain': (1000,2000), 'snow': (500, 1000)}
+        # self._illumination2fogcolor = {0: (80, 120), 1: (120, 160), 2: (160, 200), 3: (200, 240)}
+        self._illumination2fogcolor = {
+            0: (150, 180),
+            1: (180, 200),
+            2: (200, 240),
+            3: (200, 240),
+        }
+        self._rain_layer_gen = RainGenUsingNoise()
+        self._fog = fog
+
+    def getIlluminationMap(self, img: np.ndarray) -> np.ndarray:
+        self._lime.load(img)
+        T = self._lime.illumMap()
+        return T
+
+    def getIlluminationMapCheat(self, img: np.ndarray) -> np.ndarray:
+        T = color.rgb2gray(img)
+        return T
+
+    def genRainLayer(self, h=720, w=1280):
+        blur_angle = random.choice([-1, 1]) * random.randint(60, 90)
+        layer_large = self._rain_layer_gen.genRainLayer(
+            h=720,
+            w=1280,
+            noise_scale=random.uniform(0.35, 0.55),
+            noise_amount=0.2,
+            zoom_layer=random.uniform(1.0, 3.5),
+            blur_kernel_size=random.choice([15, 17, 19, 21, 23]),
+            blur_angle=blur_angle,
+        )  # large
+
+        layer_small = self._rain_layer_gen.genRainLayer(
+            h=720,
+            w=1280,
+            noise_scale=random.uniform(0.35, 0.55),
+            noise_amount=0.15,
+            zoom_layer=random.uniform(1.0, 3.5),
+            blur_kernel_size=random.choice([7, 9, 11, 13]),
+            blur_angle=blur_angle,
+        )  # small
+        layer = layer_blend(layer_small, layer_large)
+        hl, wl = layer.shape
+
+        if h != hl or w != wl:
+            layer = np.asarray(Image.fromarray(layer).resize((w, h)))
+        return layer
+
+    def genEffect(self, img_path: str, depth_path: str):
+        I = np.array(Image.open(img_path))
+        D = np.load(depth_path)
+
+        return self.genEffect_(I, D)
+
+    def genEffect_(self, I, D):
+        hI, wI, _ = I.shape
+        hD, wD = D.shape
+
+        if hI != hD or wI != wD:
+            D = scale_depth(D, hI, wI)
+
+        T = self.getIlluminationMap(I)
+        illumination_array = np.histogram(T, bins=4, range=(0, 1))[0] / (T.size)
+        illumination = illumination_array.argmax()
+
+        if self._fog:
+            if illumination > 0:
+                visibility = visibility = random.randint(
+                    self._weather2visibility[0], self._weather2visibility[1]
+                )
+                fog_color = random.randint(
+                    self._illumination2fogcolor[illumination][0],
+                    self._illumination2fogcolor[illumination][1],
+                )
+                I_dark = reduce_lightHSV(
+                    I,
+                    sat_red=self._illumination2darkness[illumination],
+                    val_red=self._illumination2darkness[illumination],
+                )
+                I_fog = fogAttenuation(
+                    I_dark, D, visibility=visibility, fog_color=fog_color
+                )
+            else:
+                fog_color = 75
+                visibility = D.max() * 0.75 if D.max() < 1000 else 750
+                I_fog = fogAttenuation(I, D, visibility=visibility, fog_color=fog_color)
+        else:
+            I_fog = I
+
+        alpha = illumination2opacity(I, illumination) * random.uniform(0.3, 0.5)
+        rain_layer = self.genRainLayer(h=hI, w=wI)
+        I_rain = alpha_blend(I_fog, rain_layer, alpha)
+        return I_rain.astype(np.uint8)
+
+
+def main():
+    args = parse_arguments()
+    raingen = RainEffectGenerator(fog=args.fog)
+
+    clearP = Path(args.clear_path)
+    depthP = Path(args.depth_path)
+    if clearP.is_file() and (depthP.is_file() and depthP.suffix == ".npy"):
+        rainy = raingen.genEffect(clearP, depthP)
+        if args.show:
+            Image.fromarray(rainy).show()
+
+    if clearP.is_dir() and depthP.is_dir():
+        if args.txt_file:
+            with open(args.txt_file, "r") as f:
+                files = f.read().split("\n")
+            image_files = [clearP / f for f in files]
+        else:
+            image_files = sorted(Path(clearP).glob("*"))
+        depth_files = [
+            Path(depthP) / (imgf.name.split(".")[0] + ".npy") for imgf in image_files
+        ]
+
+        valid_files = [idx for idx, f in enumerate(depth_files) if f.exists()]
+        image_files = [image_files[idx] for idx in valid_files]
+        depth_files = [depth_files[idx] for idx in valid_files]
+
+        save_folder = Path(args.save_folder)
+        if not save_folder.exists():
+            os.makedirs(str(save_folder))
+
+        for imgp, depthp in tqdm(zip(image_files, depth_files), total=len(image_files)):
+            rainy = raingen.genEffect(imgp, depthp)
+            Image.fromarray(rainy).save(save_folder / (imgp.stem + ".jpg"))
+
+
+if __name__ == "__main__":
+    main()

Snow_Effect_Generator.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+import os
+import random
+import argparse
+import numpy as np
+from PIL import Image
+from pathlib import Path
+from skimage import color
+from tqdm.auto import tqdm
+
+
+from lib.lime import LIME
+from lib.fog_gen import fogAttenuation
+from lib.snow_gen import SnowGenUsingNoise
+from lib.gen_utils import (
+    screen_blend,
+    layer_blend,
+    illumination2opacity,
+    reduce_lightHSV,
+    scale_depth,
+)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--clear_path", type=str, required=True, help="path to the file or the folder"
+    )
+    parser.add_argument(
+        "--depth_path", type=str, required=True, help="path to the file or the folder"
+    )
+    parser.add_argument(
+        "--save_folder", type=str, default="./generated/", help="path to the folder"
+    )
+    parser.add_argument("--txt_file", default=None, help="path to the folder")
+    parser.add_argument("--show", action="store_true")
+    parser.add_argument("--fog", action="store_true")
+
+    return parser.parse_args()
+
+
+class SnowEffectGenerator:
+    def __init__(self, fog=True):
+        self._lime = LIME(iterations=25, alpha=1.0)
+        # self._illumination2darkness = {0: 1, 1: 0.75, 2: 0.65, 3: 0.5}
+        self._illumination2darkness = {0: 1, 1: 0.9, 2: 0.8, 3: 0.7}
+        self._weather2visibility = (1000, 2500)  # (500, 1000)
+        # self._illumination2fogcolor = {0: (80, 120), 1: (120, 160), 2: (160, 200), 3: (200, 240)}
+        self._illumination2fogcolor = {
+            0: (150, 180),
+            1: (180, 200),
+            2: (200, 240),
+            3: (200, 240),
+        }
+        self._snow_layer_gen = SnowGenUsingNoise()
+        self._fog = fog
+
+    def getIlluminationMap(self, img: np.ndarray) -> np.ndarray:
+        self._lime.load(img)
+        T = self._lime.illumMap()
+        return T
+
+    def getIlluminationMapCheat(self, img: np.ndarray) -> np.ndarray:
+        T = color.rgb2gray(img)
+        return T
+
+    def genSnowLayer(self, h=720, w=1280):  # alpha,
+        num_itr_small = 2  # random.randint(1,3)
+        num_itr_large = 1  # random.randint(1,4)
+        blur_angle = random.choice([-1, 1]) * random.randint(60, 90)
+        layer_small = self._snow_layer_gen.genSnowMultiLayer(
+            h=720,
+            w=1280,
+            blur_angle=blur_angle,
+            intensity="small",
+            num_itr=num_itr_small,
+        )  # small
+
+        layer_large = self._snow_layer_gen.genSnowMultiLayer(
+            h=720,
+            w=1280,
+            blur_angle=blur_angle,
+            intensity="large",
+            num_itr=num_itr_large,
+        )  # large
+        layer = layer_blend(layer_small, layer_large)
+        hl, wl = layer.shape
+
+        if h != hl or w != wl:
+            layer = np.asarray(Image.fromarray(layer).resize((w, h)))
+        return layer  # (layer.astype(float)*alpha).astype(np.uint8)
+
+    def genEffect(self, img_path: str, depth_path: str):
+        I = np.array(Image.open(img_path))
+        D = np.load(depth_path)
+
+        return self.genEffect_(I, D)
+
+    def genEffect_(self, I, D):
+        hI, wI, _ = I.shape
+        hD, wD = D.shape
+
+        if hI != hD or wI != wD:
+            D = scale_depth(D, hI, wI)
+
+        T = self.getIlluminationMapCheat(I)
+        illumination_array = np.histogram(T, bins=4, range=(0, 1))[0] / (T.size)
+        illumination = illumination_array.argmax()
+
+        if self._fog:
+            if illumination > 0:
+                visibility = random.randint(
+                    self._weather2visibility[0], self._weather2visibility[1]
+                )
+                fog_color = random.randint(
+                    self._illumination2fogcolor[illumination][0],
+                    self._illumination2fogcolor[illumination][1],
+                )
+                I_dark = reduce_lightHSV(
+                    I,
+                    sat_red=self._illumination2darkness[illumination],
+                    val_red=self._illumination2darkness[illumination],
+                )
+                I_fog = fogAttenuation(
+                    I_dark, D, visibility=visibility, fog_color=fog_color
+                )
+            else:
+                fog_color = 75
+                visibility = D.max() * 0.75 if D.max() < 1000 else 750
+                I_fog = fogAttenuation(I, D, visibility=visibility, fog_color=fog_color)
+        else:
+            I_fog = I
+
+        snow_layer = self.genSnowLayer(h=hI, w=wI)  # , alpha=alpha) #, alpha
+        I_snow = screen_blend(
+            I_fog, snow_layer
+        )  # screen_blend(I_fog, snow_layer) , alpha
+        return I_snow.astype(np.uint8)
+
+
+def main():
+    args = parse_arguments()
+    snowgen = SnowEffectGenerator(fog=args.fog)
+
+    clearP = Path(args.clear_path)
+    depthP = Path(args.depth_path)
+    if clearP.is_file() and (depthP.is_file() and depthP.suffix == ".npy"):
+        snowy = snowgen.genEffect(clearP, depthP)
+        if args.show:
+            Image.fromarray(snowy).show()
+
+    if clearP.is_dir() and depthP.is_dir():
+        if args.txt_file:
+            with open(args.txt_file, "r") as f:
+                files = f.read().split("\n")
+            image_files = [clearP / f for f in files]
+        else:
+            image_files = sorted(Path(clearP).glob("*"))
+
+        depth_files = [
+            Path(depthP) / (imgf.name.split(".")[0] + ".npy") for imgf in image_files
+        ]
+
+        valid_files = [idx for idx, f in enumerate(depth_files) if f.exists()]
+        image_files = [image_files[idx] for idx in valid_files]
+        depth_files = [depth_files[idx] for idx in valid_files]
+
+        save_folder = Path(args.save_folder)
+        if not save_folder.exists():
+            os.makedirs(str(save_folder))
+
+        for imgp, depthp in tqdm(zip(image_files, depth_files), total=len(image_files)):
+            snowy = snowgen.genEffect(imgp, depthp)
+            Image.fromarray(snowy).save(save_folder / (imgp.stem + ".jpg"))
+
+
+if __name__ == "__main__":
+    main()

checkpoints/clear2rainy.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e9b90303e5876bb391be11a16b5fae9d3a58f83d5ff0e299a2d2a8ea3f175e
+size 45531485

checkpoints/clear2snowy.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90fd7de992455241c5f7ce7fa0bb92a01bb53b6fb422c4558a7a0231bfdd7104
+size 45531485

gen_depth_map.sh

@@ -0,0 +1,5 @@
+python MiDaS_Depth_Estimation.py \
+    --img_path ./data/images \
+    --save_folder ./data/depth_maps \
+    --use_cuda
+

gen_rain_image.sh

@@ -0,0 +1,5 @@
+python Rain_Effect_Generator.py \
+    --clear_path ./data/images \
+    --depth_path ./data/depth_maps \
+    --save_folder ./data/analyticity/rain_images \
+    --fog

gen_rain_image_ag.sh

@@ -0,0 +1,4 @@
+python Rain_Effect_Generator.py \
+    --clear_path ./data/gan/rain_images \
+    --depth_path ./data/depth_maps \
+    --save_folder ./data/analytical_gan/rain_images

gen_rain_nst.sh

@@ -0,0 +1,9 @@
+python Neural_Style_Transfer.py \
+    --content-imgs ./data/images/ \
+    --style-imgs ./data/styles/rain_images \
+    --save-folder ./data/nst/rain_images \
+    --vgg ./checkpoints/rain_vgg_512 \
+    --min-step 10 \
+    --max-step 10 \
+    --style-weight 10 \
+    --cuda

gen_snow_image.sh

@@ -0,0 +1,5 @@
+python  Snow_Effect_Generator.py \
+    --clear_path ./data/images \
+    --depth_path ./data/depth_maps \
+    --save_folder ./data/analyticity/snow_images \
+    --fog

gen_snow_image_ag.sh

@@ -0,0 +1,4 @@
+python  Snow_Effect_Generator.py \
+    --clear_path ./data/gan/snow_images \
+    --depth_path ./data/depth_maps \
+    --save_folder ./data/analytical_gan/snow_images

gen_snow_nst.sh

@@ -0,0 +1,9 @@
+python Neural_Style_Transfer.py \
+    --content-imgs ./data/images/ \
+    --style-imgs ./data/styles/ \
+    --save-folder ./data/nst/snow_images \
+    --vgg ./checkpoints/snow_vgg_512 \
+    --min-step 100 \
+    --max-step 200 \
+    --style-weight 10 \
+    --cuda

lib/fog_gen.py

@@ -0,0 +1,143 @@
+import numpy as np
+from PIL import Image
+from noise import pnoise3
+
+
+def perlin_noise(w, h, depth):
+    p1 = Image.new("L", (w, h))
+    p2 = Image.new("L", (w, h))
+    p3 = Image.new("L", (w, h))
+
+    scale = 1 / 130.0
+    for y in range(h):
+        for x in range(w):
+            v = pnoise3(
+                x * scale,
+                y * scale,
+                depth[y, x] * scale,
+                octaves=1,
+                persistence=0.5,
+                lacunarity=2.0,
+            )
+            color = int((v + 1) * 128.0)
+            p1.putpixel((x, y), color)
+
+    scale = 1 / 60.0
+    for y in range(h):
+        for x in range(w):
+            v = pnoise3(
+                x * scale,
+                y * scale,
+                depth[y, x] * scale,
+                octaves=1,
+                persistence=0.5,
+                lacunarity=2.0,
+            )
+            color = int((v + 0.5) * 128)
+            p2.putpixel((x, y), color)
+
+    scale = 1 / 10.0
+    for y in range(h):
+        for x in range(w):
+            v = pnoise3(
+                x * scale,
+                y * scale,
+                depth[y, x] * scale,
+                octaves=1,
+                persistence=0.5,
+                lacunarity=2.0,
+            )
+            color = int((v + 1.2) * 128)
+            p3.putpixel((x, y), color)
+
+    perlin = (np.array(p1) + np.array(p2) / 2 + np.array(p3) / 4) / 3
+
+    return perlin
+
+
+def generate_fog(image, depth, visibility=None, fog_color=None):
+    """
+    input:
+        image - numpy array (h, w, c)
+        depth - numpy array (h, w)
+    """
+
+    height, width = depth.shape
+    perlin = perlin_noise(width, height, depth)
+
+    depth_max = depth.max()
+
+    if visibility:
+        fog_visibility = visibility
+    else:
+        fog_visibility = float(
+            np.random.randint(
+                int(depth_max - 0.2 * depth_max), int(depth_max + 0.2 * depth_max)
+            )
+        )
+        fog_visibility = np.clip(fog_visibility, 60, 200)
+
+    VERTICLE_FOV = 60  # degrees
+    CAMERA_ALTITUDE = 1.8  # meters
+    VISIBILITY_RANGE_MOLECULE = 12  # m    12
+    VISIBILITY_RANGE_AEROSOL = fog_visibility  # m     450
+    ECM_ = 3.912 / VISIBILITY_RANGE_MOLECULE  # EXTINCTION_COEFFICIENT_MOLECULE /m
+    ECA_ = 3.912 / VISIBILITY_RANGE_AEROSOL  # EXTINCTION_COEFFICIENT_AEROSOL /m
+
+    FT = 70  # FOG_TOP m  31  70
+    HT = 34  # HAZE_TOP m  300    34
+
+    angle = np.repeat(
+        -1
+        * np.linspace(-0.5 * VERTICLE_FOV, 0.5 * VERTICLE_FOV, height).reshape(-1, 1),
+        axis=1,
+        repeats=width,
+    )
+    distance = depth / np.cos(np.radians(angle))
+    elevation = CAMERA_ALTITUDE + distance * np.sin(np.radians(angle))
+
+    distance_through_fog = np.zeros_like(distance)
+    distance_through_haze = np.zeros_like(distance)
+    distance_through_haze_free = np.zeros_like(distance)
+
+    ECA = ECA_
+    c = 1 - elevation / (FT + 0.00001)
+    c[c < 0] = 0
+    ECM = (ECM_ * c + (1 - c) * ECA_) * (perlin / 255)
+
+    idx1 = np.logical_and(FT > elevation, elevation > HT)
+    idx2 = elevation <= HT
+    idx3 = elevation >= FT
+
+    distance_through_haze[idx2] = distance[idx2]
+    distance_through_fog[idx1] = (
+        (elevation[idx1] - HT) * distance[idx1] / (elevation[idx1] - CAMERA_ALTITUDE)
+    )
+    distance_through_haze[idx1] = distance[idx1] - distance_through_fog[idx1]
+    distance_through_haze[idx3] = (
+        (HT - CAMERA_ALTITUDE) * distance[idx3] / (elevation[idx3] - CAMERA_ALTITUDE)
+    )
+    distance_through_fog[idx3] = (
+        (FT - HT) * distance[idx3] / (elevation[idx3] - CAMERA_ALTITUDE)
+    )
+    distance_through_haze_free[idx3] = (
+        distance[idx3] - distance_through_haze[idx3] - distance_through_fog[idx3]
+    )
+
+    attenuation = np.exp(-ECA * distance_through_haze - ECM * distance_through_fog)
+
+    I_ex = image * attenuation[:, :, None]
+    O_p = 1 - attenuation
+    if fog_color is None:
+        fog_color = np.random.randint(200, 255)
+    I_al = np.array([[[fog_color, fog_color, fog_color]]])
+
+    I = I_ex + O_p[:, :, None] * I_al
+    return I.astype(np.uint8)
+
+
+def fogAttenuation(img: np.ndarray, depth: np.ndarray, visibility=1000, fog_color=200):
+    img_fog = generate_fog(
+        img.copy(), depth.copy(), visibility=visibility, fog_color=fog_color
+    )
+    return img_fog

lib/gan_networks.py

@@ -0,0 +1,616 @@
+import torch
+import torch.nn as nn
+from torch.nn import init
+import functools
+from torch.optim import lr_scheduler
+
+
+###############################################################################
+# Helper Functions
+###############################################################################
+
+
+class Identity(nn.Module):
+    def forward(self, x):
+        return x
+
+
+def get_norm_layer(norm_type='instance'):
+    """Return a normalization layer
+
+    Parameters:
+        norm_type (str) -- the name of the normalization layer: batch | instance | none
+
+    For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev).
+    For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics.
+    """
+    if norm_type == 'batch':
+        norm_layer = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True)
+    elif norm_type == 'instance':
+        norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)
+    elif norm_type == 'none':
+        def norm_layer(x):
+            return Identity()
+    else:
+        raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
+    return norm_layer
+
+
+def get_scheduler(optimizer, opt):
+    """Return a learning rate scheduler
+
+    Parameters:
+        optimizer          -- the optimizer of the network
+        opt (option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions.
+                              opt.lr_policy is the name of learning rate policy: linear | step | plateau | cosine
+
+    For 'linear', we keep the same learning rate for the first <opt.n_epochs> epochs
+    and linearly decay the rate to zero over the next <opt.n_epochs_decay> epochs.
+    For other schedulers (step, plateau, and cosine), we use the default PyTorch schedulers.
+    See https://pytorch.org/docs/stable/optim.html for more details.
+    """
+    if opt.lr_policy == 'linear':
+        def lambda_rule(epoch):
+            lr_l = 1.0 - max(0, epoch + opt.epoch_count - opt.n_epochs) / float(opt.n_epochs_decay + 1)
+            return lr_l
+        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule)
+    elif opt.lr_policy == 'step':
+        scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_iters, gamma=0.1)
+    elif opt.lr_policy == 'plateau':
+        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5)
+    elif opt.lr_policy == 'cosine':
+        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=opt.n_epochs, eta_min=0)
+    else:
+        return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy)
+    return scheduler
+
+
+def init_weights(net, init_type='normal', init_gain=0.02):
+    """Initialize network weights.
+
+    Parameters:
+        net (network)   -- network to be initialized
+        init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal
+        init_gain (float)    -- scaling factor for normal, xavier and orthogonal.
+
+    We use 'normal' in the original pix2pix and CycleGAN paper. But xavier and kaiming might
+    work better for some applications. Feel free to try yourself.
+    """
+    def init_func(m):  # define the initialization function
+        classname = m.__class__.__name__
+        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
+            if init_type == 'normal':
+                init.normal_(m.weight.data, 0.0, init_gain)
+            elif init_type == 'xavier':
+                init.xavier_normal_(m.weight.data, gain=init_gain)
+            elif init_type == 'kaiming':
+                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
+            elif init_type == 'orthogonal':
+                init.orthogonal_(m.weight.data, gain=init_gain)
+            else:
+                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
+            if hasattr(m, 'bias') and m.bias is not None:
+                init.constant_(m.bias.data, 0.0)
+        elif classname.find('BatchNorm2d') != -1:  # BatchNorm Layer's weight is not a matrix; only normal distribution applies.
+            init.normal_(m.weight.data, 1.0, init_gain)
+            init.constant_(m.bias.data, 0.0)
+
+    print('initialize network with %s' % init_type)
+    net.apply(init_func)  # apply the initialization function <init_func>
+
+
+def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]):
+    """Initialize a network: 1. register CPU/GPU device (with multi-GPU support); 2. initialize the network weights
+    Parameters:
+        net (network)      -- the network to be initialized
+        init_type (str)    -- the name of an initialization method: normal | xavier | kaiming | orthogonal
+        gain (float)       -- scaling factor for normal, xavier and orthogonal.
+        gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2
+
+    Return an initialized network.
+    """
+    if len(gpu_ids) > 0:
+        assert(torch.cuda.is_available())
+        net.to(gpu_ids[0])
+        net = torch.nn.DataParallel(net, gpu_ids)  # multi-GPUs
+    init_weights(net, init_type, init_gain=init_gain)
+    return net
+
+
+def define_G(input_nc, output_nc, ngf, netG, norm='batch', use_dropout=False, init_type='normal', init_gain=0.02, gpu_ids=[]):
+    """Create a generator
+
+    Parameters:
+        input_nc (int) -- the number of channels in input images
+        output_nc (int) -- the number of channels in output images
+        ngf (int) -- the number of filters in the last conv layer
+        netG (str) -- the architecture's name: resnet_9blocks | resnet_6blocks | unet_256 | unet_128
+        norm (str) -- the name of normalization layers used in the network: batch | instance | none
+        use_dropout (bool) -- if use dropout layers.
+        init_type (str)    -- the name of our initialization method.
+        init_gain (float)  -- scaling factor for normal, xavier and orthogonal.
+        gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2
+
+    Returns a generator
+
+    Our current implementation provides two types of generators:
+        U-Net: [unet_128] (for 128x128 input images) and [unet_256] (for 256x256 input images)
+        The original U-Net paper: https://arxiv.org/abs/1505.04597
+
+        Resnet-based generator: [resnet_6blocks] (with 6 Resnet blocks) and [resnet_9blocks] (with 9 Resnet blocks)
+        Resnet-based generator consists of several Resnet blocks between a few downsampling/upsampling operations.
+        We adapt Torch code from Justin Johnson's neural style transfer project (https://github.com/jcjohnson/fast-neural-style).
+
+
+    The generator has been initialized by <init_net>. It uses RELU for non-linearity.
+    """
+    net = None
+    norm_layer = get_norm_layer(norm_type=norm)
+
+    if netG == 'resnet_9blocks':
+        net = ResnetGenerator(input_nc, output_nc, ngf, norm_layer=norm_layer, use_dropout=use_dropout, n_blocks=9)
+    elif netG == 'resnet_6blocks':
+        net = ResnetGenerator(input_nc, output_nc, ngf, norm_layer=norm_layer, use_dropout=use_dropout, n_blocks=6)
+    elif netG == 'unet_128':
+        net = UnetGenerator(input_nc, output_nc, 7, ngf, norm_layer=norm_layer, use_dropout=use_dropout)
+    elif netG == 'unet_256':
+        net = UnetGenerator(input_nc, output_nc, 8, ngf, norm_layer=norm_layer, use_dropout=use_dropout)
+    else:
+        raise NotImplementedError('Generator model name [%s] is not recognized' % netG)
+    return init_net(net, init_type, init_gain, gpu_ids)
+
+
+def define_D(input_nc, ndf, netD, n_layers_D=3, norm='batch', init_type='normal', init_gain=0.02, gpu_ids=[]):
+    """Create a discriminator
+
+    Parameters:
+        input_nc (int)     -- the number of channels in input images
+        ndf (int)          -- the number of filters in the first conv layer
+        netD (str)         -- the architecture's name: basic | n_layers | pixel
+        n_layers_D (int)   -- the number of conv layers in the discriminator; effective when netD=='n_layers'
+        norm (str)         -- the type of normalization layers used in the network.
+        init_type (str)    -- the name of the initialization method.
+        init_gain (float)  -- scaling factor for normal, xavier and orthogonal.
+        gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2
+
+    Returns a discriminator
+
+    Our current implementation provides three types of discriminators:
+        [basic]: 'PatchGAN' classifier described in the original pix2pix paper.
+        It can classify whether 70×70 overlapping patches are real or fake.
+        Such a patch-level discriminator architecture has fewer parameters
+        than a full-image discriminator and can work on arbitrarily-sized images
+        in a fully convolutional fashion.
+
+        [n_layers]: With this mode, you can specify the number of conv layers in the discriminator
+        with the parameter <n_layers_D> (default=3 as used in [basic] (PatchGAN).)
+
+        [pixel]: 1x1 PixelGAN discriminator can classify whether a pixel is real or not.
+        It encourages greater color diversity but has no effect on spatial statistics.
+
+    The discriminator has been initialized by <init_net>. It uses Leakly RELU for non-linearity.
+    """
+    net = None
+    norm_layer = get_norm_layer(norm_type=norm)
+
+    if netD == 'basic':  # default PatchGAN classifier
+        net = NLayerDiscriminator(input_nc, ndf, n_layers=3, norm_layer=norm_layer)
+    elif netD == 'n_layers':  # more options
+        net = NLayerDiscriminator(input_nc, ndf, n_layers_D, norm_layer=norm_layer)
+    elif netD == 'pixel':     # classify if each pixel is real or fake
+        net = PixelDiscriminator(input_nc, ndf, norm_layer=norm_layer)
+    else:
+        raise NotImplementedError('Discriminator model name [%s] is not recognized' % netD)
+    return init_net(net, init_type, init_gain, gpu_ids)
+
+
+##############################################################################
+# Classes
+##############################################################################
+class GANLoss(nn.Module):
+    """Define different GAN objectives.
+
+    The GANLoss class abstracts away the need to create the target label tensor
+    that has the same size as the input.
+    """
+
+    def __init__(self, gan_mode, target_real_label=1.0, target_fake_label=0.0):
+        """ Initialize the GANLoss class.
+
+        Parameters:
+            gan_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp.
+            target_real_label (bool) - - label for a real image
+            target_fake_label (bool) - - label of a fake image
+
+        Note: Do not use sigmoid as the last layer of Discriminator.
+        LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss.
+        """
+        super(GANLoss, self).__init__()
+        self.register_buffer('real_label', torch.tensor(target_real_label))
+        self.register_buffer('fake_label', torch.tensor(target_fake_label))
+        self.gan_mode = gan_mode
+        if gan_mode == 'lsgan':
+            self.loss = nn.MSELoss()
+        elif gan_mode == 'vanilla':
+            self.loss = nn.BCEWithLogitsLoss()
+        elif gan_mode in ['wgangp']:
+            self.loss = None
+        else:
+            raise NotImplementedError('gan mode %s not implemented' % gan_mode)
+
+    def get_target_tensor(self, prediction, target_is_real):
+        """Create label tensors with the same size as the input.
+
+        Parameters:
+            prediction (tensor) - - tpyically the prediction from a discriminator
+            target_is_real (bool) - - if the ground truth label is for real images or fake images
+
+        Returns:
+            A label tensor filled with ground truth label, and with the size of the input
+        """
+
+        if target_is_real:
+            target_tensor = self.real_label
+        else:
+            target_tensor = self.fake_label
+        return target_tensor.expand_as(prediction)
+
+    def __call__(self, prediction, target_is_real):
+        """Calculate loss given Discriminator's output and grount truth labels.
+
+        Parameters:
+            prediction (tensor) - - tpyically the prediction output from a discriminator
+            target_is_real (bool) - - if the ground truth label is for real images or fake images
+
+        Returns:
+            the calculated loss.
+        """
+        if self.gan_mode in ['lsgan', 'vanilla']:
+            target_tensor = self.get_target_tensor(prediction, target_is_real)
+            loss = self.loss(prediction, target_tensor)
+        elif self.gan_mode == 'wgangp':
+            if target_is_real:
+                loss = -prediction.mean()
+            else:
+                loss = prediction.mean()
+        return loss
+
+
+def cal_gradient_penalty(netD, real_data, fake_data, device, type='mixed', constant=1.0, lambda_gp=10.0):
+    """Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028
+
+    Arguments:
+        netD (network)              -- discriminator network
+        real_data (tensor array)    -- real images
+        fake_data (tensor array)    -- generated images from the generator
+        device (str)                -- GPU / CPU: from torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')
+        type (str)                  -- if we mix real and fake data or not [real | fake | mixed].
+        constant (float)            -- the constant used in formula ( ||gradient||_2 - constant)^2
+        lambda_gp (float)           -- weight for this loss
+
+    Returns the gradient penalty loss
+    """
+    if lambda_gp > 0.0:
+        if type == 'real':   # either use real images, fake images, or a linear interpolation of two.
+            interpolatesv = real_data
+        elif type == 'fake':
+            interpolatesv = fake_data
+        elif type == 'mixed':
+            alpha = torch.rand(real_data.shape[0], 1, device=device)
+            alpha = alpha.expand(real_data.shape[0], real_data.nelement() // real_data.shape[0]).contiguous().view(*real_data.shape)
+            interpolatesv = alpha * real_data + ((1 - alpha) * fake_data)
+        else:
+            raise NotImplementedError('{} not implemented'.format(type))
+        interpolatesv.requires_grad_(True)
+        disc_interpolates = netD(interpolatesv)
+        gradients = torch.autograd.grad(outputs=disc_interpolates, inputs=interpolatesv,
+                                        grad_outputs=torch.ones(disc_interpolates.size()).to(device),
+                                        create_graph=True, retain_graph=True, only_inputs=True)
+        gradients = gradients[0].view(real_data.size(0), -1)  # flat the data
+        gradient_penalty = (((gradients + 1e-16).norm(2, dim=1) - constant) ** 2).mean() * lambda_gp        # added eps
+        return gradient_penalty, gradients
+    else:
+        return 0.0, None
+
+
+class ResnetGenerator(nn.Module):
+    """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations.
+
+    We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style)
+    """
+
+    def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'):
+        """Construct a Resnet-based generator
+
+        Parameters:
+            input_nc (int)      -- the number of channels in input images
+            output_nc (int)     -- the number of channels in output images
+            ngf (int)           -- the number of filters in the last conv layer
+            norm_layer          -- normalization layer
+            use_dropout (bool)  -- if use dropout layers
+            n_blocks (int)      -- the number of ResNet blocks
+            padding_type (str)  -- the name of padding layer in conv layers: reflect | replicate | zero
+        """
+        assert(n_blocks >= 0)
+        super(ResnetGenerator, self).__init__()
+        if type(norm_layer) == functools.partial:
+            use_bias = norm_layer.func == nn.InstanceNorm2d
+        else:
+            use_bias = norm_layer == nn.InstanceNorm2d
+
+        model = [nn.ReflectionPad2d(3),
+                 nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),
+                 norm_layer(ngf),
+                 nn.ReLU(True)]
+
+        n_downsampling = 2
+        for i in range(n_downsampling):  # add downsampling layers
+            mult = 2 ** i
+            model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias),
+                      norm_layer(ngf * mult * 2),
+                      nn.ReLU(True)]
+
+        mult = 2 ** n_downsampling
+        for i in range(n_blocks):       # add ResNet blocks
+
+            model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
+
+        for i in range(n_downsampling):  # add upsampling layers
+            mult = 2 ** (n_downsampling - i)
+            model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),
+                                         kernel_size=3, stride=2,
+                                         padding=1, output_padding=1,
+                                         bias=use_bias),
+                      norm_layer(int(ngf * mult / 2)),
+                      nn.ReLU(True)]
+        model += [nn.ReflectionPad2d(3)]
+        model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]
+        model += [nn.Tanh()]
+
+        self.model = nn.Sequential(*model)
+
+    def forward(self, input):
+        """Standard forward"""
+        return self.model(input)
+
+
+class ResnetBlock(nn.Module):
+    """Define a Resnet block"""
+
+    def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias):
+        """Initialize the Resnet block
+
+        A resnet block is a conv block with skip connections
+        We construct a conv block with build_conv_block function,
+        and implement skip connections in <forward> function.
+        Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf
+        """
+        super(ResnetBlock, self).__init__()
+        self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias)
+
+    def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias):
+        """Construct a convolutional block.
+
+        Parameters:
+            dim (int)           -- the number of channels in the conv layer.
+            padding_type (str)  -- the name of padding layer: reflect | replicate | zero
+            norm_layer          -- normalization layer
+            use_dropout (bool)  -- if use dropout layers.
+            use_bias (bool)     -- if the conv layer uses bias or not
+
+        Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU))
+        """
+        conv_block = []
+        p = 0
+        if padding_type == 'reflect':
+            conv_block += [nn.ReflectionPad2d(1)]
+        elif padding_type == 'replicate':
+            conv_block += [nn.ReplicationPad2d(1)]
+        elif padding_type == 'zero':
+            p = 1
+        else:
+            raise NotImplementedError('padding [%s] is not implemented' % padding_type)
+
+        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)]
+        if use_dropout:
+            conv_block += [nn.Dropout(0.5)]
+
+        p = 0
+        if padding_type == 'reflect':
+            conv_block += [nn.ReflectionPad2d(1)]
+        elif padding_type == 'replicate':
+            conv_block += [nn.ReplicationPad2d(1)]
+        elif padding_type == 'zero':
+            p = 1
+        else:
+            raise NotImplementedError('padding [%s] is not implemented' % padding_type)
+        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)]
+
+        return nn.Sequential(*conv_block)
+
+    def forward(self, x):
+        """Forward function (with skip connections)"""
+        out = x + self.conv_block(x)  # add skip connections
+        return out
+
+
+class UnetGenerator(nn.Module):
+    """Create a Unet-based generator"""
+
+    def __init__(self, input_nc, output_nc, num_downs, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False):
+        """Construct a Unet generator
+        Parameters:
+            input_nc (int)  -- the number of channels in input images
+            output_nc (int) -- the number of channels in output images
+            num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7,
+                                image of size 128x128 will become of size 1x1 # at the bottleneck
+            ngf (int)       -- the number of filters in the last conv layer
+            norm_layer      -- normalization layer
+
+        We construct the U-Net from the innermost layer to the outermost layer.
+        It is a recursive process.
+        """
+        super(UnetGenerator, self).__init__()
+        # construct unet structure
+        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)  # add the innermost layer
+        for i in range(num_downs - 5):          # add intermediate layers with ngf * 8 filters
+            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)
+        # gradually reduce the number of filters from ngf * 8 to ngf
+        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
+        unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
+        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
+        self.model = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)  # add the outermost layer
+
+    def forward(self, input):
+        """Standard forward"""
+        return self.model(input)
+
+
+class UnetSkipConnectionBlock(nn.Module):
+    """Defines the Unet submodule with skip connection.
+        X -------------------identity----------------------
+        |-- downsampling -- |submodule| -- upsampling --|
+    """
+
+    def __init__(self, outer_nc, inner_nc, input_nc=None,
+                 submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):
+        """Construct a Unet submodule with skip connections.
+
+        Parameters:
+            outer_nc (int) -- the number of filters in the outer conv layer
+            inner_nc (int) -- the number of filters in the inner conv layer
+            input_nc (int) -- the number of channels in input images/features
+            submodule (UnetSkipConnectionBlock) -- previously defined submodules
+            outermost (bool)    -- if this module is the outermost module
+            innermost (bool)    -- if this module is the innermost module
+            norm_layer          -- normalization layer
+            use_dropout (bool)  -- if use dropout layers.
+        """
+        super(UnetSkipConnectionBlock, self).__init__()
+        self.outermost = outermost
+        if type(norm_layer) == functools.partial:
+            use_bias = norm_layer.func == nn.InstanceNorm2d
+        else:
+            use_bias = norm_layer == nn.InstanceNorm2d
+        if input_nc is None:
+            input_nc = outer_nc
+        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,
+                             stride=2, padding=1, bias=use_bias)
+        downrelu = nn.LeakyReLU(0.2, True)
+        downnorm = norm_layer(inner_nc)
+        uprelu = nn.ReLU(True)
+        upnorm = norm_layer(outer_nc)
+
+        if outermost:
+            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
+                                        kernel_size=4, stride=2,
+                                        padding=1)
+            down = [downconv]
+            up = [uprelu, upconv, nn.Tanh()]
+            model = down + [submodule] + up
+        elif innermost:
+            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,
+                                        kernel_size=4, stride=2,
+                                        padding=1, bias=use_bias)
+            down = [downrelu, downconv]
+            up = [uprelu, upconv, upnorm]
+            model = down + up
+        else:
+            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
+                                        kernel_size=4, stride=2,
+                                        padding=1, bias=use_bias)
+            down = [downrelu, downconv, downnorm]
+            up = [uprelu, upconv, upnorm]
+
+            if use_dropout:
+                model = down + [submodule] + up + [nn.Dropout(0.5)]
+            else:
+                model = down + [submodule] + up
+
+        self.model = nn.Sequential(*model)
+
+    def forward(self, x):
+        if self.outermost:
+            return self.model(x)
+        else:   # add skip connections
+            return torch.cat([x, self.model(x)], 1)
+
+
+class NLayerDiscriminator(nn.Module):
+    """Defines a PatchGAN discriminator"""
+
+    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d):
+        """Construct a PatchGAN discriminator
+
+        Parameters:
+            input_nc (int)  -- the number of channels in input images
+            ndf (int)       -- the number of filters in the last conv layer
+            n_layers (int)  -- the number of conv layers in the discriminator
+            norm_layer      -- normalization layer
+        """
+        super(NLayerDiscriminator, self).__init__()
+        if type(norm_layer) == functools.partial:  # no need to use bias as BatchNorm2d has affine parameters
+            use_bias = norm_layer.func == nn.InstanceNorm2d
+        else:
+            use_bias = norm_layer == nn.InstanceNorm2d
+
+        kw = 4
+        padw = 1
+        sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+        nf_mult = 1
+        nf_mult_prev = 1
+        for n in range(1, n_layers):  # gradually increase the number of filters
+            nf_mult_prev = nf_mult
+            nf_mult = min(2 ** n, 8)
+            sequence += [
+                nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
+                norm_layer(ndf * nf_mult),
+                nn.LeakyReLU(0.2, True)
+            ]
+
+        nf_mult_prev = nf_mult
+        nf_mult = min(2 ** n_layers, 8)
+        sequence += [
+            nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
+            norm_layer(ndf * nf_mult),
+            nn.LeakyReLU(0.2, True)
+        ]
+
+        sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]  # output 1 channel prediction map
+        self.model = nn.Sequential(*sequence)
+
+    def forward(self, input):
+        """Standard forward."""
+        return self.model(input)
+
+
+class PixelDiscriminator(nn.Module):
+    """Defines a 1x1 PatchGAN discriminator (pixelGAN)"""
+
+    def __init__(self, input_nc, ndf=64, norm_layer=nn.BatchNorm2d):
+        """Construct a 1x1 PatchGAN discriminator
+
+        Parameters:
+            input_nc (int)  -- the number of channels in input images
+            ndf (int)       -- the number of filters in the last conv layer
+            norm_layer      -- normalization layer
+        """
+        super(PixelDiscriminator, self).__init__()
+        if type(norm_layer) == functools.partial:  # no need to use bias as BatchNorm2d has affine parameters
+            use_bias = norm_layer.func == nn.InstanceNorm2d
+        else:
+            use_bias = norm_layer == nn.InstanceNorm2d
+
+        self.net = [
+            nn.Conv2d(input_nc, ndf, kernel_size=1, stride=1, padding=0),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(ndf, ndf * 2, kernel_size=1, stride=1, padding=0, bias=use_bias),
+            norm_layer(ndf * 2),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(ndf * 2, 1, kernel_size=1, stride=1, padding=0, bias=use_bias)]
+
+        self.net = nn.Sequential(*self.net)
+
+    def forward(self, input):
+        """Standard forward."""
+        return self.net(input)

lib/gen_utils.py

@@ -0,0 +1,263 @@
+import cv2
+import numpy as np
+from skimage import measure
+from skimage import color, filters
+from sklearn.neighbors import NearestNeighbors
+
+
+def get_otsu_threshold(image):
+    image = cv2.GaussianBlur(image.astype(float), (7, 7), 0)
+    ret, _ = cv2.threshold(
+        image.astype(np.uint8), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU
+    )
+    return ret
+
+
+def reduce_lightHSV(rgb, sat_red=0.5, val_red=0.5):
+    hsv = color.rgb2hsv(rgb / 255)
+    hsv[..., 1] *= sat_red
+    hsv[..., 2] *= val_red
+    return (color.hsv2rgb(hsv) * 255).astype(np.uint8)
+
+
+def apply_motion_blur_(image, size):
+    """
+    input:
+        image - numpy array of image
+        size - in pixels, size of motion blur
+    output:
+        blurred image as numpy array
+    """
+    k = np.zeros((size, size), dtype=np.float32)
+    k[(size - 1) // 2, :] = np.ones(size, dtype=np.float32)
+    k = k * (1.0 / np.sum(k))
+    return cv2.filter2D(image, -1, k).astype(np.uint8)
+
+
+def apply_motion_blur(image, size, angle):
+    """
+    input:
+        image - numpy array of image
+        size - in pixels, size of motion blur
+        angel - in degrees, direction of motion blur
+    output:
+        blurred image as numpy array
+    """
+    k = np.zeros((size, size), dtype=np.float32)
+    k[(size - 1) // 2, :] = np.ones(size, dtype=np.float32)
+    k = cv2.warpAffine(
+        k,
+        cv2.getRotationMatrix2D((size / 2 - 0.5, size / 2 - 0.5), angle, 1.0),
+        (size, size),
+    )
+    k = k * (1.0 / np.sum(k))
+    return cv2.filter2D(image, -1, k).astype(np.uint8)
+
+
+def illumination2opacity(img: np.ndarray, illumination):
+    alpha = color.rgb2gray(img)
+    if illumination > 0:
+        alpha = np.clip(
+            filters.gaussian((1 - alpha), sigma=20, channel_axis=None), 0, 1
+        )
+    else:
+        alpha = np.clip(
+            2 * filters.gaussian((alpha), sigma=20, channel_axis=None), 0, 1
+        )
+    return alpha
+
+
+def color_level_adjustment(
+    image, inBlack=0, inWhite=255, inGamma=1.0, outBlack=0, outWhite=255
+):
+    """
+    Adjust color level.
+    input:
+        image    - numpy array of greyscale image
+        inBlack  - lower limit of intensity
+        inWhite  - upper limit of intensity
+        inGamma  - scaling the intensity values by Gamma value
+        outBlack - lower intensity value for scaling
+        outWhite - upper intensity value for scaling
+    """
+    assert image.ndim == 2
+
+    # image = np.clip( (image - inBlack) / (inWhite - inBlack), 0, 1)
+    image = (image - inBlack) / (inWhite - inBlack)
+    image[image < 0] = 0
+    image[image > 1] = 0
+    image = (image ** (1 / inGamma)) * (outWhite - outBlack) + outBlack
+    image = np.clip(image, 0, 255).astype(np.uint8)
+    return image.astype(np.uint8)
+
+
+def crystallize(img, r):
+    """
+    Crystallization Effect
+    input: img - Numpy Array
+           r   - fraction of  pixels to select as center for crystallization
+    outpur: res- Numpy Array for crystallized filter
+    """
+    if img.ndim == 2:
+        h, w = img.shape
+    elif img.ndim == 3:
+        h, w, _ = img.shape
+
+    # Get the center for crystallization
+    pixels = np.zeros((h * w, 2), dtype=np.uint16)
+    pixels[:, 0] = np.tile(np.arange(h), (w, 1)).T.reshape(-1)
+    pixels[:, 1] = (np.tile(np.arange(w), (h, 1))).reshape(-1)
+
+    sel_pixels = pixels.copy()
+    sel_pixels = sel_pixels[np.random.randint(0, h * w, int(len(sel_pixels) * r))]
+
+    # Perform nearest neighbour for all pixels
+    nbrs = NearestNeighbors(n_neighbors=1, algorithm="ball_tree", n_jobs=4).fit(
+        sel_pixels
+    )
+    distances, indices = nbrs.kneighbors(pixels)
+    color_pixels = sel_pixels[indices[:, 0]]
+
+    # Perform crystallization (copy the color pixels of crystal center)
+    res = np.zeros_like(img)
+    res[pixels[:, 0], pixels[:, 1]] = img[color_pixels[:, 0], color_pixels[:, 1]]
+    return res
+
+
+def zoom_image_and_crop(image, r=1.5):
+    """
+    input:
+        image: numpy array
+        r = upscale fraction >1.0
+    output:
+        image: scale image as numpy array
+    """
+    if image.ndim == 2:
+        h, w = image.shape
+    elif image.ndim == 3:
+        h, w, _ = image.shape
+    image_resize = cv2.resize(
+        image.astype(np.uint8),
+        (int(w * r), int(h * r)),
+        interpolation=cv2.INTER_LANCZOS4,
+    )
+
+    x = int(r * w / 2 - w / 2)
+    y = int(r * h / 2 - h / 2)
+    crop_img = image_resize[int(y) : int(y + h), int(x) : int(x + w)]
+
+    return crop_img.astype(np.uint8)
+
+
+def repeat_and_combine(layer, repeat_scale=2):
+    orgh, orgw = layer.shape
+    compressh = int(np.floor(orgh / repeat_scale))
+    compressw = int(np.floor(orgw / repeat_scale))
+
+    resize_layer = cv2.resize(
+        layer, (compressw, compressh), interpolation=cv2.INTER_LANCZOS4
+    )
+    layer_tile = np.tile(resize_layer, (repeat_scale, repeat_scale))
+    h, w = layer_tile.shape
+
+    repeat = np.zeros_like(layer)
+    repeat[:h, :w] = layer_tile
+    return repeat.astype(np.uint8)
+
+
+def generate_noisy_image(h, w, sigma=0.5, p=0.5):
+    """
+    input:
+        h - height of the image
+        w - width of the image
+        scale - scale of Gaussian noise
+    output:
+        im_noisy - uint8 array with Gaussian noise
+    """
+    im_array = np.zeros((h, w))
+
+    # Generate random Gaussian noise
+    noise = np.random.normal(scale=sigma, size=(h, w))
+    prob = np.random.rand(h, w)
+    im_array[prob < p] = 255 * noise[prob < p]
+    im_array = np.clip(im_array, 0, 255)
+    return im_array.astype(np.uint8)
+
+
+def binarizeImage(image: np.ndarray):
+    """Binarize grey image using OTSU threshold"""
+    if image.ndim == 3:
+        if image.shape[2] == 3:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        else:
+            image = image[:, :, 0]
+    binarize = np.copy(image)
+    ret = get_otsu_threshold(image=image)
+    binarize[binarize < ret] = 0
+    binarize[binarize > ret] = 255
+    return binarize
+
+
+def bwAreaFilter(mask, area_range=(0, np.inf)):
+    """Extract objects from binary image by size"""
+    labels = measure.label(mask.astype("uint8"), background=0)
+    unq, areas = np.unique(labels, return_counts=True)
+    areas = areas[1:]
+    area_idx = np.arange(1, np.max(labels) + 1)
+
+    inside_range_idx = np.logical_and(areas >= area_range[0], areas <= area_range[1])
+    area_idx = area_idx[inside_range_idx]
+    areas = areas[inside_range_idx]
+    layer = np.isin(labels, area_idx)
+    return layer.astype(int)
+
+
+def centreCrop(image, reqH, reqW):
+    center = image.shape
+    x = center[1] / 2 - reqW / 2
+    y = center[0] / 2 - reqH / 2
+
+    crop_img = image[int(y) : int(y + reqH), int(x) : int(x + reqW)]
+    return crop_img
+
+
+def alpha_blend(img, layer, alpha):
+    if layer.ndim == 3:
+        layer = cv2.cvtColor(layer.astype(np.uint8), cv2.COLOR_RGB2GRAY)
+
+    assert alpha.ndim == 2
+    assert layer.ndim == 2
+    blended = img * (1 - alpha[:, :, None]) + layer[:, :, None] * alpha[:, :, None]
+    return blended
+
+
+def screen_blend(image, layer):
+    """
+    input:
+        image - numpy array of RGB image
+        layer - numpy array of layer to blend
+    """
+    result = 255.0 * (1 - (1 - image / 255.0) * (1 - layer[:, :, None] / 255.0))
+    return result.astype(np.uint8)
+
+
+def layer_blend(layer1, layer2):
+    """
+    input:
+        layer1 - numpy array of RGB image
+        layer2 - numpy array of layer to blend
+    """
+    assert layer1.shape == layer2.shape
+    result = 255.0 * (1 - (1 - layer1 / 255.0) * (1 - layer2 / 255.0))
+    return result.astype(np.uint8)
+
+
+def scale_depth(im, nR, nC):
+    nR0 = len(im)  # source number of rows
+    nC0 = len(im[0])  # source number of columns
+    return np.asarray(
+        [
+            [im[int(nR0 * r / nR)][int(nC0 * c / nC)] for c in range(nC)]
+            for r in range(nR)
+        ]
+    )

lib/lime.py

@@ -0,0 +1,111 @@
+import tqdm
+import numpy as np
+from scipy import fft
+from skimage import io, exposure, img_as_ubyte, img_as_float
+
+def firstOrderDerivative(n, k=1):
+    return np.eye(n) * (-1) + np.eye(n, k=k)
+
+
+def toeplitizMatrix(n, row):
+    vecDD = np.zeros(n)
+    vecDD[0] = 4
+    vecDD[1] = -1
+    vecDD[row] = -1
+    vecDD[-1] = -1
+    vecDD[-row] = -1
+    return vecDD
+
+
+def vectorize(matrix):
+    return matrix.T.ravel()
+
+
+def reshape(vector, row, col):
+    return vector.reshape((row, col), order='F')
+
+
+class LIME:
+    def __init__(self, iterations=10, alpha=2, rho=1.5, gamma=0.7, strategy=2, *args, **kwargs):
+        self.iterations = iterations
+        self.alpha = alpha
+        self.rho = rho
+        self.gamma = gamma
+        self.strategy = strategy
+
+    def load(self, imgPath):
+        if isinstance(imgPath, str):
+            self.L = img_as_float(io.imread(imgPath))
+        elif isinstance(imgPath, np.ndarray):
+            self.L = img_as_float(imgPath)
+        else:
+            print(f"The input should be path to image of numpy array.")
+
+        self.row = self.L.shape[0]
+        self.col = self.L.shape[1]
+
+        self.T_hat = np.max(self.L, axis=2)
+        self.dv = firstOrderDerivative(self.row)
+        self.dh = firstOrderDerivative(self.col, -1)
+        self.vecDD = toeplitizMatrix(self.row * self.col, self.row)
+        self.W = self.weightingStrategy()
+
+    def weightingStrategy(self):
+        if self.strategy == 2:
+            dTv = self.dv @ self.T_hat
+            dTh = self.T_hat @ self.dh
+            Wv = 1 / (np.abs(dTv) + 1)
+            Wh = 1 / (np.abs(dTh) + 1)
+            return np.vstack([Wv, Wh])
+        else:
+            return np.ones((self.row * 2, self.col))
+
+    def __T_subproblem(self, G, Z, u):
+        X = G - Z / u
+        Xv = X[:self.row, :]
+        Xh = X[self.row:, :]
+        temp = self.dv @ Xv + Xh @ self.dh
+        numerator = fft.fft(vectorize(2 * self.T_hat + u * temp))
+        denominator = fft.fft(self.vecDD * u) + 2
+        T = fft.ifft(numerator / denominator)
+        T = np.real(reshape(T, self.row, self.col))
+        return exposure.rescale_intensity(T, (0, 1), (0.001, 1))
+
+    def __G_subproblem(self, T, Z, u, W):
+        dT = self.__derivative(T)
+        epsilon = self.alpha * W / u
+        X = dT + Z / u
+        return np.sign(X) * np.maximum(np.abs(X) - epsilon, 0)
+
+    def __Z_subproblem(self, T, G, Z, u):
+        dT = self.__derivative(T)
+        return Z + u * (dT - G)
+
+    def __u_subproblem(self, u):
+        return u * self.rho
+
+    def __derivative(self, matrix):
+        v = self.dv @ matrix
+        h = matrix @ self.dh
+        return np.vstack([v, h])
+
+    def illumMap(self):
+        T = np.zeros((self.row, self.col))
+        G = np.zeros((self.row * 2, self.col))
+        Z = np.zeros((self.row * 2, self.col))
+        u = 1
+
+        for _ in tqdm.trange(0, self.iterations):
+            T = self.__T_subproblem(G, Z, u)
+            G = self.__G_subproblem(T, Z, u, self.W)
+            Z = self.__Z_subproblem(T, G, Z, u)
+            u = self.__u_subproblem(u)
+
+        return T ** self.gamma
+
+    def enhance(self):
+        self.T = self.illumMap()
+        self.R = self.L / np.repeat(self.T[:, :, np.newaxis], 3, axis=2)
+        self.R = exposure.rescale_intensity(self.R, (0, 1))
+        self.R = img_as_ubyte(self.R)
+        return self.R

lib/motionblur.py

@@ -0,0 +1,419 @@
+import numpy as np
+from PIL import Image, ImageDraw, ImageFilter
+from numpy.random import uniform, triangular, beta
+from math import pi
+from pathlib import Path
+from scipy.signal import convolve
+
+# tiny error used for nummerical stability
+eps = 0.1
+
+
+def softmax(x):
+    """Compute softmax values for each sets of scores in x."""
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum()
+
+
+def norm(lst: list) -> float:
+    """[summary]
+    L^2 norm of a list
+    [description]
+    Used for internals
+    Arguments:
+        lst {list} -- vector
+    """
+    if not isinstance(lst, list):
+        raise ValueError("Norm takes a list as its argument")
+
+    if lst == []:
+        return 0
+
+    return (sum((i**2 for i in lst)))**0.5
+
+
+def polar2z(r: np.ndarray, θ: np.ndarray) -> np.ndarray:
+    """[summary]
+    Takes a list of radii and angles (radians) and
+    converts them into a corresponding list of complex
+    numbers x + yi.
+    [description]
+
+    Arguments:
+        r {np.ndarray} -- radius
+        θ {np.ndarray} -- angle
+
+    Returns:
+        [np.ndarray] -- list of complex numbers r e^(i theta) as x + iy
+    """
+    return r * np.exp(1j * θ)
+
+
+class Kernel(object):
+    """[summary]
+    Class representing a motion blur kernel of a given intensity.
+
+    [description]
+    Keyword Arguments:
+            size {tuple} -- Size of the kernel in px times px
+            (default: {(100, 100)})
+
+            intensity {float} -- Float between 0 and 1.
+            Intensity of the motion blur.
+
+            :   0 means linear motion blur and 1 is a highly non linear
+                and often convex motion blur path. (default: {0})
+
+    Attribute:
+    kernelMatrix -- Numpy matrix of the kernel of given intensity
+
+    Properties:
+    applyTo -- Applies kernel to image
+               (pass as path, pillow image or np array)
+
+    Raises:
+        ValueError
+    """
+
+    def __init__(self, size: tuple = (100, 100), intensity: float=0):
+
+        # checking if size is correctly given
+        if not isinstance(size, tuple):
+            raise ValueError("Size must be TUPLE of 2 positive integers")
+        elif len(size) != 2 or type(size[0]) != type(size[1]) != int:
+            raise ValueError("Size must be tuple of 2 positive INTEGERS")
+        elif size[0] < 0 or size[1] < 0:
+            raise ValueError("Size must be tuple of 2 POSITIVE integers")
+
+        # check if intensity is float (int) between 0 and 1
+        if type(intensity) not in [int, float, np.float32, np.float64]:
+            raise ValueError("Intensity must be a number between 0 and 1")
+        elif intensity < 0 or intensity > 1:
+            raise ValueError("Intensity must be a number between 0 and 1")
+
+        # saving args
+        self.SIZE = size
+        self.INTENSITY = intensity
+
+        # deriving quantities
+
+        # we super size first and then downscale at the end for better
+        # anti-aliasing
+        self.SIZEx2 = tuple([2 * i for i in size])
+        self.x, self.y = self.SIZEx2
+
+        # getting length of kernel diagonal
+        self.DIAGONAL = (self.x**2 + self.y**2)**0.5
+
+        # flag to see if kernel has been calculated already
+        self.kernel_is_generated = False
+
+    def _createPath(self):
+        """[summary]
+        creates a motion blur path with the given intensity.
+        [description]
+        Proceede in 5 steps
+        1. Get a random number of random step sizes
+        2. For each step get a random angle
+        3. combine steps and angles into a sequence of increments
+        4. create path out of increments
+        5. translate path to fit the kernel dimensions
+
+        NOTE: "random" means random but might depend on the given intensity
+        """
+
+        # first we find the lengths of the motion blur steps
+        def getSteps():
+            """[summary]
+            Here we calculate the length of the steps taken by
+            the motion blur
+            [description]
+            We want a higher intensity lead to a longer total motion
+            blur path and more different steps along the way.
+
+            Hence we sample
+
+            MAX_PATH_LEN =[U(0,1) + U(0, intensity^2)] * diagonal * 0.75
+
+            and each step: beta(1, 30) * (1 - self.INTENSITY + eps) * diagonal)
+            """
+
+            # getting max length of blur motion
+            self.MAX_PATH_LEN = 0.75 * self.DIAGONAL * \
+                (uniform() + uniform(0, self.INTENSITY**2))
+
+            # getting step
+            steps = []
+
+            while sum(steps) < self.MAX_PATH_LEN:
+
+                # sample next step
+                step = beta(1, 30) * (1 - self.INTENSITY + eps) * self.DIAGONAL
+                if step < self.MAX_PATH_LEN:
+                    steps.append(step)
+
+            # note the steps and the total number of steps
+            self.NUM_STEPS = len(steps)
+            self.STEPS = np.asarray(steps)
+
+        def getAngles():
+            """[summary]
+            Gets an angle for each step
+            [description]
+            The maximal angle should be larger the more
+            intense the motion is. So we sample it from a
+            U(0, intensity * pi)
+
+            We sample "jitter" from a beta(2,20) which is the probability
+            that the next angle has a different sign than the previous one.
+            """
+
+            # same as with the steps
+
+            # first we get the max angle in radians
+            self.MAX_ANGLE = uniform(0, self.INTENSITY * pi)
+
+            # now we sample "jitter" which is the probability that the
+            # next angle has a different sign than the previous one
+            self.JITTER = beta(2, 20)
+
+            # initialising angles (and sign of angle)
+            angles = [uniform(low=-self.MAX_ANGLE, high=self.MAX_ANGLE)]
+
+            while len(angles) < self.NUM_STEPS:
+
+                # sample next angle (absolute value)
+                angle = triangular(0, self.INTENSITY *
+                                   self.MAX_ANGLE, self.MAX_ANGLE + eps)
+
+                # with jitter probability change sign wrt previous angle
+                if uniform() < self.JITTER:
+                    angle *= - np.sign(angles[-1])
+                else:
+                    angle *= np.sign(angles[-1])
+
+                angles.append(angle)
+
+            # save angles
+            self.ANGLES = np.asarray(angles)
+
+        # Get steps and angles
+        getSteps()
+        getAngles()
+
+        # Turn them into a path
+        ####
+
+        # we turn angles and steps into complex numbers
+        complex_increments = polar2z(self.STEPS, self.ANGLES)
+
+        # generate path as the cumsum of these increments
+        self.path_complex = np.cumsum(complex_increments)
+
+        # find center of mass of path
+        self.com_complex = sum(self.path_complex) / self.NUM_STEPS
+
+        # Shift path s.t. center of mass lies in the middle of
+        # the kernel and a apply a random rotation
+        ###
+
+        # center it on COM
+        center_of_kernel = (self.x + 1j * self.y) / 2
+        self.path_complex -= self.com_complex
+
+        # randomly rotate path by an angle a in (0, pi)
+        self.path_complex *= np.exp(1j * uniform(0, pi))
+
+        # center COM on center of kernel
+        self.path_complex += center_of_kernel
+
+        # convert complex path to final list of coordinate tuples
+        self.path = [(i.real, i.imag) for i in self.path_complex]
+
+    def _createKernel(self, save_to: Path=None, show: bool=False):
+        """[summary]
+        Finds a kernel (psf) of given intensity.
+        [description]
+        use displayKernel to actually see the kernel.
+
+        Keyword Arguments:
+            save_to {Path} -- Image file to save the kernel to. {None}
+            show {bool} -- shows kernel if true
+        """
+
+        # check if we haven't already generated a kernel
+        if self.kernel_is_generated:
+            return None
+
+        # get the path
+        self._createPath()
+
+        # Initialise an image with super-sized dimensions
+        # (pillow Image object)
+        self.kernel_image = Image.new("RGB", self.SIZEx2)
+
+        # ImageDraw instance that is linked to the kernel image that
+        # we can use to draw on our kernel_image
+        self.painter = ImageDraw.Draw(self.kernel_image)
+
+        # draw the path
+        self.painter.line(xy=self.path, width=int(self.DIAGONAL / 150))
+
+        # applying gaussian blur for realism
+        self.kernel_image = self.kernel_image.filter(
+            ImageFilter.GaussianBlur(radius=int(self.DIAGONAL * 0.01)))
+
+        # Resize to actual size
+        self.kernel_image = self.kernel_image.resize(
+            self.SIZE, resample=Image.LANCZOS)
+
+        # convert to gray scale
+        self.kernel_image = self.kernel_image.convert("L")
+
+        # flag that we have generated a kernel
+        self.kernel_is_generated = True
+
+    def displayKernel(self, save_to: Path=None, show: bool=True):
+        """[summary]
+        Finds a kernel (psf) of given intensity.
+        [description]
+        Saves the kernel to save_to if needed or shows it
+        is show true
+
+        Keyword Arguments:
+            save_to {Path} -- Image file to save the kernel to. {None}
+            show {bool} -- shows kernel if true
+        """
+
+        # generate kernel if needed
+        self._createKernel()
+
+        # save if needed
+        if save_to is not None:
+
+            save_to_file = Path(save_to)
+
+            # save Kernel image
+            self.kernel_image.save(save_to_file)
+        else:
+            # Show kernel
+            self.kernel_image.show()
+
+    @property
+    def kernelMatrix(self) -> np.ndarray:
+        """[summary]
+        Kernel matrix of motion blur of given intensity.
+        [description]
+        Once generated, it stays the same.
+        Returns:
+            numpy ndarray
+        """
+
+        # generate kernel if needed
+        self._createKernel()
+        kernel = np.asarray(self.kernel_image, dtype=np.float32)
+        kernel /= np.sum(kernel)
+
+        return kernel
+
+    @kernelMatrix.setter
+    def kernelMatrix(self, *kargs):
+        raise NotImplementedError("Can't manually set kernel matrix yet")
+
+    def applyTo(self, image, keep_image_dim: bool = False) -> Image:
+        """[summary]
+        Applies kernel to one of the following:
+
+        1. Path to image file
+        2. Pillow image object
+        3. (H,W,3)-shaped numpy array
+        [description]
+
+        Arguments:
+            image {[str, Path, Image, np.ndarray]}
+            keep_image_dim {bool} -- If true, then we will
+                    conserve the image dimension after blurring
+                    by using "same" convolution instead of "valid"
+                    convolution inside the scipy convolve function.
+
+        Returns:
+            Image -- [description]
+        """
+        # calculate kernel if haven't already
+        self._createKernel()
+
+        def applyToPIL(image: Image, keep_image_dim: bool = False) -> Image:
+            """[summary]
+            Applies the kernel to an PIL.Image instance
+            [description]
+            converts to RGB and applies the kernel to each
+            band before recombining them.
+            Arguments:
+                image {Image} -- Image to convolve
+                keep_image_dim {bool} -- If true, then we will
+                    conserve the image dimension after blurring
+                    by using "same" convolution instead of "valid"
+                    convolution inside the scipy convolve function.
+
+            Returns:
+                Image -- blurred image
+            """
+            # convert to RGB
+            image = image.convert(mode="RGB")
+
+            conv_mode = "valid"
+            if keep_image_dim:
+                conv_mode = "same"
+
+            result_bands = ()
+
+            for band in image.split():
+
+                # convolve each band individually with kernel
+                result_band = convolve(
+                    band, self.kernelMatrix, mode=conv_mode).astype("uint8")
+
+                # collect bands
+                result_bands += result_band,
+
+            # stack bands back together
+            result = np.dstack(result_bands)
+
+            # Get image
+            return Image.fromarray(result)
+
+        # If image is Path
+        if isinstance(image, str) or isinstance(image, Path):
+
+            # open image as Image class
+            image_path = Path(image)
+            image = Image.open(image_path)
+
+            return applyToPIL(image, keep_image_dim)
+
+        elif isinstance(image, Image.Image):
+
+            # apply kernel
+            return applyToPIL(image, keep_image_dim)
+
+        elif isinstance(image, np.ndarray):
+
+            # ASSUMES we have an array of the form (H, W, 3)
+            ###
+
+            # initiate Image object from array
+            image = Image.fromarray(image)
+
+            return applyToPIL(image, keep_image_dim)
+
+        else:
+
+            raise ValueError("Cannot apply kernel to this type.")
+
+
+if __name__ == '__main__':
+    image = Image.open("./images/moon.png")
+    image.show()
+    k = Kernel()
+
+    k.applyTo(image, keep_image_dim=True).show()

lib/rain_gen.py

@@ -0,0 +1,162 @@
+import cv2
+import scipy
+import random
+import numpy as np
+from pathlib import Path
+from lib.gen_utils import (
+    generate_noisy_image,
+    centreCrop,
+    binarizeImage,
+    bwAreaFilter,
+    apply_motion_blur,
+    zoom_image_and_crop,
+    get_otsu_threshold,
+    color_level_adjustment,
+)
+
+
+class RainGenUsingNoise:
+    def genRainLayer(
+        self,
+        h,
+        w,
+        noise_scale=0.5,
+        noise_amount=0.25,
+        zoom_layer=2.0,
+        blur_kernel_size=15,
+        blur_angle=-60,
+    ):
+        layer = generate_noisy_image(h, w, sigma=noise_scale, p=noise_amount)
+
+        if blur_kernel_size > 0:
+            layer = apply_motion_blur(layer.copy(), blur_kernel_size, int(blur_angle))
+
+        if zoom_layer > 1:
+            layer = zoom_image_and_crop(layer.copy(), r=zoom_layer)
+
+        th = get_otsu_threshold(layer.copy())
+        layer = color_level_adjustment(
+            layer.copy(), inBlack=th, inWhite=th + 100, outWhite=250, inGamma=1.0
+        )
+        return layer
+
+
+class RainGenUsingMasks:
+    def __init__(self, mask_folder: str, ext="png"):
+        self._mask_path_list = sorted(Path(mask_folder).glob("*." + ext))
+
+    def genSingleLayer(self, scale=4, area=(10, 500), blur=False, rotate=0):
+        streak_file = random.choice(self._mask_path_list)
+        streak = cv2.cvtColor(cv2.imread(str(streak_file)), cv2.COLOR_BGR2GRAY)
+        hs, ws = streak.shape
+        if scale > 1:
+            streak = cv2.resize(streak, (int(ws * scale), int(hs * scale)))
+
+        if rotate != 0:
+            M = cv2.getRotationMatrix2D(
+                (int(ws * scale) / 2, int(hs * scale) / 2), rotate, 1
+            )
+            streak = cv2.warpAffine(streak, M, (int(ws * scale), int(hs * scale)))
+
+        binarized_streak = binarizeImage(streak)
+        mask = bwAreaFilter(binarized_streak, area_range=area)
+
+        # radius=2*ceil(2*sigma)+1
+        streak_masked = streak * mask
+        if blur:
+            streak_masked = scipy.ndimage.gaussian_filter(
+                streak_masked, sigma=1, mode="reflect", radius=5
+            )
+        return streak_masked
+
+    def genStreaks(
+        self,
+        reqH=720,
+        reqW=1280,
+        rotate=0,
+        num_itr=10,
+        scale=2,
+        area=(50, 150),
+        blur=False,
+        resize=False,
+        inGamma=1.0,
+    ):
+        layer = np.zeros((reqH, reqW))
+
+        blur_kernel_size = 3
+        blur_angle = np.random.randint(-60, 60)
+
+        for i in range(num_itr):
+            streak = self.genSingleLayer(scale=scale, area=area, rotate=rotate)
+            if blur:
+                streak = apply_motion_blur(
+                    streak.astype(float), blur_kernel_size, blur_angle
+                )
+            if resize:
+                streak = cv2.resize(streak.astype(float), (reqW, reqH))
+            streak = centreCrop(streak, reqH, reqW)
+            tr = random.random() * 0.2 + 0.25
+            layer = layer + streak * tr
+
+        layer = color_level_adjustment(
+            layer.copy(),
+            inBlack=10,
+            inWhite=100,
+            inGamma=inGamma,
+            outBlack=0,
+            outWhite=200,
+        )
+        return layer
+
+    def genRainEffect(self, intensnity):
+        rotate = random.randint(-30, 30)
+        if intensnity == "high":
+            layer_far = self.genStreaks(
+                reqH=720,
+                reqW=1280,
+                rotate=rotate,
+                num_itr=random.randint(40, 75),
+                scale=1,
+                area=(5, 150),
+                blur=False,
+                resize=False,
+                inGamma=1.0,
+            )
+            layer_close = self.genStreaks(
+                reqH=720,
+                reqW=1280,
+                rotate=rotate,
+                num_itr=random.randint(15, 30),
+                scale=1,
+                area=(150, 450),
+                blur=False,
+                resize=False,
+                inGamma=1.0,
+            )
+
+        if intensnity == "mod":
+            layer_far = self.genStreaks(
+                reqH=720,
+                reqW=1280,
+                rotate=rotate,
+                num_itr=random.randint(15, 25),
+                scale=1,
+                area=(75, 150),
+                blur=False,
+                resize=False,
+                inGamma=2.0,
+            )
+            layer_close = self.genStreaks(
+                reqH=720,
+                reqW=1280,
+                rotate=rotate,
+                num_itr=random.randint(4, 10),
+                scale=1,
+                area=(150, 500),
+                blur=False,
+                resize=False,
+                inGamma=2.0,
+            )
+        tr = random.random() * 0.2 + 0.25
+        layer = layer_far + layer_close * tr
+        return layer

lib/snow_gen.py

@@ -0,0 +1,83 @@
+import random
+import numpy as np
+from lib.gen_utils import (generate_noisy_image, zoom_image_and_crop, get_otsu_threshold,
+                           apply_motion_blur, color_level_adjustment, repeat_and_combine, crystallize,
+                           layer_blend)
+
+
+class SnowGenUsingNoise:
+    def __init__(self):
+        self._noise_scale_range = {
+            # 'small': (0.24, 0.45), 'large': (0.45, 0.65)
+            'small': (0.1, 0.2), 'large': (0.3, 0.5)
+            }
+        self._noise_amount_range = {
+            # 'small': (0.35, 0.65), 'large': (0.05, 0.15)
+            'small': (0.25, 0.45), 'large': (0.05, 0.15)
+            }
+        # self._zoom_range = {'small': (1.75, 3.0), 'large': (7, 10)}
+        self._zoom_range = {'small': (1.5, 2.0), 'large': (4, 6)}
+        self._blur_kernel_range = {'small': [3, 5, 7], 'large': [9, 11, 13]}
+        self._repeat_scale = {'small': [0], 'large': [0]}
+        self._max_level = {'small': (100, 150), 'large': (200, 250)}
+        self._cyrstalize_range = (0.55, 0.75)
+
+    def genSnowLayer(self,
+                     h,
+                     w,
+                     noise_scale=0.5,
+                     noise_amount=0.25,
+                     zoom_layer=2.0,
+                     blur_kernel_size=15,
+                     blur_angle=-60,
+                     max_level=250,
+                     compress_scale=0,
+                     cyrstalize_amount=0.5
+                     ):
+        im_noisy = generate_noisy_image(
+            h, w, sigma=noise_scale, p=noise_amount)
+        im_zoom = zoom_image_and_crop(im_noisy, r=zoom_layer)
+        im_blurr = apply_motion_blur(im_zoom, blur_kernel_size, blur_angle)
+
+        ret = get_otsu_threshold(im_blurr)
+        layer = color_level_adjustment(
+            im_blurr.copy(), inBlack=ret, inWhite=max_level, inGamma=1.0)
+
+        if compress_scale > 0:
+            layer = repeat_and_combine(layer, compress_scale)
+
+        if cyrstalize_amount > 0:
+            layer = crystallize(np.flipud(layer), r=0.75)
+
+        return layer.astype(np.uint8)
+
+    def genSnowMultiLayer(self, h, w, blur_angle=75, intensity="large", num_itr=2):
+        noise_scale_range = self._noise_scale_range[intensity]
+        noise_amount_range = self._noise_amount_range[intensity]
+        zoom_range = self._zoom_range[intensity]
+        blur_kernel_range = self._blur_kernel_range[intensity]
+        repeat_scale = self._repeat_scale[intensity][0]
+        max_level = self._max_level[intensity]
+
+        layer = np.zeros((h, w), dtype=np.uint8)
+        for _ in range(num_itr):
+            l = self.genSnowLayer(h, w,
+                                  noise_scale=random.uniform(
+                                      noise_scale_range[0], noise_scale_range[1]),
+                                  noise_amount=random.uniform(
+                                      noise_amount_range[0], noise_amount_range[1]),
+                                  zoom_layer=random.uniform(
+                                      zoom_range[0], zoom_range[1]),
+                                  blur_kernel_size=random.choice(
+                                      blur_kernel_range),
+                                  blur_angle=blur_angle,
+                                  max_level=random.randint(
+                                      max_level[0], max_level[1]),
+                                  compress_scale=repeat_scale,
+                                  cyrstalize_amount=random.uniform(
+                                      self._cyrstalize_range[0], self._cyrstalize_range[1])
+                                  )
+            # tr = 0.25 + random.random()*0.5
+            # layer = layer + tr*l  
+            layer = layer_blend(layer, l)
+        return layer

lib/style_transfer_utils.py

@@ -0,0 +1,239 @@
+import torch
+import torch.nn as nn
+from PIL import Image
+from tqdm.auto import tqdm
+import torch.nn.functional as F
+import torchvision.models as models
+import torchvision.transforms as transforms
+
+
+def pil2tensor(pil: Image) -> torch.Tensor:
+    return transforms.functional.to_tensor(pil)
+
+
+def tensor2pil(tensor: torch.Tensor) -> Image:
+    return transforms.functional.to_pil_image(tensor)
+
+
+def load_style_transfer_model(pretrained: str = None) -> nn.Module:
+    if pretrained:
+        print(f"Loading VGG with {pretrained} weights.")
+        cnn = models.vgg19(weights=None).features
+        state_dict = torch.load(pretrained)
+        state_dict = {
+            k.replace("features.", ""): v
+            for k, v in state_dict.items()
+            if "features" in k
+        }
+        cnn.load_state_dict(state_dict)
+    else:
+        print(f"Loading VGG with IMAGENET1K weights.")
+        cnn = models.vgg19(weights=models.VGG19_Weights.IMAGENET1K_V1).features
+    cnn.eval()
+    return cnn
+
+
+def style_content_image_loader(content_path, style_path):
+    wreq = 640
+
+    content_img = Image.open(content_path)
+    wc, hc = content_img.size
+    wc_new, hc_new = wreq, int(hc * wreq / wc)
+    content_img = content_img.resize((wc_new, hc_new))
+
+    style_img = Image.open(style_path)
+    ws, hs = style_img.size
+
+    ws_new = wreq
+    hs_new = int(hs * ws_new / ws)
+
+    if hs_new < hc_new:
+        hs_new = hc_new
+
+    style_img = style_img.resize((ws_new, hs_new))
+
+    if hs_new > hc_new:
+        top = int((hs_new - hc_new) * 0.5)
+        bottom = top + hc_new
+        style_img = style_img.crop((0, top, ws_new, bottom))
+
+    assert style_img.size == content_img.size
+
+    style_img = pil2tensor(style_img).unsqueeze(0)
+    content_img = pil2tensor(content_img).unsqueeze(0)
+    return content_img, style_img
+
+
+class ContentLoss(nn.Module):
+    def __init__(
+        self,
+        target,
+    ):
+        super(ContentLoss, self).__init__()
+        # we 'detach' the target content from the tree used
+        # to dynamically compute the gradient: this is a stated value,
+        # not a variable. Otherwise the forward method of the criterion
+        # will throw an error.
+        # self.target = target.detach()
+        self.register_buffer("target", target.detach())
+
+    def forward(self, input):
+        self.loss = F.mse_loss(input, self.target)
+        return input
+
+
+def gram_matrix(input):
+    a, b, c, d = input.size()  # a=batch size(=1)
+    # b=number of feature maps
+    # (c,d)=dimensions of a f. map (N=c*d)
+
+    features = input.view(a * b, c * d)  # resise F_XL into \hat F_XL
+
+    G = torch.mm(features, features.t())  # compute the gram product
+
+    # we 'normalize' the values of the gram matrix
+    # by dividing by the number of element in each feature maps.
+    return G.div(a * b * c * d)
+
+
+class StyleLoss(nn.Module):
+    def __init__(self, target_feature):
+        super(StyleLoss, self).__init__()
+        # self.target = gram_matrix(target_feature).detach()
+        self.register_buffer("target", gram_matrix(target_feature).detach())
+
+    def forward(self, input):
+        G = gram_matrix(input)
+        sup = ((G**2).sum() + self.target.sum()) / input.numel()
+        self.loss = F.mse_loss(G, self.target) / sup
+        return input
+
+
+def get_style_model_and_losses(cnn, style_img, content_img, device="cpu"):
+    # desired depth layers to compute style/content losses :
+    content_layers = ["conv_4"]
+    style_layers = ["conv_1", "conv_2", "conv_3", "conv_4", "conv_5"]
+
+    # just in order to have an iterable access to or list of content/syle losses
+    content_losses = []
+    style_losses = []
+
+    # assuming that cnn is a nn.Sequential, so we make a new nn.Sequential
+    # to put in modules that are supposed to be activated sequentially
+    mean = torch.tensor([0.485, 0.456, 0.406])
+    std = torch.tensor([0.229, 0.224, 0.225])
+    model = nn.Sequential(transforms.Normalize(mean=mean, std=std))
+
+    i = 0  # increment every time we see a conv
+    for layer in cnn.children():
+        if isinstance(layer, nn.Conv2d):
+            i += 1
+            name = "conv_{}".format(i)
+        elif isinstance(layer, nn.ReLU):
+            name = "relu_{}".format(i)
+            # The in-place version doesn't play very nicely with the ContentLoss
+            # and StyleLoss we insert below. So we replace with out-of-place
+            # ones here.
+            layer = nn.ReLU(inplace=False)
+        elif isinstance(layer, nn.MaxPool2d):
+            name = "pool_{}".format(i)
+        elif isinstance(layer, nn.BatchNorm2d):
+            name = "bn_{}".format(i)
+        else:
+            raise RuntimeError(
+                "Unrecognized layer: {}".format(layer.__class__.__name__)
+            )
+
+        model.add_module(name, layer)
+
+        if name in content_layers:
+            # add content loss:
+            target = model(content_img).detach()
+            content_loss = ContentLoss(target)
+            model.add_module("content_loss_{}".format(i), content_loss)
+            content_losses.append(content_loss)
+
+        if name in style_layers:
+            # add style loss:
+            target_feature = model(style_img).detach()
+            style_loss = StyleLoss(target_feature)
+            model.add_module("style_loss_{}".format(i), style_loss)
+            style_losses.append(style_loss)
+
+    # now we trim off the layers after the last content and style losses
+    for i in range(len(model) - 1, -1, -1):
+        if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):
+            break
+
+    model = model[: (i + 1)]
+    for sl in style_losses:
+        sl.to(device)
+    for sl in content_losses:
+        sl.to(device)
+    return model.to(device), style_losses, content_losses
+
+
+def get_input_optimizer(input_img):
+    # this line to show that input is a parameter that requires a gradient
+    optimizer = torch.optim.LBFGS([input_img], lr=1)  # , lr=1e-2
+    return optimizer
+
+
+def run_style_transfer(
+    cnn,
+    content_img,
+    style_img,
+    input_img,
+    num_steps=300,
+    style_weight=1000000,
+    content_weight=1,
+    device="cpu",
+):
+    """Run the style transfer."""
+    # print('Building the style transfer model..')
+    model, style_losses, content_losses = get_style_model_and_losses(
+        cnn, style_img, content_img, device=device
+    )
+
+    # We want to optimize the input and not the model parameters so we
+    # update all the requires_grad fields accordingly
+    input_img.requires_grad_(True)
+    model.requires_grad_(False)
+
+    optimizer = get_input_optimizer(input_img)
+
+    for run in tqdm(range(num_steps)):
+
+        def closure():
+            # correct the values of updated input image
+            with torch.no_grad():
+                input_img.clamp_(0, 1)
+
+            optimizer.zero_grad()
+            model(input_img)
+            style_score = 0
+            content_score = 0
+
+            for sl in style_losses:
+                style_score += sl.loss
+            for cl in content_losses:
+                content_score += cl.loss
+
+            style_score *= style_weight
+            content_score *= content_weight
+
+            print(
+                f"Style Loss: {style_score.item()} Content Loss: {content_score.item()}"
+            )
+
+            loss = style_score + content_score
+            loss.backward()
+            return style_score + content_score
+
+        optimizer.step(closure)
+
+    # a last correction...
+    with torch.no_grad():
+        input_img.clamp_(0, 1)
+
+    return input_img

presentation.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa13d18e69b36e587c5b0bd4509954fddbf3fef9bf8c8918db9d9265ffa558e9
+size 12779382

requirements.txt

@@ -0,0 +1,6 @@
+numpy
+matplotlib
+opencv-python
+scikit-image
+scikit-learn
+torchsummary