init app

InstanceNorm.py

@@ -0,0 +1,146 @@
+from keras.engine import Layer, InputSpec
+from keras import initializers, regularizers, constraints
+from keras import backend as K
+from keras.utils.generic_utils import get_custom_objects
+
+import tensorflow as tf
+
+
+class InstanceNormalization(Layer):
+ """Instance normalization layer (Lei Ba et al, 2016, Ulyanov et al., 2016).
+ Normalize the activations of the previous layer at each step,
+ i.e. applies a transformation that maintains the mean activation
+ close to 0 and the activation standard deviation close to 1.
+ # Arguments
+ axis: Integer, the axis that should be normalized
+ (typically the features axis).
+ For instance, after a `Conv2D` layer with
+ `data_format="channels_first"`,
+ set `axis=1` in `InstanceNormalization`.
+ Setting `axis=None` will normalize all values in each instance of the batch.
+ Axis 0 is the batch dimension. `axis` cannot be set to 0 to avoid errors.
+ epsilon: Small float added to variance to avoid dividing by zero.
+ center: If True, add offset of `beta` to normalized tensor.
+ If False, `beta` is ignored.
+ scale: If True, multiply by `gamma`.
+ If False, `gamma` is not used.
+ When the next layer is linear (also e.g. `nn.relu`),
+ this can be disabled since the scaling
+ will be done by the next layer.
+ beta_initializer: Initializer for the beta weight.
+ gamma_initializer: Initializer for the gamma weight.
+ beta_regularizer: Optional regularizer for the beta weight.
+ gamma_regularizer: Optional regularizer for the gamma weight.
+ beta_constraint: Optional constraint for the beta weight.
+ gamma_constraint: Optional constraint for the gamma weight.
+ # Input shape
+ Arbitrary. Use the keyword argument `input_shape`
+ (tuple of integers, does not include the samples axis)
+ when using this layer as the first layer in a model.
+ # Output shape
+ Same shape as input.
+ # References
+ - [Layer Normalization](https://arxiv.org/abs/1607.06450)
+ - [Instance Normalization: The Missing Ingredient for Fast Stylization](https://arxiv.org/abs/1607.08022)
+ """
+ def __init__(self,
+ axis=None,
+ epsilon=1e-3,
+ center=True,
+ scale=True,
+ beta_initializer='zeros',
+ gamma_initializer='ones',
+ beta_regularizer=None,
+ gamma_regularizer=None,
+ beta_constraint=None,
+ gamma_constraint=None,
+ **kwargs):
+ super(InstanceNormalization, self).__init__(**kwargs)
+ self.supports_masking = True
+ self.axis = axis
+ self.epsilon = epsilon
+ self.center = center
+ self.scale = scale
+ self.beta_initializer = initializers.get(beta_initializer)
+ self.gamma_initializer = initializers.get(gamma_initializer)
+ self.beta_regularizer = regularizers.get(beta_regularizer)
+ self.gamma_regularizer = regularizers.get(gamma_regularizer)
+ self.beta_constraint = constraints.get(beta_constraint)
+ self.gamma_constraint = constraints.get(gamma_constraint)
+
+ def build(self, input_shape):
+ ndim = len(input_shape)
+ if self.axis == 0:
+ raise ValueError('Axis cannot be zero')
+
+ if (self.axis is not None) and (ndim == 2):
+ raise ValueError('Cannot specify axis for rank 1 tensor')
+
+ self.input_spec = InputSpec(ndim=ndim)
+
+ if self.axis is None:
+ shape = (1,)
+ else:
+ shape = (input_shape[self.axis],)
+
+ if self.scale:
+ self.gamma = self.add_weight(shape=shape,
+ name='gamma',
+ initializer=self.gamma_initializer,
+ regularizer=self.gamma_regularizer,
+ constraint=self.gamma_constraint)
+ else:
+ self.gamma = None
+ if self.center:
+ self.beta = self.add_weight(shape=shape,
+ name='beta',
+ initializer=self.beta_initializer,
+ regularizer=self.beta_regularizer,
+ constraint=self.beta_constraint)
+ else:
+ self.beta = None
+ self.built = True
+
+ def call(self, inputs, training=None):
+ input_shape = K.int_shape(inputs)
+ reduction_axes = list(range(0, len(input_shape)))
+
+ if (self.axis is not None):
+ del reduction_axes[self.axis]
+
+ del reduction_axes[0]
+
+ mean, var = tf.nn.moments(inputs, reduction_axes, keep_dims=True)
+ stddev = tf.sqrt(var) + self.epsilon
+ normed = (inputs - mean) / stddev
+
+ broadcast_shape = [1] * len(input_shape)
+ if self.axis is not None:
+ broadcast_shape[self.axis] = input_shape[self.axis]
+
+ if self.scale:
+ broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
+ normed = normed * broadcast_gamma
+ if self.center:
+ broadcast_beta = K.reshape(self.beta, broadcast_shape)
+ normed = normed + broadcast_beta
+ return normed
+
+ def get_config(self):
+ config = {
+ 'axis': self.axis,
+ 'epsilon': self.epsilon,
+ 'center': self.center,
+ 'scale': self.scale,
+ 'beta_initializer': initializers.serialize(self.beta_initializer),
+ 'gamma_initializer': initializers.serialize(self.gamma_initializer),
+ 'beta_regularizer': regularizers.serialize(self.beta_regularizer),
+ 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
+ 'beta_constraint': constraints.serialize(self.beta_constraint),
+ 'gamma_constraint': constraints.serialize(self.gamma_constraint)
+ }
+ base_config = super(InstanceNormalization, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+
+get_custom_objects().update({'InstanceNormalization': InstanceNormalization})

ai.py

@@ -0,0 +1,203 @@
+import tensorflow
+
+tensorflow.compat.v1.disable_v2_behavior()
+tf = tensorflow.compat.v1
+
+import keras
+import numpy as np
+from config import *
+from keras.models import load_model
+from smoother import *
+import keras.backend as K
+from models import *
+
+
+def ToGray(x):
+ R = x[:, :, :, 0:1]
+ G = x[:, :, :, 1:2]
+ B = x[:, :, :, 2:3]
+ return 0.30 * R + 0.59 * G + 0.11 * B
+
+
+def RGB2YUV(x):
+ R = x[:, :, :, 0:1]
+ G = x[:, :, :, 1:2]
+ B = x[:, :, :, 2:3]
+ Y = 0.299 * R + 0.587 * G + 0.114 * B
+ U = 0.492 * (B - Y) + 128
+ V = 0.877 * (R - Y) + 128
+ return tf.concat([Y, U, V], axis=3)
+
+
+def YUV2RGB(x):
+ Y = x[:, :, :, 0:1]
+ U = x[:, :, :, 1:2]
+ V = x[:, :, :, 2:3]
+ R = Y + 1.140 * (V - 128)
+ G = Y - 0.394 * (U - 128) - 0.581 * (V - 128)
+ B = Y + 2.032 * (U - 128)
+ return tf.concat([R, G, B], axis=3)
+
+
+def VGG2RGB(x):
+ return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1]
+
+
+def blur(x):
+ return Smoother({'data': tf.pad(x, [[0, 0], [9, 9], [9, 9], [0, 0]], 'SYMMETRIC')}, 7, 2).get_output()[:, 9: -9, 9: -9, :]
+
+
+def norm_feature(x, core):
+ cs0 = tf.shape(core)[1]
+ cs1 = tf.shape(core)[2]
+ small = tf.image.resize_area(x, (cs0, cs1))
+ avged = tf.nn.avg_pool(tf.pad(small, [[0, 0], [2, 2], [2, 2], [0, 0]], 'REFLECT'), [1, 5, 5, 1], [1, 1, 1, 1], 'VALID')
+ return tf.image.resize_bicubic(avged, tf.shape(x)[1:3])
+
+
+def upsample(x):
+ return K.resize_images(x, 2, 2, 'channels_last')
+
+
+def downsample(x):
+ return tf.nn.avg_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+
+
+def nts(x):
+ return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1] / 255.0
+
+
+session = keras.backend.get_session()
+
+ip1 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 1))
+ip3 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 3))
+ip4 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 4))
+
+print('1')
+
+vector = make_diff_net()
+vector_op = 255.0 - tf.nn.sigmoid(vector(ip3 / 255.0)) * 255.0
+
+print('4')
+
+reader = load_model('./nets/reader.net')
+features = reader(ip3 / 255.0)
+
+print('5')
+
+head = load_model('./nets/head.net')
+feed = [1 - ip1 / 255.0, (ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0]
+for _ in range(len(features)):
+ feed.append(keras.backend.mean(features[_], axis=[1, 2]))
+nil0, nil1, head_temp = head(feed)
+
+print('6')
+
+neck = load_model('./nets/neck.net')
+nil2, nil3, neck_temp = neck(feed)
+feed[0] = tf.clip_by_value(1 - tf.image.resize_bilinear(ToGray(VGG2RGB(head_temp) / 255.0), tf.shape(ip1)[1:3]), 0.0, 1.0)
+nil4, nil5, head_temp = neck(feed)
+head_op = VGG2RGB(head_temp)
+neck_op = VGG2RGB(neck_temp)
+
+print('7')
+
+inception = load_model('./nets/inception.net')
+features_render = inception((ip3 + (downsample(ip1) - blur(downsample(ip1))) * 2.0) / 255.0)
+precessed_feed = [(ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0] + [
+ norm_feature(item, features_render[-1]) for item in features_render]
+
+print('8')
+
+render_head = load_model('./nets/render_head.net')
+render_neck = load_model('./nets/render_neck.net')
+nil6, nil7, render_A = render_head([1 - ip1 / 255.0] + precessed_feed)
+nil8, nil9, render_B = render_neck(
+ [1 - tf.image.resize_bilinear(ToGray(nts(render_A)), tf.shape(ip1)[1:3])] + precessed_feed)
+render_op = nts(render_B) * 255.0
+
+print('9')
+
+tail = load_model('./nets/tail.net')
+pads = 7
+tail_op = tail(tf.pad(ip3 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads * 2:-pads * 2, pads * 2:-pads * 2, :][:, 1:-1, 1:-1, :] * 255.0
+
+print('10')
+
+
+vgg7 = load_model('./nets/vgg7.net')
+pads = 7
+vgg7_op = vgg7(tf.pad(ip1 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads:-pads, pads:-pads, :] * 255.0
+
+print('11')
+
+
+mat = make_unet512()
+mat_op = mat(ip3 / 255.0) * 255.0
+
+print('11')
+
+norm = load_model('./nets/norm.net')
+norm_op = norm(ip1 / 255.0) * 255.0
+
+print('12')
+
+session.run(tf.global_variables_initializer())
+
+print('begin load')
+
+
+tail.load_weights('./nets/tail.net')
+vgg7.load_weights('./nets/vgg7.net')
+head.load_weights('./nets/head.net')
+neck.load_weights('./nets/neck.net')
+reader.load_weights('./nets/reader.net')
+vector.load_weights('./nets/vector.net')
+render_head.load_weights('./nets/render_head.net')
+render_neck.load_weights('./nets/render_neck.net')
+inception.load_weights('./nets/inception.net')
+mat.load_weights('./nets/mat.net')
+norm.load_weights('./nets/norm.net')
+
+
+def go_head(sketch, global_hint, local_hint):
+ return session.run(head_op, feed_dict={
+ ip1: sketch[None, :, :, None], ip3: global_hint[None, :, :, :], ip4: local_hint[None, :, :, :]
+ })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_render(sketch, segmentation, points):
+ return session.run(render_op, feed_dict={
+ ip1: sketch[None, :, :, None], ip3: segmentation[None, :, :, :], ip4: points[None, :, :, :]
+ })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_tail(x):
+ return session.run(tail_op, feed_dict={
+ ip3: x[None, :, :, :]
+ })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_vgg7(x):
+ return session.run(vgg7_op, feed_dict={
+ ip1: x[None, :, :, None]
+ })[0, :, :, 0].clip(0, 255).astype(np.uint8)
+
+
+def go_vector(x):
+ return session.run(vector_op, feed_dict={
+ ip3: x[None, :, :, :]
+ })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_mat(x):
+ return session.run(mat_op, feed_dict={
+ ip3: x[None, :, :, :]
+ })[0, :, :, 0].clip(0, 255).astype(np.uint8)
+
+
+def go_norm(x):
+ return session.run(norm_op, feed_dict={
+ ip1: x[None, :, :, None]
+ })[0].clip(0, 255).astype(np.uint8)
+

app.py

@@ -0,0 +1,304 @@
+import re
+import base64
+import datetime
+import sys
+import pickle
+import gzip
+import json
+import time
+import gradio as gr
+
+from ai import *
+from tricks import *
+from decompositioner import *
+from rendering import *
+
+
+import os
+import glob
+import shutil
+
+splash = glob.glob('ui/web-mobile/splash*')[0]
+os.remove(splash)
+shutil.copy('res/splash.png', splash)
+with open('ui/web-mobile/index.html', 'r', encoding='utf-8') as f:
+ page = f.read()
+with open('ui/web-mobile/index.html', 'w', encoding='utf-8') as f:
+ f.write(page.replace('Cocos Creator | ', ''))
+
+
+def cv2_encode(image: np.ndarray, name):
+ if image is None:
+ return 'null'
+ if '.jpg' in name:
+ _, data = cv2.imencode('.jpeg', image)
+ return 'data:image/jpeg;base64,' + base64.b64encode(data).decode('utf8')
+ else:
+ _, data = cv2.imencode('.png', image)
+ return 'data:image/png;base64,' + base64.b64encode(data).decode('utf8')
+
+
+def get_request_image(request, name):
+ img = request.get(name)
+ img = re.sub('^data:image/.+;base64,', '', img)
+ img = base64.b64decode(img)
+ img = np.fromstring(img, dtype=np.uint8)
+ img = cv2.imdecode(img, -1)
+ return img
+
+
+def npcache(history, room_id, name, data):
+ rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+ if len(rooms) == 0:
+ room = { "id": room_id }
+ history.append(room)
+ else:
+ room = rooms[0]
+ room[name] = data
+
+
+def npread(history, room_id, name):
+ rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+ if len(rooms) == 0:
+ return None
+ else:
+ room = rooms[0]
+ return room.get(name, None)
+
+
+def upload_sketch(json_str, history):
+ request = json.loads(json_str)
+ timenow = time.time()
+ ID = datetime.datetime.now().strftime('H%HM%MS%S')
+ room = datetime.datetime.now().strftime('%b%dH%HM%MS%S') + 'R' + str(np.random.randint(100, 999))
+ sketch = from_png_to_jpg(get_request_image(request, 'sketch'))
+ npcache(history, room, 'sketch.original.jpg', sketch)
+ sketch = go_tail(cli_norm(min_resize(sketch, 512)))
+ print('original_sketch saved')
+ s256 = go_vector(go_cal(mk_resize(sketch, 8)))[:, :, 0]
+ print('s256')
+ s512 = go_vector(go_cal(d_resize(sketch, s256.shape, 2.0)))[:, :, 0]
+ print('s512')
+ s1024 = go_vector(go_cal(d_resize(sketch, s256.shape, 4.0)))[:, :, 0]
+ print('s1024')
+ npcache(history, room, 'sketch.s1024.png', s1024)
+ npcache(history, room, 'sketch.s512.png', s512)
+ npcache(history, room, 'sketch.s256.png', s256)
+ print('edge processed')
+ fill = double_fill(s1024, s512, s256)
+ npcache(history, room, 'sketch.fill', fill)
+ print('filled')
+ npcache(history, room, 'sketch.colorization.png', np.min(sketch, axis=2))
+ print('sketch processed')
+ print(time.time() - timenow)
+ if len(history) > 5:
+ history = history[-5:]
+ return room + '_' + ID, history
+
+
+def request_result(json_str, history):
+ request = json.loads(json_str)
+ timenow = time.time()
+ room = request.get("room")
+ if len(list(filter(lambda _room: _room["id"] == room, history))) == 0:
+ return None, history
+ skipper = str(request.get("skipper"))
+ light_r = float(request.get("r"))
+ light_g = float(request.get("g"))
+ light_b = float(request.get("b"))
+ light_h = float(request.get("h"))
+ light_max = max([light_r, light_g, light_b, light_h])
+ inv4 = int(request.get("inv4"))
+ print('inv4=' + str(inv4))
+ light_r = (light_r + 1e-5) / (light_max + 1e-5)
+ light_g = (light_g + 1e-5) / (light_max + 1e-5)
+ light_b = (light_b + 1e-5) / (light_max + 1e-5)
+ light_h *= 600.0
+ light_d = request.get("d")
+ need_render = int(request.get("need_render"))
+ print([light_r, light_g, light_b, light_d])
+ ID = datetime.datetime.now().strftime('H%HM%MS%S')
+ points = request.get("points")
+ points = json.loads(points)
+ npcache(history, room, 'points.' + ID + '.txt', points)
+ if len(points) > 500:
+ return None, history
+ for _ in range(len(points)):
+ points[_][1] = 1 - points[_][1]
+ fill = npread(history, room, 'sketch.fill')
+ s1024 = npread(history, room, 'sketch.s1024.png')
+ sketch = npread(history, room, 'sketch.colorization.png')
+ print(skipper)
+ if npread(history, room, 'albedo.' + skipper + '.png') is not None:
+ albedo = npread(history, room, 'albedo.' + skipper + '.png')
+ npcache(history, room, 'albedo.' + ID + '.png', albedo)
+ print('albedo readed')
+ else:
+ faceID = int(request.get("faceID")) - 65535
+ print(faceID)
+ if faceID > -1:
+ print('fake ref')
+ face = from_png_to_jpg(cv2.imread("refs/" + str(faceID + 1) + ".png", cv2.IMREAD_UNCHANGED))
+ else:
+ print('get ref')
+ face = from_png_to_jpg(get_request_image(request, 'face'))
+ npcache(history, room, 'face.' + ID + '.jpg', face)
+ face = s_enhance(face)
+ print('request result room = ' + str(room) + ', ID = ' + str(ID))
+ print('processing painting in ' + room)
+ if inv4 > 0:
+ sketch_1024 = k_resize(sketch, 64)
+ else:
+ sketch_1024 = k_resize(sketch, 48)
+ hints_1024 = ini_hint(sketch_1024)
+ hints_1024 = opreate_normal_hint(hints_1024, points, length=2, skip_sp=True)
+ baby = go_head(
+ sketch=sketch_1024,
+ global_hint=k_resize(face, 14),
+ local_hint=hints_1024
+ )
+ npcache(history, room, 'baby.' + ID + '.jpg', baby)
+ print('baby born')
+ composition = d_resize(re_deatlize(deatlize(balance_fill(baby, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=True), s1024)), s1024), sketch.shape)
+ npcache(history, room, 'composition.' + ID + '.jpg', composition)
+ gird = process_overlay(composition, sketch)
+ npcache(history, room, 'gird.' + ID + '.jpg', gird)
+ print('composition saved')
+ if inv4 > 0:
+ albedo = go_render(sketch_1024, d_resize(composition, sketch_1024.shape, 0.5), hints_1024)
+ albedo = go_tail(albedo)
+ albedo = d_resize(re_deatlize(d_resize(albedo, s1024.shape), s1024), sketch.shape)
+ albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2YUV)
+ albedo[:, :, 0] = go_vgg7(albedo[:, :, 0])
+ albedo = cv2.cvtColor(albedo, cv2.COLOR_YUV2RGB)
+ else:
+ albedo = re_deatlize(d_resize(baby, s1024.shape), s1024)
+ albedo = d_resize(albedo, sketch.shape, 0.25)
+ albedo = go_tail(albedo)
+ albedo = go_tail(albedo)
+ albedo = d_resize(albedo, sketch.shape)
+ boundary = sketch.astype(np.float32)
+ boundary = cv2.GaussianBlur(boundary, (0, 0), 1.618) - boundary
+ boundary = boundary.clip(0, 255)
+ albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2HSV).astype(np.float32)
+ albedo[:, :, 1] += albedo[:, :, 1] * boundary / 48.0
+ albedo[:, :, 2] -= boundary
+ albedo = cv2.cvtColor(albedo.clip(0, 255).astype(np.uint8), cv2.COLOR_HSV2RGB)
+ npcache(history, room, 'albedo.' + ID + '.png', albedo)
+ print('albedo saved')
+ if need_render == 0:
+ npcache(history, room, 'result.' + ID + '.jpg', albedo)
+ # cv2.imwrite('results/' + room + '.' + ID + '.jpg', albedo)
+ print(time.time() - timenow)
+ return room + '_' + ID, history
+ HSV, YUV, DEL = process_albedo(albedo, composition, sketch)
+ npcache(history, room, 'HSV.' + ID + '.jpg', HSV)
+ npcache(history, room, 'YUV.' + ID + '.jpg', YUV)
+ npcache(history, room, 'DEL.' + ID + '.jpg', DEL)
+ print('HSV YUV DEL')
+ albedo_s1024 = d_resize(albedo, s1024.shape)
+ matting = go_mat(albedo_s1024)
+ matting = np.tile(matting[:, :, None], [1, 1, 3])
+ matting = shade_fill(matting, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=False), s1024)
+ matting = matting[:, :, 0]
+ depth = np.zeros_like(matting, dtype=np.uint8) + 255
+ depth[matting < 127] = 127
+ depth[s1024 < 250] = 0
+ npcache(history, room, 'depth.' + ID + '.jpg', depth)
+ print('depth saved')
+ normal = go_norm(depth).astype(np.float32)
+ normal = ((normal + 1e-4) / (np.max(normal, axis=2, keepdims=True) + 1e-4) * 255.0).clip(0, 255).astype(np.uint8)
+ normal[matting < 127] = 255
+ normal = re_deatlize(normal, s1024)
+ normal = d_resize(normal, sketch.shape)
+ npcache(history, room, 'normal.' + ID + '.jpg', normal)
+ print('norm saved')
+ mask = np.zeros_like(matting, dtype=np.uint8) + 255
+ mask[matting < 127] = 0
+ mask = d_resize(mask, sketch.shape)
+ mask[mask < 127] = 0
+ mask[mask > 0] = 255
+ if int(light_d) == 0:
+ result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=True)
+ elif int(light_d) == 1:
+ result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=True)
+ elif int(light_d) == 2:
+ result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=False)
+ else:
+ result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=False)
+ if need_render == 2:
+ npcache(history, room, 'result.' + ID + '.jpg', result)
+ # cv2.imwrite('results/' + room + '.' + ID + '.jpg', result)
+ print(time.time() - timenow)
+ return room + '_' + ID, history
+ print('result saved')
+ preview = np.concatenate([np.tile(sketch[:, :, None], [1, 1, 3]), albedo, result], axis=1)
+ npcache(history, room, 'preview.' + ID + '.jpg', preview)
+ print('preview saved')
+ npcache(history, room, 'result.' + ID + '.jpg', result)
+ # cv2.imwrite('results/' + room + '.' + ID + '.jpg', preview)
+ print(time.time() - timenow)
+ return room + '_' + ID, history
+
+
+def download_result(name, history):
+ room_id, name = name.split('/')
+ rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+ if len(rooms) == 0:
+ return None
+ else:
+ room = rooms[0]
+
+ real_name = None
+ for k in room.keys():
+ if name in k:
+ real_name = k
+ break
+ if real_name is None:
+ return None
+ name = real_name
+ 
+ result = room.get(name, None)
+
+ if 'points' in name:
+ return json.dumps(result)
+ 
+ return cv2_encode(result, name)
+ 
+
+with gr.Blocks() as demo:
+ history = gr.State(value=[])
+ with gr.Row():
+ with gr.Column():
+ btn_show = gr.Button("Open Style2Paints V4.2")
+ btn_show.click(None, _js="(_) => open('file/ui/web-mobile/index.html')")
+
+ with gr.Row():
+ with gr.Box():
+ with gr.Row():
+ upload_sketch_json = gr.Textbox(label="upload_sketch(json string)")
+ with gr.Row():
+ upload_sketch_btn = gr.Button(label="Submit sketch json")
+ with gr.Row():
+ upload_sketch_result = gr.Textbox(label="Result", interactive=False)
+ upload_sketch_btn.click(upload_sketch, [upload_sketch_json, history], [upload_sketch_result, history], api_name="upload_sketch")
+ 
+ with gr.Box():
+ with gr.Row():
+ request_result_json = gr.Textbox(label="request_result(json string)")
+ with gr.Row():
+ request_result_btn = gr.Button(label="Submit json of request for result")
+ with gr.Row():
+ request_result_result = gr.Textbox(label="Result", interactive=False)
+ upload_sketch_btn.click(request_result, [request_result_json, history], [request_result_result, history], api_name="request_result")
+ 
+ with gr.Box():
+ with gr.Row():
+ download_result_json = gr.Textbox(label="download_result(json string)")
+ with gr.Row():
+ download_result_btn = gr.Button(label="Submit json of download for result")
+ with gr.Row():
+ download_result_result = gr.Textbox(label="Result", interactive=False)
+ upload_sketch_btn.click(download_result, [download_result_json, history], [download_result_result], api_name="download_result")
+
+demo.launch()

config.py

@@ -0,0 +1,2 @@
+multiple_process = True
+

decompositioner.py

@@ -0,0 +1,185 @@
+import os
+import numpy as np
+from scipy.spatial import ConvexHull
+from sklearn.cluster import MiniBatchKMeans
+from tricks import *
+import cv2
+
+
+ksd = 8
+mbc = MiniBatchKMeans(ksd)
+
+
+def get_theme(img):
+ images = np.reshape(cv2.resize(img, (256, 256)), (256 * 256, 3))
+ hull = ConvexHull(images)
+ return hull.points[hull.vertices]
+
+
+def simplify_points(points, img):
+ labels = mbc.fit(points)
+ new_points = []
+ all_center = np.mean(labels.cluster_centers_, axis=0)
+ distances = np.sum((points - all_center) ** 2, axis=1) ** 0.5
+
+ for idx in range(ksd):
+ candidates = points[labels.labels_ == idx]
+ scores = distances[labels.labels_ == idx]
+ best_id = np.argmax(scores)
+ new_points.append(candidates[best_id])
+
+ new_points.sort(key=np.sum, reverse=True)
+
+ new_points = np.stack(new_points, axis=0)
+ return new_points.clip(0, 255).astype(np.uint8)
+
+
+def get_ini_layers(miku, points):
+ results = []
+ final_target = miku.astype(np.float32)
+ bg = np.zeros_like(final_target, dtype=np.float32) + points[0]
+ results.append(np.concatenate([bg, np.zeros_like(bg, dtype=np.float32) + 255], axis=2)[:, :, 0:4])
+ current_result = bg.copy()
+ for layer_index in range(1, ksd):
+ current_base = current_result.astype(np.float32)
+ current_color = np.zeros_like(final_target, dtype=np.float32) + points[layer_index]
+ overall_direction = final_target - current_base
+ avaliable_direction = current_color - current_base
+ current_alpha = np.sum(overall_direction * avaliable_direction, axis=2, keepdims=True) / np.sum(
+ avaliable_direction * avaliable_direction, axis=2, keepdims=True)
+ current_alpha = current_alpha.clip(0, 1)
+ current_result = (current_color * current_alpha + current_base * (1 - current_alpha)).clip(0, 255)
+ results.append(np.concatenate([current_color, current_alpha * 255.0], axis=2))
+ return results
+
+
+def make_reconstruction(layers):
+ bg = np.zeros_like(layers[0], dtype=np.float32)[:, :, 0:3] + 255
+ for item in layers:
+ current_alpha = item[:, :, 3:4] / 255.0
+ bg = item[:, :, 0:3] * current_alpha + bg * (1 - current_alpha)
+ return bg
+
+
+def improve_layers(layers, miku):
+ reconstruction = make_reconstruction(layers)
+ b = miku - reconstruction
+ new_layers = []
+ for item in layers:
+ new_item = item.copy()
+ new_item[:, :, 0:3] = (new_item[:, :, 0:3] + b).clip(0, 255)
+ new_layers.append(new_item)
+ return new_layers
+
+
+def cluster_all(labeled_array, num_features):
+ xs = [[] for _ in range(num_features)]
+ ys = [[] for _ in range(num_features)]
+ M = labeled_array.shape[0]
+ N = labeled_array.shape[1]
+ for x in range(M):
+ for y in range(N):
+ i = labeled_array[x, y]
+ xs[i].append(x)
+ ys[i].append(y)
+ result = []
+ for _ in range(num_features):
+ result.append((np.array(xs[_]), np.array(ys[_])))
+ return result
+
+
+def meder(x):
+ y = x.copy()
+ y = cv2.medianBlur(y, 5)
+ y = cv2.medianBlur(y, 5)
+ y = cv2.medianBlur(y, 3)
+ y = cv2.medianBlur(y, 3)
+ return y
+
+
+def re_med(s_2048):
+
+ sample_2048 = s_2048.astype(np.float32)
+ sample_1024 = cv2.pyrDown(sample_2048)
+ sample_512 = cv2.pyrDown(sample_1024)
+ sample_256 = cv2.pyrDown(sample_512)
+
+ gradient_2048 = sample_2048 - cv2.pyrUp(sample_1024)
+ gradient_1024 = sample_1024 - cv2.pyrUp(sample_512)
+ gradient_512 = sample_512 - cv2.pyrUp(sample_256)
+
+ rec_256 = meder(sample_256)
+ rec_512 = cv2.pyrUp(rec_256) + meder(gradient_512)
+ rec_1024 = cv2.pyrUp(rec_512) + meder(gradient_1024)
+ rec_2048 = cv2.pyrUp(rec_1024) + meder(gradient_2048)
+ return rec_2048
+
+
+def process_ctx(sketch, solid, render):
+ solid = solid.astype(np.float32)
+ sketch = d_resize(cv2.cvtColor(sketch, cv2.COLOR_GRAY2RGB), solid.shape).astype(np.float32)
+ render = d_resize(render, solid.shape).astype(np.float32)
+ alpha = sketch / 255.0
+ all_diff = render - solid
+ all_lines = render.copy()
+ all_lines = cv2.erode(all_lines, np.ones((3,3), np.uint8)) * 0.618
+ all_diff = re_med(all_diff)
+ all_lines = re_med(all_lines)
+ recon = solid + all_diff
+ recon = recon * alpha + all_lines * (1 - alpha)
+ recon2 = (solid + all_diff) * alpha + re_med(solid) * (1 - alpha)
+ recon3 = reason_blending(recon2, sketch)
+ return recon.clip(0, 255).astype(np.uint8), recon2.clip(0, 255).astype(np.uint8), recon3.clip(0, 255).astype(np.uint8)
+
+
+def process_psd(sketch, solid, render, path='./'):
+ recon = process_ctx(sketch, solid, render)
+ points = get_theme(solid)
+ points = simplify_points(points, solid)
+ compositions = get_ini_layers(solid, points)
+ compositions = improve_layers(compositions, solid)
+ for _ in range(ksd):
+ cv2.imwrite(path + str(_ + 1) + '.color.png', compositions[_].clip(0, 255).astype(np.uint8))
+ solid = make_reconstruction(compositions).clip(0, 255).astype(np.uint8)
+ os.makedirs(path, exist_ok=True)
+ alpha = 1 - sketch.astype(np.float32) / 255.0
+ now = solid
+ now = (now.astype(np.float32) + sketch.astype(np.float32) - 255.0).clip(0, 255)
+ sketch = 255 + now - solid
+ cv2.imwrite(path + '9.sketch.png', sketch.clip(0, 255).astype(np.uint8))
+ all_diff = recon.astype(np.float32) - now
+ all_light = all_diff.copy()
+ all_shadow = - all_diff.copy()
+ all_light[all_light < 0] = 0
+ all_shadow[all_shadow < 0] = 0
+ sketch_color = all_light * alpha
+ light = all_light * (1 - alpha)
+ all_shadow = 255 - all_shadow
+ cv2.imwrite(path + '10.sketch_color.png', sketch_color.clip(0, 255).astype(np.uint8))
+ cv2.imwrite(path + '11.light.png', light.clip(0, 255).astype(np.uint8))
+ cv2.imwrite(path + '12.shadow.png', all_shadow.clip(0, 255).astype(np.uint8))
+ return recon
+
+
+def process_albedo(albedo, composition, sketch):
+ DEL = albedo.astype(np.float32)
+ HSV = cv2.cvtColor(albedo, cv2.COLOR_RGB2HSV).astype(np.float32)
+ YUV = cv2.cvtColor(albedo, cv2.COLOR_RGB2YUV).astype(np.float32)
+ solid = composition.astype(np.float32)
+ light = sketch[:, :, None].astype(np.float32)
+
+ DEL = DEL * light / 255.0 + solid * (1 - light / 255.0)
+ HSV[:, :, 2:3] = np.minimum(HSV[:, :, 2:3], light)
+ YUV[:, :, 0:1] = np.minimum(YUV[:, :, 0:1], light)
+
+ DEL = DEL.clip(0, 255).astype(np.uint8)
+ HSV = HSV.clip(0, 255).astype(np.uint8)
+ YUV = YUV.clip(0, 255).astype(np.uint8)
+
+ return cv2.cvtColor(HSV, cv2.COLOR_HSV2RGB), cv2.cvtColor(YUV, cv2.COLOR_YUV2RGB), DEL
+
+
+def process_overlay(composition, sketch):
+ RGB = composition.astype(np.float32)
+ alpha = sketch[:, :, None].astype(np.float32) / 255.0
+ return (RGB * alpha).clip(0, 255).astype(np.uint8)

generate_bash.py

@@ -0,0 +1,35 @@
+lines = []
+
+for _ in range(50, 66):
+ lines.append("screen -dmS \"server.80"+str(_)+"\" bash -c \'while : ;do python3 server.py 80"+str(_)+"; done;\'\n")
+
+with open('run.bash', 'wt') as f:
+ f.writelines(lines)
+
+lines = []
+
+for _ in range(50, 66):
+ lines.append("screen -S \"server.80"+str(_)+"\" -X quit\n")
+
+with open('kill.bash', 'wt') as f:
+ f.writelines(lines)
+
+lines = []
+
+for _ in range(50, 66):
+ lines.append("server 127.0.0.1:80"+str(_)+" weight=4 max_fails=2 fail_timeout=600s;\n")
+
+with open('nginx.txt', 'wt') as f:
+ f.writelines(lines)
+
+lines = ["ufw allow 80/tcp\n"]
+
+for _ in range(50, 66):
+ lines.append("ufw allow 80"+str(_)+"/tcp\n")
+
+with open('tcp.bash', 'wt') as f:
+ f.writelines(lines)
+
+#/etc/nginx/sites-available
+#/var/log/nginx
+

linefiller/thinning.py

@@ -0,0 +1,44 @@
+import numpy as np
+import cv2
+from numba import njit
+
+
+@njit
+def njit_thin(points, maps):
+ result = maps.copy()
+ h, w = maps.shape[:2]
+ for _ in range(len(points[0])):
+ x = points[0][_]
+ y = points[1][_]
+ if x > 0:
+ a = maps[x-1, y]
+ if a > 0:
+ result[x, y] = a
+ continue
+ if y > 0:
+ a = maps[x, y-1]
+ if a > 0:
+ result[x, y] = a
+ continue
+ if x + 1 < h:
+ a = maps[x+1, y]
+ if a > 0:
+ result[x, y] = a
+ continue
+ if y + 1 < w:
+ a = maps[x, y+1]
+ if a > 0:
+ result[x, y] = a
+ continue
+ return result
+
+
+def thinning(fillmap, max_iter=100):
+ result = fillmap.copy()
+ for iterNum in range(max_iter):
+ line_points = np.where(result == 0)
+ if not len(line_points[0]) > 0:
+ break
+ result = njit_thin(line_points, result)
+ return result
+

linefiller/third_party.py

@@ -0,0 +1,396 @@
+import cv2
+from .thinning import *
+from .trappedball_fill import *
+from skimage.measure import block_reduce
+from skimage.morphology import disk, dilation, erosion
+from numba import njit
+
+
+def np_min_pool(x):
+ return block_reduce(x, (2, 2), np.min)
+
+
+def np_max_pool(x):
+ return block_reduce(x, (2, 2), np.max)
+
+
+def np_max_441(x):
+ return block_reduce(x, (4, 4, 1), np.max)
+
+
+def np_max_pool_221(x):
+ return block_reduce(x, (2, 2, 1), np.max)
+
+
+def np_max_pool_s(x, s):
+ return block_reduce(x, (s, s, 1), np.max)
+
+
+def binarize(x):
+ xp = x.copy()
+ xp[xp < 250] = 0
+ xp[xp > 0] = 255
+ return xp
+
+
+def get_initial_fillmap(boundary, merge=True):
+ fillmap = build_fill_map(boundary, flood_fill_multi(boundary, merge=merge))
+ return fillmap
+
+
+def up_propagate(small_fillmap, big_boundary):
+ new_fillmap = cv2.resize(small_fillmap, (big_boundary.shape[1], big_boundary.shape[0]), interpolation=cv2.INTER_NEAREST)
+ padded_fillmap = np.pad(new_fillmap, [[1, 1], [1, 1]], 'constant', constant_values=0)
+ new_mask = np.ones_like(new_fillmap, dtype=np.uint8) * 255
+ new_mask[new_fillmap > 0] = 0
+ new_mask[big_boundary < 240] = 0
+ fills = flood_fill_multi(new_mask, merge=True)
+ max_id = np.max(new_fillmap)
+ for item in fills:
+ points0 = padded_fillmap[(item[0] + 1, item[1] + 0)]
+ points1 = padded_fillmap[(item[0] + 1, item[1] + 2)]
+ points2 = padded_fillmap[(item[0] + 0, item[1] + 1)]
+ points3 = padded_fillmap[(item[0] + 2, item[1] + 1)]
+
+ all_points = np.concatenate([points0, points1, points2, points3], axis=0)
+ pointsets, pointcounts = np.unique(all_points[all_points > 0], return_counts=True)
+
+ if len(pointsets) > 0:
+ new_fillmap[item] = pointsets[np.argmax(pointcounts)]
+ else:
+ max_id += 1
+ new_fillmap[item] = max_id
+ return new_fillmap
+
+
+def laplas_fill(b_512, b_256, b_128):
+ b_512 = binarize(b_512)
+ b_256 = binarize(b_256)
+ b_128 = binarize(b_128)
+ f128 = get_initial_fillmap(b_128)
+ f256 = up_propagate(f128, b_256)
+ f512 = up_propagate(f256, b_512)
+ fin = thinning(f512)
+ return fin
+
+
+@ njit
+def get_corner(x):
+ corner = x.copy()
+ s0 = corner.shape[0]
+ s1 = corner.shape[1]
+ for i0 in range(1, s0 - 1):
+ for i1 in range(1, s1 - 1):
+ if x[i0, i1] == 0:
+ continue
+ if x[i0, i1 - 1] == 0:
+ if x[i0 - 1, i1 - 1] == 0:
+ continue
+ if x[i0 + 1, i1 - 1] == 0:
+ continue
+ corner[i0, i1] = 0
+ continue
+ if x[i0, i1 + 1] == 0:
+ if x[i0 - 1, i1 + 1] == 0:
+ continue
+ if x[i0 + 1, i1 + 1] == 0:
+ continue
+ corner[i0, i1] = 0
+ continue
+ if x[i0 - 1, i1] == 0:
+ if x[i0 - 1, i1 - 1] == 0:
+ continue
+ if x[i0 - 1, i1 + 1] == 0:
+ continue
+ corner[i0, i1] = 0
+ continue
+ if x[i0 + 1, i1] == 0:
+ if x[i0 + 1, i1 - 1] == 0:
+ continue
+ if x[i0 + 1, i1 + 1] == 0:
+ continue
+ corner[i0, i1] = 0
+ continue
+ return corner
+
+
+def monogrouh(x):
+ y = 255 - x
+ y = dilation(y, disk(1))
+ y = dilation(y, disk(1))
+ y = erosion(y, disk(1))
+ y = erosion(y, disk(1))
+ y = 255 - y
+ return y
+
+
+def corners(x):
+ y = x.copy()
+ y = monogrouh(y)
+ y = get_corner(y)
+ y = monogrouh(y)
+ y = get_corner(y)
+ y = monogrouh(y)
+ return y
+
+
+def save_fill(name, fill):
+ cv2.imwrite(name, show_fill_map(fill))
+
+
+def double_fill(b_1024, b_512, b256):
+ b256 = binarize(b256)
+ b_512 = binarize(b_512)
+ b_1024 = binarize(b_1024)
+ b_1024 = corners(b_1024)
+ b_512 = np.min(np.stack([b_512, np_min_pool(b_1024)], axis=2), axis=2)
+ b_512 = corners(b_512)
+ b_256 = np.min(np.stack([b256, np_min_pool(b_512)], axis=2), axis=2)
+ b_256 = corners(b_256)
+ b_128 = np_min_pool(b_256)
+ b_128 = corners(b_128)
+ b_64 = np_min_pool(b_128)
+ f64 = get_initial_fillmap(b_64)
+ print('get_initial_fillmap(b_64)')
+ f128 = up_propagate(f64, b_128)
+ print('up_propagate(f64, b_128)')
+ f256 = up_propagate(f128, b_256)
+ print('up_propagate(f128, b_256)')
+ f512 = up_propagate(f256, b_512)
+ print('up_propagate(f256, b_512)')
+ f1024 = up_propagate(f512, b_1024)
+ print('up_propagate(f512, b_1024)')
+ fin = thinning(f1024)
+ print('thinning(f1024)')
+
+ # cv2.imwrite('b_64.png', b_64)
+ # cv2.imwrite('b_128.png', b_128)
+ # cv2.imwrite('b_256.png', b_256)
+ # cv2.imwrite('b_512.png', b_512)
+ # cv2.imwrite('b_1024.png', b_1024)
+ # save_fill('f64.png', f64)
+ # save_fill('f128.png', f128)
+ # save_fill('f256.png', f256)
+ # save_fill('f512.png', f512)
+ # save_fill('f1024.png', f1024)
+ # save_fill('fin.png', fin)
+
+ return find_all(fin)
+
+
+def single_fill(b_2048, path):
+ b_2048 = corners(binarize(b_2048))
+ f2048 = get_initial_fillmap(b_2048, merge=False)
+ print(path + 'get_initial_fillmap(b_2048, merge=False)')
+ fin = thinning(f2048)
+ print(path + 'thinning(f2048)')
+ # cv2.imwrite(path + 'b_2048.png', b_2048)
+ # save_fill(path + 'f2048.png', f2048)
+ # save_fill(path + 'fin.png', fin)
+ return find_all(fin)
+
+
+def deatlize(x):
+ x = cv2.GaussianBlur(x, (0, 0), 0.8)
+ x = cv2.medianBlur(x, 3)
+ return x
+
+
+def low_down(gradient_mask):
+ return 1.0 - cv2.dilate(255 - gradient_mask, np.ones((3, 3), np.uint8), iterations=2).astype(np.float32) / 255.0
+
+
+def cv2pyrDown(x):
+ return cv2.pyrDown(cv2.medianBlur(cv2.medianBlur(x, 3), 3))
+
+
+def cv2pyrUp(x):
+ return cv2.pyrUp(cv2.medianBlur(cv2.medianBlur(x, 3), 3))
+
+
+def re_deatlize(visulized, s1024):
+
+ gradient_mask_1024 = binarize(s1024)
+ gradient_mask_512 = np_min_pool(gradient_mask_1024)
+ gradient_mask_256 = np_min_pool(gradient_mask_512)
+ gradient_mask_128 = np_min_pool(gradient_mask_256)
+ gradient_mask_64 = np_min_pool(gradient_mask_128)
+
+ gradient_mask_1024 = low_down(gradient_mask_1024)
+ gradient_mask_512 = low_down(gradient_mask_512)
+ gradient_mask_256 = low_down(gradient_mask_256)
+ gradient_mask_128 = low_down(gradient_mask_128)
+ gradient_mask_64 = low_down(gradient_mask_64)
+
+ sample_1024 = visulized.astype(np.float32)
+ sample_512 = cv2pyrDown(sample_1024)
+ sample_256 = cv2pyrDown(sample_512)
+ sample_128 = cv2pyrDown(sample_256)
+ sample_64 = cv2pyrDown(sample_128)
+ sample_32 = cv2pyrDown(sample_64)
+
+ gradient_1024 = sample_1024 - cv2pyrUp(sample_512)
+ gradient_512 = sample_512 - cv2pyrUp(sample_256)
+ gradient_256 = sample_256 - cv2pyrUp(sample_128)
+ gradient_128 = sample_128 - cv2pyrUp(sample_64)
+ gradient_64 = sample_64 - cv2pyrUp(sample_32)
+
+ rec_32 = sample_32
+ rec_64 = cv2pyrUp(rec_32) + gradient_64 * (1 - gradient_mask_64[:, :, None])
+ rec_128 = cv2pyrUp(rec_64) + gradient_128 * (1 - gradient_mask_128[:, :, None])
+ rec_256 = cv2pyrUp(rec_128) + gradient_256 * (1 - gradient_mask_256[:, :, None])
+ rec_512 = cv2pyrUp(rec_256) + gradient_512 * (1 - gradient_mask_512[:, :, None])
+ rec_1024 = cv2pyrUp(rec_512) + gradient_1024 * (1 - gradient_mask_1024[:, :, None])
+
+ return rec_1024.clip(0, 255).astype(np.uint8)
+
+
+def tiny_deatlize(visulized, s2048):
+ gradient_mask_2048 = s2048.copy()
+ gradient_mask_1024 = np_min_pool(gradient_mask_2048)
+ gradient_mask_512 = np_min_pool(gradient_mask_1024)
+ gradient_mask_256 = np_min_pool(gradient_mask_512)
+
+ gradient_mask_2048 = low_down(gradient_mask_2048)
+ gradient_mask_1024 = low_down(gradient_mask_1024)
+ gradient_mask_512 = low_down(gradient_mask_512)
+ gradient_mask_256 = low_down(gradient_mask_256)
+
+ sample_2048 = visulized.astype(np.float32)
+ sample_1024 = cv2.pyrDown(sample_2048)
+ sample_512 = cv2.pyrDown(sample_1024)
+ sample_256 = cv2.pyrDown(sample_512)
+ sample_128 = cv2.pyrDown(sample_256)
+
+ gradient_2048 = sample_2048 - cv2.pyrUp(sample_1024)
+ gradient_1024 = sample_1024 - cv2.pyrUp(sample_512)
+ gradient_512 = sample_512 - cv2.pyrUp(sample_256)
+ gradient_256 = sample_256 - cv2.pyrUp(sample_128)
+
+ rec_128 = sample_128
+ rec_256 = cv2.pyrUp(rec_128) + gradient_256 * (1 - gradient_mask_256[:, :, None])
+ rec_512 = cv2.pyrUp(rec_256) + gradient_512 * (1 - gradient_mask_512[:, :, None])
+ rec_1024 = cv2.pyrUp(rec_512) + gradient_1024 * (1 - gradient_mask_1024[:, :, None])
+ rec_2048 = cv2.pyrUp(rec_1024) + gradient_2048 * (1 - gradient_mask_2048[:, :, None])
+ return rec_2048.clip(0, 255).astype(np.uint8)
+
+
+def adain(x, y):
+ x_high = cv2.GaussianBlur(x, (0, 0), 3.0)
+ y_high = cv2.GaussianBlur(y, (0, 0), 3.0)
+ return (x.astype(np.float32) - x_high.astype(np.float32) + y_high.astype(np.float32)).clip(0, 255).astype(np.uint8)
+
+
+def corrupt(x, b128):
+ float_sketch = x.astype(float)
+ float_base = cv2.resize(float_sketch, (b128.shape[1], b128.shape[0]), cv2.INTER_AREA)
+ alpha = b128[:, :, 0] / 255.0
+ float_base = alpha * float_base + (1 - alpha) * np.mean(float_base)
+ float_base = cv2.GaussianBlur(float_base, (0, 0), 8.0)
+ float_base = cv2.resize(float_base, (x.shape[1], x.shape[0]), cv2.INTER_CUBIC)
+ result = float_sketch / (float_base + 1e-10)
+ result = result.clip(0, 1)
+ result -= np.min(result)
+ result /= np.max(result)
+ return (result * 255.0).clip(0, 255).astype(np.uint8)
+
+
+def fuse_sketch(color, sketch, fills, fixer, points_arr, colors_arr):
+ sketch = cv2.resize(sketch, (color.shape[1], color.shape[0]))
+ fills = cv2.resize(fills, (color.shape[1], color.shape[0]), interpolation=cv2.INTER_NEAREST)
+ fill_id = np.unique(fills.flatten())
+ bg = np.zeros_like(color, dtype=np.uint8)
+ checking_result = np.zeros(dtype=np.int32, shape=(np.max(fills) + 1,)) - 1
+ length_points = int(len(points_arr))
+ for _ in range(length_points):
+ checking_result[fills[points_arr[_][0], points_arr[_][1]]] = _
+ for id in fill_id:
+ points = np.where(fills == id)
+ if len(points[0]) > 0:
+ color_id = checking_result[id]
+ if color_id > -1:
+ bg[points] = np.array(colors_arr[color_id])
+ else:
+ bg[points] = np.median(color[points], axis=0)
+ fixed = adain(fixer(sketch, bg), bg)
+ result = (fixed.astype(np.float32) + sketch[:, :, None].astype(np.float32) - 255.0).clip(0, 255).astype(np.uint8)
+ return result, fixed, bg
+
+
+def balance_fill(color, fills, points, sizer):
+ color = cv2.resize(color, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+ points = cv2.resize(points, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+ bg = np.zeros_like(color, dtype=np.uint8)
+ for region in fills:
+ if len(region[0]) > 0:
+ region_points = points[region]
+ region_points = region_points[region_points[:, 3] > 0]
+ if region_points.shape[0] > 0:
+ points_color, points_color_count = np.unique(region_points, return_counts=True, axis=0)
+ bg[region] = points_color[np.argmax(points_color_count)][0:3]
+ else:
+ bg[region] = np.median(color[region], axis=0)
+ return bg
+
+
+def shade_fill(color, fills, points, sizer):
+ color = cv2.resize(color, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+ points = cv2.resize(points, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+ bg = np.zeros_like(color, dtype=np.uint8)
+ for region in fills:
+ if len(region[0]) > 0:
+ region_points = points[region]
+ region_points = region_points[region_points[:, 3] > 0]
+ if region_points.shape[0] > 0:
+ points_color, points_color_count = np.unique(region_points, return_counts=True, axis=0)
+ c = points_color[np.argmax(points_color_count)][0:3]
+ r = c[0]
+ g = c[1]
+ b = c[2]
+ if r == 1 and g == 233 and b == 0:
+ bg[region] = 255
+ elif r == 0 and g == 233 and b == 1:
+ bg[region] = 0
+ else:
+ bg[region] = np.median(color[region], axis=0)
+ else:
+ bg[region] = np.median(color[region], axis=0)
+ return bg
+
+
+def get_alpha_piece(points):
+ padded_points = np.pad(points, [[1, 1], [1, 1], [0, 0]], 'constant', constant_values=127)
+ lines = 255 - padded_points[:, :, 3]
+ lines[lines < 240] = 0
+ fills = flood_fill_multi(lines, merge=True)
+ result = np.zeros_like(padded_points)
+ for item in fills:
+ points0 = padded_points[(item[0], item[1] + 1)]
+ points1 = padded_points[(item[0], item[1] - 1)]
+ points2 = padded_points[(item[0] + 1, item[1])]
+ points3 = padded_points[(item[0] - 1, item[1])]
+ all_points = np.concatenate([points0, points1, points2, points3], axis=0)
+ all_points = all_points[all_points[:, 3] > 0]
+ all_points = np.unique(all_points, axis=0)
+ if all_points.shape[0] == 1:
+ result[item] = all_points[0]
+ piece = result[1:-1, 1:-1, :]
+ piece = np.maximum(piece, points)
+ return piece, points
+
+
+def fin_deatlize(color, sketch):
+
+ cf = color.astype(np.float32)
+ alpha = sketch.astype(np.float32)[:, :, None] / 255.0
+
+ plain = cf * alpha
+ lines = cf * (1 - alpha)
+
+ plain = cv2.medianBlur(plain, 5)
+ plain = cv2.medianBlur(plain, 3)
+
+ fin = plain + lines
+
+ return fin.clip(0, 255).astype(np.uint8)
+

linefiller/trappedball_fill.py

@@ -0,0 +1,436 @@
+import cv2
+import numpy as np
+from scipy.ndimage import label
+from numba import njit
+
+
+def get_ball_structuring_element(radius):
+ """Get a ball shape structuring element with specific radius for morphology operation.
+ The radius of ball usually equals to (leaking_gap_size / 2).
+ 
+ # Arguments
+ radius: radius of ball shape.
+ 
+ # Returns
+ an array of ball structuring element.
+ """
+ return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * radius + 1, 2 * radius + 1))
+
+
+def get_unfilled_point(image):
+ """Get points belong to unfilled(value==255) area.
+
+ # Arguments
+ image: an image.
+
+ # Returns
+ an array of points.
+ """
+ y, x = np.where(image == 255)
+
+ return np.stack((x.astype(int), y.astype(int)), axis=-1)
+
+
+def exclude_area(image, radius):
+ """Perform erosion on image to exclude points near the boundary.
+ We want to pick part using floodfill from the seed point after dilation. 
+ When the seed point is near boundary, it might not stay in the fill, and would
+ not be a valid point for next floodfill operation. So we ignore these points with erosion.
+
+ # Arguments
+ image: an image.
+ radius: radius of ball shape.
+
+ # Returns
+ an image after dilation.
+ """
+ return cv2.morphologyEx(image, cv2.MORPH_ERODE, get_ball_structuring_element(radius), anchor=(-1, -1), iterations=1)
+
+
+def trapped_ball_fill_single(image, seed_point, radius):
+ """Perform a single trapped ball fill operation.
+
+ # Arguments
+ image: an image. the image should consist of white background, black lines and black fills.
+ the white area is unfilled area, and the black area is filled area.
+ seed_point: seed point for trapped-ball fill, a tuple (integer, integer).
+ radius: radius of ball shape.
+ # Returns
+ an image after filling.
+ """
+ ball = get_ball_structuring_element(radius)
+
+ pass1 = np.full(image.shape, 255, np.uint8)
+ pass2 = np.full(image.shape, 255, np.uint8)
+
+ im_inv = cv2.bitwise_not(image)
+
+ # Floodfill the image
+ mask1 = cv2.copyMakeBorder(im_inv, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+ _, pass1, _, _ = cv2.floodFill(pass1, mask1, seed_point, 0, 0, 0, 4)
+
+ # Perform dilation on image. The fill areas between gaps became disconnected.
+ pass1 = cv2.morphologyEx(pass1, cv2.MORPH_DILATE, ball, anchor=(-1, -1), iterations=1)
+ mask2 = cv2.copyMakeBorder(pass1, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+
+ # Floodfill with seed point again to select one fill area.
+ _, pass2, _, rect = cv2.floodFill(pass2, mask2, seed_point, 0, 0, 0, 4)
+ # Perform erosion on the fill result leaking-proof fill.
+ pass2 = cv2.morphologyEx(pass2, cv2.MORPH_ERODE, ball, anchor=(-1, -1), iterations=1)
+
+ return pass2
+
+
+def trapped_ball_fill_multi(image, radius, method='mean', max_iter=1000):
+ """Perform multi trapped ball fill operations until all valid areas are filled.
+
+ # Arguments
+ image: an image. The image should consist of white background, black lines and black fills.
+ the white area is unfilled area, and the black area is filled area.
+ radius: radius of ball shape.
+ method: method for filtering the fills. 
+ 'max' is usually with large radius for select large area such as background.
+ max_iter: max iteration number.
+ # Returns
+ an array of fills' points.
+ """
+ print('trapped-ball ' + str(radius))
+
+ unfill_area = image
+ filled_area, filled_area_size, result = [], [], []
+
+ for _ in range(max_iter):
+ points = get_unfilled_point(exclude_area(unfill_area, radius))
+
+ if not len(points) > 0:
+ break
+
+ fill = trapped_ball_fill_single(unfill_area, (points[0][0], points[0][1]), radius)
+ unfill_area = cv2.bitwise_and(unfill_area, fill)
+
+ filled_area.append(np.where(fill == 0))
+ filled_area_size.append(len(np.where(fill == 0)[0]))
+
+ filled_area_size = np.asarray(filled_area_size)
+
+ if method == 'max':
+ area_size_filter = np.max(filled_area_size)
+ elif method == 'median':
+ area_size_filter = np.median(filled_area_size)
+ elif method == 'mean':
+ area_size_filter = np.mean(filled_area_size)
+ else:
+ area_size_filter = 0
+
+ result_idx = np.where(filled_area_size >= area_size_filter)[0]
+
+ for i in result_idx:
+ result.append(filled_area[i])
+
+ return result
+
+
+def flood_fill_single(im, seed_point):
+ """Perform a single flood fill operation.
+
+ # Arguments
+ image: an image. the image should consist of white background, black lines and black fills.
+ the white area is unfilled area, and the black area is filled area.
+ seed_point: seed point for trapped-ball fill, a tuple (integer, integer).
+ # Returns
+ an image after filling.
+ """
+ pass1 = np.full(im.shape, 255, np.uint8)
+
+ im_inv = cv2.bitwise_not(im)
+
+ mask1 = cv2.copyMakeBorder(im_inv, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+ _, pass1, _, _ = cv2.floodFill(pass1, mask1, seed_point, 0, 0, 0, 4)
+
+ return pass1
+
+
+@njit
+def count_all(labeled_array, all_counts):
+ M = labeled_array.shape[0]
+ N = labeled_array.shape[1]
+ for x in range(M):
+ for y in range(N):
+ i = labeled_array[x, y] - 1
+ if i > -1:
+ all_counts[i] = all_counts[i] + 1
+ return
+
+
+@njit
+def trace_all(labeled_array, xs, ys, cs):
+ M = labeled_array.shape[0]
+ N = labeled_array.shape[1]
+ for x in range(M):
+ for y in range(N):
+ current_label = labeled_array[x, y] - 1
+ if current_label > -1:
+ current_label_count = cs[current_label]
+ xs[current_label][current_label_count] = x
+ ys[current_label][current_label_count] = y
+ cs[current_label] = current_label_count + 1
+ return
+
+
+def find_all(labeled_array):
+ hist_size = int(np.max(labeled_array))
+ if hist_size == 0:
+ return []
+ all_counts = [0 for _ in range(hist_size)]
+ count_all(labeled_array, all_counts)
+ xs = [np.zeros(shape=(item, ), dtype=np.uint32) for item in all_counts]
+ ys = [np.zeros(shape=(item, ), dtype=np.uint32) for item in all_counts]
+ cs = [0 for item in all_counts]
+ trace_all(labeled_array, xs, ys, cs)
+ filled_area = []
+ for _ in range(hist_size):
+ filled_area.append((xs[_], ys[_]))
+ return filled_area
+
+
+def flood_fill_multi(image, merge=False):
+ print('floodfill')
+
+ labeled_array, num_features = label(image / 255)
+ print('floodfill_ok1')
+
+ filled_area = find_all(labeled_array)
+
+ print('floodfill_ok2')
+
+ if merge:
+ new_fill = []
+ for item in filled_area:
+ if len(item[0]) > 8:
+ new_fill.append(item)
+ return new_fill
+
+ print('floodfill_ok3')
+
+ return filled_area
+
+
+def old_flood_fill_multi(image, max_iter=20000):
+ """Perform multi flood fill operations until all valid areas are filled.
+ This operation will fill all rest areas, which may result large amount of fills.
+
+ # Arguments
+ image: an image. the image should contain white background, black lines and black fills.
+ the white area is unfilled area, and the black area is filled area.
+ max_iter: max iteration number.
+ # Returns
+ an array of fills' points.
+ """
+ print('floodfill')
+
+ unfill_area = image
+ filled_area = []
+
+ for _ in range(max_iter):
+ points = get_unfilled_point(unfill_area)
+
+ if not len(points) > 0:
+ break
+
+ fill = flood_fill_single(unfill_area, (points[0][0], points[0][1]))
+ unfill_area = cv2.bitwise_and(unfill_area, fill)
+
+ filled_area.append(np.where(fill == 0))
+
+ return filled_area
+
+
+def mark_fill(image, fills):
+ """Mark filled areas with 0.
+
+ # Arguments
+ image: an image.
+ fills: an array of fills' points.
+ # Returns
+ an image.
+ """
+ result = image.copy()
+
+ for fill in fills:
+ result[fill] = 0
+
+ return result
+
+
+def build_fill_map(image, fills):
+ """Make an image(array) with each pixel(element) marked with fills' id. id of line is 0.
+
+ # Arguments
+ image: an image.
+ fills: an array of fills' points.
+ # Returns
+ an array.
+ """
+ result = np.zeros(image.shape[:2], np.int)
+
+ for index, fill in enumerate(fills):
+
+ if(len(fill[0]) == 0):
+ continue
+
+ result[fill] = index + 1
+
+ return result
+
+
+def show_fill_map(fillmap):
+ """Mark filled areas with colors. It is useful for visualization.
+
+ # Arguments
+ image: an image.
+ fills: an array of fills' points.
+ # Returns
+ an image.
+ """
+ # Generate color for each fill randomly.
+ colors = np.random.randint(0, 255, (np.max(fillmap) + 1, 3))
+ # Id of line is 0, and its color is black.
+ colors[0] = [0, 0, 0]
+
+ return colors[fillmap]
+
+
+def get_bounding_rect(points):
+ """Get a bounding rect of points.
+
+ # Arguments
+ points: array of points.
+ # Returns
+ rect coord
+ """
+ x1, y1, x2, y2 = np.min(points[1]), np.min(points[0]), np.max(points[1]), np.max(points[0])
+ return x1, y1, x2, y2
+
+
+def get_border_bounding_rect(h, w, p1, p2, r):
+ """Get a valid bounding rect in the image with border of specific size.
+
+ # Arguments
+ h: image max height.
+ w: image max width.
+ p1: start point of rect.
+ p2: end point of rect.
+ r: border radius.
+ # Returns
+ rect coord
+ """
+ x1, y1, x2, y2 = p1[0], p1[1], p2[0], p2[1]
+
+ x1 = x1 - r if 0 < x1 - r else 0
+ y1 = y1 - r if 0 < y1 - r else 0
+ x2 = x2 + r + 1 if x2 + r + 1 < w else w
+ y2 = y2 + r + 1 if y2 + r + 1 < h else h
+
+ return x1, y1, x2, y2
+
+
+def get_border_point(points, rect, max_height, max_width):
+ """Get border points of a fill area
+
+ # Arguments
+ points: points of fill .
+ rect: bounding rect of fill.
+ max_height: image max height.
+ max_width: image max width.
+ # Returns
+ points , convex shape of points
+ """
+ # Get a local bounding rect.
+ border_rect = get_border_bounding_rect(max_height, max_width, rect[:2], rect[2:], 2)
+
+ # Get fill in rect.
+ fill = np.zeros((border_rect[3] - border_rect[1], border_rect[2] - border_rect[0]), np.uint8)
+ # Move points to the rect.
+ fill[(points[0] - border_rect[1], points[1] - border_rect[0])] = 255
+
+ # Get shape.
+ _, contours, _ = cv2.findContours(fill, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ # approx_shape = cv2.approxPolyDP(contours[0], 0.02 * cv2.arcLength(contours[0], True), True)
+
+ # Get border pixel.
+ # Structuring element in cross shape is used instead of box to get 4-connected border.
+ cross = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3))
+ border_pixel_mask = cv2.morphologyEx(fill, cv2.MORPH_DILATE, cross, anchor=(-1, -1), iterations=1) - fill
+ border_pixel_points = np.where(border_pixel_mask == 255)
+
+ # Transform points back to fillmap.
+ border_pixel_points = (border_pixel_points[0] + border_rect[1], border_pixel_points[1] + border_rect[0])
+
+ return border_pixel_points
+
+
+def merge_fill(fillmap, max_iter=20):
+ """Merge fill areas.
+
+ # Arguments
+ fillmap: an image.
+ max_iter: max iteration number.
+ # Returns
+ an image.
+ """
+ max_height, max_width = fillmap.shape[:2]
+ result = fillmap.copy()
+
+ for i in range(max_iter):
+ print('merge ' + str(i + 1))
+
+ result[np.where(fillmap == 0)] = 0
+
+ fill_id = np.unique(result.flatten())
+ fills = []
+
+ for j in fill_id:
+ point = np.where(result == j)
+
+ fills.append({
+ 'id': j,
+ 'point': point,
+ 'area': len(point[0]),
+ })
+
+ for j, f in enumerate(fills):
+ # ignore lines
+ if f['id'] == 0:
+ continue
+
+ if f['area'] < 5:
+ result[f['point']] = 0
+
+ if len(fill_id) == len(np.unique(result.flatten())):
+ break
+
+ return result
+
+
+def merge_one(fillmap):
+ result = fillmap.copy()
+ print('merge')
+ result[np.where(fillmap == 0)] = 0
+ fill_id = np.unique(result.flatten())
+ fills = []
+ for j in fill_id:
+ point = np.where(result == j)
+ fills.append({
+ 'id': j,
+ 'point': point,
+ 'area': len(point[0]),
+ })
+ for j, f in enumerate(fills):
+ # ignore lines
+ if f['id'] == 0:
+ continue
+
+ if f['area'] < 5:
+ result[f['point']] = 0
+ return result
+

models.py

@@ -0,0 +1,299 @@
+from keras.layers import Conv2D, Activation, Input, Concatenate, LeakyReLU, Lambda, AveragePooling2D, UpSampling2D, Convolution2D, BatchNormalization, Deconvolution2D, Add
+from keras.models import Model
+from InstanceNorm import InstanceNormalization
+
+
+def make_standard_UNET(channels,outs):
+
+ def relu(x):
+ return Activation('relu')(x)
+
+ def concat(x):
+ return Concatenate()(x)
+
+ c0 = Convolution2D(filters=32, kernel_size=3, strides=1, padding='same', name='c0')
+ c1 = Convolution2D(filters=64, kernel_size=4, strides=2, padding='same', name='c1')
+ c2 = Convolution2D(filters=64, kernel_size=3, strides=1, padding='same', name='c2')
+ c3 = Convolution2D(filters=128, kernel_size=4, strides=2, padding='same', name='c3')
+ c4 = Convolution2D(filters=128, kernel_size=3, strides=1, padding='same', name='c4')
+ c5 = Convolution2D(filters=256, kernel_size=4, strides=2, padding='same', name='c5')
+ c6 = Convolution2D(filters=256, kernel_size=3, strides=1, padding='same', name='c6')
+ c7 = Convolution2D(filters=512, kernel_size=4, strides=2, padding='same', name='c7')
+ c8 = Convolution2D(filters=512, kernel_size=3, strides=1, padding='same', name='c8')
+
+ bnc0 = BatchNormalization(axis=3, name='bnc0')
+ bnc1 = BatchNormalization(axis=3, name='bnc1')
+ bnc2 = BatchNormalization(axis=3, name='bnc2')
+ bnc3 = BatchNormalization(axis=3, name='bnc3')
+ bnc4 = BatchNormalization(axis=3, name='bnc4')
+ bnc5 = BatchNormalization(axis=3, name='bnc5')
+ bnc6 = BatchNormalization(axis=3, name='bnc6')
+ bnc7 = BatchNormalization(axis=3, name='bnc7')
+ bnc8 = BatchNormalization(axis=3, name='bnc8')
+
+ dc8 = Deconvolution2D(filters=512, kernel_size=4, strides=2, padding='same', name='dc8_')
+ dc7 = Convolution2D(filters=256, kernel_size=3, strides=1, padding='same', name='dc7')
+ dc6 = Deconvolution2D(filters=256, kernel_size=4, strides=2, padding='same', name='dc6_')
+ dc5 = Convolution2D(filters=128, kernel_size=3, strides=1, padding='same', name='dc5')
+ dc4 = Deconvolution2D(filters=128, kernel_size=4, strides=2, padding='same', name='dc4_')
+ dc3 = Convolution2D(filters=64, kernel_size=3, strides=1, padding='same', name='dc3')
+ dc2 = Deconvolution2D(filters=64, kernel_size=4, strides=2, padding='same', name='dc2_')
+ dc1 = Convolution2D(filters=32, kernel_size=3, strides=1, padding='same', name='dc1')
+ dc0 = Convolution2D(filters=outs, kernel_size=3, strides=1, padding='same', name='dc0')
+
+ bnd1 = BatchNormalization(axis=3, name='bnd1')
+ bnd2 = BatchNormalization(axis=3, name='bnd2')
+ bnd3 = BatchNormalization(axis=3, name='bnd3')
+ bnd4 = BatchNormalization(axis=3, name='bnd4')
+ bnd5 = BatchNormalization(axis=3, name='bnd5')
+ bnd6 = BatchNormalization(axis=3, name='bnd6')
+ bnd7 = BatchNormalization(axis=3, name='bnd7')
+ bnd8 = BatchNormalization(axis=3, name='bnd8')
+
+ x = Input(shape=(128, 128, channels))
+
+ e0 = relu(bnc0(c0(x), training = False))
+ e1 = relu(bnc1(c1(e0), training = False))
+ e2 = relu(bnc2(c2(e1), training = False))
+ e3 = relu(bnc3(c3(e2), training = False))
+ e4 = relu(bnc4(c4(e3), training = False))
+ e5 = relu(bnc5(c5(e4), training = False))
+ e6 = relu(bnc6(c6(e5), training = False))
+ e7 = relu(bnc7(c7(e6), training = False))
+ e8 = relu(bnc8(c8(e7), training = False))
+
+ d8 = relu(bnd8(dc8(concat([e7, e8])), training = False))
+ d7 = relu(bnd7(dc7(d8), training = False))
+ d6 = relu(bnd6(dc6(concat([e6, d7])), training = False))
+ d5 = relu(bnd5(dc5(d6), training = False))
+ d4 = relu(bnd4(dc4(concat([e4, d5])), training = False))
+ d3 = relu(bnd3(dc3(d4), training = False))
+ d2 = relu(bnd2(dc2(concat([e2, d3])), training = False))
+ d1 = relu(bnd1(dc1(d2), training = False))
+ d0 = dc0(concat([e0, d1]))
+
+ model = Model(inputs=x,outputs=d0)
+
+ return model
+
+
+def make_diff_net():
+
+ def conv(x, filters, name):
+ return Conv2D(filters=filters, strides=(1, 1), kernel_size=(3, 3), padding='same', name=name)(x)
+
+ def relu(x):
+ return Activation('relu')(x)
+
+ def lrelu(x):
+ return LeakyReLU(alpha=0.1)(x)
+
+ def r_block(x, filters, name=None):
+ return relu(conv(relu(conv(x, filters, None if name is None else name + '_c1')), filters,
+ None if name is None else name + '_c2'))
+
+ def cat(a, b):
+ return Concatenate()([UpSampling2D((2, 2))(a), b])
+
+ def dog(x):
+ down = AveragePooling2D((2, 2))(x)
+ up = UpSampling2D((2, 2))(down)
+ diff = Lambda(lambda p: p[0] - p[1])([x, up])
+ return down, diff
+
+ ip = Input(shape=(512, 512, 3))
+
+ c512 = r_block(ip, 16, 'c512')
+
+ c256, l512 = dog(c512)
+ c256 = r_block(c256, 32, 'c256')
+
+ c128, l256 = dog(c256)
+ c128 = r_block(c128, 64, 'c128')
+
+ c64, l128 = dog(c128)
+ c64 = r_block(c64, 128, 'c64')
+
+ c32, l64 = dog(c64)
+ c32 = r_block(c32, 256, 'c32')
+
+ c16, l32 = dog(c32)
+ c16 = r_block(c16, 512, 'c16')
+
+ d32 = cat(c16, l32)
+ d32 = r_block(d32, 256, 'd32')
+
+ d64 = cat(d32, l64)
+ d64 = r_block(d64, 128, 'd64')
+
+ d128 = cat(d64, l128)
+ d128 = r_block(d128, 64, 'd128')
+
+ d256 = cat(d128, l256)
+ d256 = r_block(d256, 32, 'd256')
+
+ d512 = cat(d256, l512)
+ d512 = r_block(d512, 16, 'd512')
+
+ op = conv(d512, 1, 'op')
+
+ return Model(inputs=ip, outputs=op)
+
+
+def make_wnet256():
+
+ def conv(x, filters):
+ return Conv2D(filters=filters, strides=(1, 1), kernel_size=(3, 3), padding='same')(x)
+
+ def relu(x):
+ return Activation('relu')(x)
+
+ def lrelu(x):
+ return LeakyReLU(alpha=0.1)(x)
+
+ def r_block(x, filters):
+ return relu(conv(relu(conv(x, filters)), filters))
+
+ def res_block(x, filters):
+ return relu(Add()([x, conv(relu(conv(x, filters)), filters)]))
+
+ def cat(a, b):
+ return Concatenate()([UpSampling2D((2, 2))(a), b])
+
+ def dog(x):
+ down = AveragePooling2D((2, 2))(x)
+ up = UpSampling2D((2, 2))(down)
+ diff = Lambda(lambda p: p[0] - p[1])([x, up])
+ return down, diff
+
+ ip_sketch = Input(shape=(256, 256, 1))
+ ip_color = Input(shape=(256, 256, 3))
+
+ c256 = r_block(ip_sketch, 32)
+
+ c128, l256 = dog(c256)
+ c128 = r_block(c128, 64)
+
+ c64, l128 = dog(c128)
+ c64 = r_block(c64, 128)
+
+ c32, l64 = dog(c64)
+ c32 = r_block(Concatenate()([c32, AveragePooling2D((8, 8))(ip_color)]), 256)
+
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+ c32 = res_block(c32, 256)
+
+ d64 = cat(c32, l64)
+ d64 = r_block(d64, 128)
+
+ d128 = cat(d64, l128)
+ d128 = r_block(d128, 64)
+
+ d256 = cat(d128, l256)
+ d256 = r_block(d256, 32)
+
+ op = conv(d256, 3)
+
+ return Model(inputs=[ip_sketch, ip_color], outputs=op)
+
+
+def make_unet512():
+
+ def conv(x, filters, strides=(1, 1), kernel_size=(3, 3)):
+ return Conv2D(filters=filters, strides=strides, kernel_size=kernel_size, padding='same')(x)
+
+ def donv(x, filters, strides=(2, 2), kernel_size=(4, 4)):
+ return Deconvolution2D(filters=filters, strides=strides, kernel_size=kernel_size, padding='same')(x)
+
+ def relu(x):
+ return Activation('relu')(x)
+
+ def sigmoid(x):
+ return Activation('sigmoid')(x)
+
+ def norm(x):
+ return InstanceNormalization(axis=3)(x)
+
+ def cat(a, b):
+ return Concatenate()([a, b])
+
+ def res(x, filters):
+ c1 = relu(norm(conv(x, filters // 2)))
+ c2 = norm(conv(c1, filters))
+ ad = Add()([x, c2])
+ return relu(ad)
+
+ ip = Input(shape=(512, 512, 3))
+
+ c512 = relu(norm(conv(ip, 16, strides=(1, 1), kernel_size=(3, 3))))
+ c256 = relu(norm(conv(c512, 32, strides=(2, 2), kernel_size=(4, 4))))
+
+ c128 = relu(norm(conv(c256, 64, strides=(2, 2), kernel_size=(4, 4))))
+ c128 = res(c128, 64)
+
+ c64 = relu(norm(conv(c128, 128, strides=(2, 2), kernel_size=(4, 4))))
+ c64 = res(c64, 128)
+ c64 = res(c64, 128)
+
+ c32 = relu(norm(conv(c64, 256, strides=(2, 2), kernel_size=(4, 4))))
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+ c32 = res(c32, 256)
+
+ c16 = relu(norm(conv(c32, 512, strides=(2, 2), kernel_size=(4, 4))))
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+ c16 = res(c16, 512)
+
+ c8 = relu(norm(conv(c16, 1024, strides=(2, 2), kernel_size=(4, 4))))
+ c8 = res(c8, 1024)
+ c8 = res(c8, 1024)
+ c8 = res(c8, 1024)
+ c8 = res(c8, 1024)
+
+ e16 = relu(norm(donv(c8, 512, strides=(2, 2), kernel_size=(4, 4))))
+ e16 = cat(e16, c16)
+ e16 = relu(norm(conv(e16, 512, strides=(1, 1), kernel_size=(3, 3))))
+
+ e32 = relu(norm(donv(e16, 256, strides=(2, 2), kernel_size=(4, 4))))
+ e32 = cat(e32, c32)
+ e32 = relu(norm(conv(e32, 256, strides=(1, 1), kernel_size=(3, 3))))
+
+ e64 = relu(norm(donv(e32, 128, strides=(2, 2), kernel_size=(4, 4))))
+ e64 = cat(e64, c64)
+ e64 = relu(norm(conv(e64, 128, strides=(1, 1), kernel_size=(3, 3))))
+
+ e128 = relu(norm(donv(e64, 64, strides=(2, 2), kernel_size=(4, 4))))
+ e128 = cat(e128, c128)
+ e128 = relu(norm(conv(e128, 64, strides=(1, 1), kernel_size=(3, 3))))
+
+ e256 = relu(norm(donv(e128, 32, strides=(2, 2), kernel_size=(4, 4))))
+ e256 = cat(e256, c256)
+ e256 = relu(norm(conv(e256, 32, strides=(1, 1), kernel_size=(3, 3))))
+
+ e512 = relu(norm(donv(e256, 16, strides=(2, 2), kernel_size=(4, 4))))
+ e512 = cat(e512, c512)
+ e512 = relu(norm(conv(e512, 16, strides=(1, 1), kernel_size=(3, 3))))
+
+ ot = sigmoid(conv(e512, 1))
+
+ return Model(inputs=ip, outputs=ot)

nets/head.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5f608bf5511293cd582e6d87e55f483259b8d606feba7a5042382a19cde630
+size 411622416

nets/inception.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a503841e955ee56f2e0b71219952f0f554c9eaff9e887f82853f600f0ddee80
+size 41501888

nets/mat.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2998a3cd6446ad62d87259dd5b279e344a9ba8daaf2c3c88f59ebb5ddfe09cd
+size 362262288

nets/neck.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e91e4e688379dc95a14ea8f33fa1c2141584b0387e16be5efcc7acb5299ad5d7
+size 411623808

nets/norm.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d230c0a595dc21d2924911b2d2aa9f334c2180a6526d947d11a4d00e5ce4d0
+size 113030088

nets/reader.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a65a53091acb96dbdcf325afb1b233ff374e220cb822fe8b1771dcd65f999df
+size 41502832

nets/render_head.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cacac78d2e19bd38b2efbdd8eb372594e333e5284e9f2c2b605652c8dd26ffad
+size 411612752

nets/render_neck.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66050656839363d7977dd970cc2f6d7f81722a8cf5cd67a1793bae8214a07c0b
+size 411613336

nets/tail.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2255073ccdc311d8c61c074ffeb4ad7db200ca9116011e1cf213d6d4b1967e15
+size 4018208

nets/vector.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2738164810e2c998c4e5c7598c9ad991b9aad6c300d075119da9e4201212066a
+size 31618276

nets/vgg7.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84539cc2f46a0937a31ff770af592242e1e2a06f9f3c3fd285ab596a7ab90be6
+size 1188680