import tensorflow tensorflow.compat.v1.disable_v2_behavior() tf = tensorflow.compat.v1 import keras import numpy as np from config import * from keras.models import load_model from smoother import * import keras.backend as K from models import * def ToGray(x): R = x[:, :, :, 0:1] G = x[:, :, :, 1:2] B = x[:, :, :, 2:3] return 0.30 * R + 0.59 * G + 0.11 * B def RGB2YUV(x): R = x[:, :, :, 0:1] G = x[:, :, :, 1:2] B = x[:, :, :, 2:3] Y = 0.299 * R + 0.587 * G + 0.114 * B U = 0.492 * (B - Y) + 128 V = 0.877 * (R - Y) + 128 return tf.concat([Y, U, V], axis=3) def YUV2RGB(x): Y = x[:, :, :, 0:1] U = x[:, :, :, 1:2] V = x[:, :, :, 2:3] R = Y + 1.140 * (V - 128) G = Y - 0.394 * (U - 128) - 0.581 * (V - 128) B = Y + 2.032 * (U - 128) return tf.concat([R, G, B], axis=3) def VGG2RGB(x): return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1] def blur(x): return Smoother({'data': tf.pad(x, [[0, 0], [9, 9], [9, 9], [0, 0]], 'SYMMETRIC')}, 7, 2).get_output()[:, 9: -9, 9: -9, :] def norm_feature(x, core): cs0 = tf.shape(core)[1] cs1 = tf.shape(core)[2] small = tf.image.resize_area(x, (cs0, cs1)) avged = tf.nn.avg_pool(tf.pad(small, [[0, 0], [2, 2], [2, 2], [0, 0]], 'REFLECT'), [1, 5, 5, 1], [1, 1, 1, 1], 'VALID') return tf.image.resize_bicubic(avged, tf.shape(x)[1:3]) def upsample(x): return K.resize_images(x, 2, 2, 'channels_last') def downsample(x): return tf.nn.avg_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') def nts(x): return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1] / 255.0 session = keras.backend.get_session() ip1 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 1)) ip3 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 3)) ip4 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 4)) print('1') vector = make_diff_net() vector_op = 255.0 - tf.nn.sigmoid(vector(ip3 / 255.0)) * 255.0 print('4') reader = load_model('./nets/reader.net') features = reader(ip3 / 255.0) print('5') head = load_model('./nets/head.net') feed = [1 - ip1 / 255.0, (ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0] for _ in range(len(features)): feed.append(keras.backend.mean(features[_], axis=[1, 2])) nil0, nil1, head_temp = head(feed) print('6') neck = load_model('./nets/neck.net') nil2, nil3, neck_temp = neck(feed) feed[0] = tf.clip_by_value(1 - tf.image.resize_bilinear(ToGray(VGG2RGB(head_temp) / 255.0), tf.shape(ip1)[1:3]), 0.0, 1.0) nil4, nil5, head_temp = neck(feed) head_op = VGG2RGB(head_temp) neck_op = VGG2RGB(neck_temp) print('7') inception = load_model('./nets/inception.net') features_render = inception((ip3 + (downsample(ip1) - blur(downsample(ip1))) * 2.0) / 255.0) precessed_feed = [(ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0] + [ norm_feature(item, features_render[-1]) for item in features_render] print('8') render_head = load_model('./nets/render_head.net') render_neck = load_model('./nets/render_neck.net') nil6, nil7, render_A = render_head([1 - ip1 / 255.0] + precessed_feed) nil8, nil9, render_B = render_neck( [1 - tf.image.resize_bilinear(ToGray(nts(render_A)), tf.shape(ip1)[1:3])] + precessed_feed) render_op = nts(render_B) * 255.0 print('9') tail = load_model('./nets/tail.net') pads = 7 tail_op = tail(tf.pad(ip3 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads * 2:-pads * 2, pads * 2:-pads * 2, :][:, 1:-1, 1:-1, :] * 255.0 print('10') vgg7 = load_model('./nets/vgg7.net') pads = 7 vgg7_op = vgg7(tf.pad(ip1 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads:-pads, pads:-pads, :] * 255.0 print('11') mat = make_unet512() mat_op = mat(ip3 / 255.0) * 255.0 print('11') norm = load_model('./nets/norm.net') norm_op = norm(ip1 / 255.0) * 255.0 print('12') session.run(tf.global_variables_initializer()) print('begin load') tail.load_weights('./nets/tail.net') vgg7.load_weights('./nets/vgg7.net') head.load_weights('./nets/head.net') neck.load_weights('./nets/neck.net') reader.load_weights('./nets/reader.net') vector.load_weights('./nets/vector.net') render_head.load_weights('./nets/render_head.net') render_neck.load_weights('./nets/render_neck.net') inception.load_weights('./nets/inception.net') mat.load_weights('./nets/mat.net') norm.load_weights('./nets/norm.net') def go_head(sketch, global_hint, local_hint): return session.run(head_op, feed_dict={ ip1: sketch[None, :, :, None], ip3: global_hint[None, :, :, :], ip4: local_hint[None, :, :, :] })[0].clip(0, 255).astype(np.uint8) def go_render(sketch, segmentation, points): return session.run(render_op, feed_dict={ ip1: sketch[None, :, :, None], ip3: segmentation[None, :, :, :], ip4: points[None, :, :, :] })[0].clip(0, 255).astype(np.uint8) def go_tail(x): return session.run(tail_op, feed_dict={ ip3: x[None, :, :, :] })[0].clip(0, 255).astype(np.uint8) def go_vgg7(x): return session.run(vgg7_op, feed_dict={ ip1: x[None, :, :, None] })[0, :, :, 0].clip(0, 255).astype(np.uint8) def go_vector(x): return session.run(vector_op, feed_dict={ ip3: x[None, :, :, :] })[0].clip(0, 255).astype(np.uint8) def go_mat(x): return session.run(mat_op, feed_dict={ ip3: x[None, :, :, :] })[0, :, :, 0].clip(0, 255).astype(np.uint8) def go_norm(x): return session.run(norm_op, feed_dict={ ip1: x[None, :, :, None] })[0].clip(0, 255).astype(np.uint8)