TensorFlow Depth Learning 23: Image Style Migration _ depth Learning

First, the paper reference

The methods used here refer mainly to the paper "A Neural algorithm of artistic Style".

In simple terms, the low-level layers of the neural network extract the lower-level information, such as straight lines, corners, etc., the advanced layer extracts more complex content, such as semantic information (the picture is a cat or a dog), the combination of the two can transfer the style of a picture to another picture.

Specific content can refer to the paper.

Second, code implementation
Original Picture:

Style Picture:

This uses the VGG19 model: a pre-training model

The code is as follows (most of which I have commented on):

Import TensorFlow as TF import numpy as NP import OS import cv2 Image_w = Image_h = + Ratio = None # the picture Needed converting content_img = './images/taipei101.jpg ' # the style picture style_img = './images/starrynight.jpg ' # Th  e noise ratio, which is used to generate the initial of the picture Ini_noise_ratio = 0.7 # The balance parameter, which is used To to balance the weights of style loss and content loss Style_strength = + # How many iterations your want to run ITER

ATION = Which layers of the content network you want to use content_layers = [(' Conv4_2 ', 1.)] # Which layers of the ' style network you want to-use style_layers = [(' Conv1_1 ', 2.), (' Conv2_1 ', 1.), (' Conv3_1 ', 0.5), (' Conv4_ 1 ', 0.25), (' Conv5_1 ', 0.125)] # all the layers layers = [' conv1_1 ', ' conv1_2 ', ' conv2_1 ', ' conv2_2 ', ' conv3_1 ', ' conv3_2 ',


' Conv3_3 ', ' conv3_4 ', ' conv4_1 ', ' conv4_2 ', ' conv4_3 ', ' conv4_4 ', ' conv5_1 ', ' conv5_2 ', ' conv5_3 ', ' Conv5_4 ', ' def vgg19 (input, model_path=none): '' Build the VGG19 network, which are initialized with the pre-trained model.
    :p Aram input:the input image.
    :p Aram Model_path:which Path The VGG19 model is stored.
    : Return:a python dict, which contains all layers needed.  ' If Model_path is None:model_path = ' vgg19.npy ' if Os.path.isfile (Model_path) is false:raise

    Filenotfounderror (' vgg19.npy cannot be found!!! ')  Wdict = Np.load (model_path, encoding= "bytes"). Item () NET = {} net[' input ' = input # conv1_1 weight1_1 = Tf. Variable (wdict[' conv1_1 '][0], trainable=false) bias1_1 = tf. Variable (wdict[' conv1_1 '][1], trainable=false) net[' conv1_1 '] = Tf.nn.relu (tf.nn.conv2d (net[' input '), weight1_1, [1, 1, 1, 1], ' SAME ') + bias1_1) # conv1_2 weight1_2 = tf. Variable (wdict[' conv1_2 '][0], trainable=false) bias1_2 = tf. Variable (wdict[' conv1_2 '][1], trainable=false) net[' conv1_2 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv1_1 '), weight1_2, [ 1, 1, 1, 1], 'SAME ') + bias1_2) # pool1 net[' pool1 '] = Tf.nn.avg_pool (net[' conv1_2 '], [1, 2, 2, 1], [1, 2, 2, 1], ' SAME ') # conv2_1 weight2_1 = tf. Variable (wdict[' conv2_1 '][0], trainable=false) bias2_1 = tf. Variable (wdict[' conv2_2 '][1], trainable=false) net[' conv2_1 '] = Tf.nn.relu (tf.nn.conv2d (net[' pool1 '), Weight2_1, [1, 1, 1, 1], ' SAME ') + bias2_1) # conv2_2 weight2_2 = tf. Variable (wdict[' conv2_2 '][0], trainable=false) bias2_2 = tf. Variable (wdict[' conv2_2 '][1], trainable=false) net[' conv2_2 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv2_1 '), weight2_2, [  1, 1, 1, 1], ' SAME ') + bias2_2) # pool2 net[' pool2 '] = Tf.nn.avg_pool (net[' conv2_2 '), [1, 2, 2, 1], [1, 2, 2, 1], ' SAME ') # conv3_1 weight3_1 = tf. Variable (wdict[' Conv3_1 '][0], trainable=false) Bias3_1 = tf. Variable (wdict[' Conv3_1 '][1], trainable=false) net[' conv3_1 '] = Tf.nn.relu (tf.nn.conv2d (net[' pool2 '), Weight3_1, [1, 1, 1, 1], ' SAME ') + bias3_1) # conv3_2 weight3_2 = tf. VariAble (wdict[' conv3_2 '][0], trainable=false) bias3_2 = tf. Variable (wdict[' conv3_2 '][1], trainable=false) net[' conv3_2 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv3_1 '), Weight3_2, [ 1, 1, 1, 1], ' SAME ') + bias3_2) # conv3_3 weight3_3 = tf. Variable (wdict[' Conv3_3 '][0], trainable=false) Bias3_3 = tf. Variable (wdict[' Conv3_3 '][1], trainable=false) net[' conv3_3 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv3_2 '), Weight3_3, [ 1, 1, 1, 1], ' SAME ') + bias3_3) # Conv3_4 weight3_4 = tf. Variable (wdict[' Conv3_4 '][0], trainable=false) Bias3_4 = tf. Variable (wdict[' Conv3_4 '][1], trainable=false) net[' conv3_4 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv3_3 '), Weight3_4, [  1, 1, 1, 1], ' SAME ') + bias3_4) # pool3 net[' pool3 '] = Tf.nn.avg_pool (net[' conv3_4 '), [1, 2, 2, 1], [1, 2, 2, 1], ' SAME ') # conv4_1 weight4_1 = tf. Variable (wdict[' Conv4_1 '][0], trainable=false) Bias4_1 = tf. Variable (wdict[' Conv4_1 '][1], trainable=false) net[' conv4_1 '] = Tf.nn.relu (tf.nn.conv2d(net[' pool3 '], weight4_1, [1, 1, 1, 1], ' SAME ') + bias4_1) # conv4_2 weight4_2 = tf. Variable (wdict[' Conv4_2 '][0], trainable=false) Bias4_2 = tf. Variable (wdict[' Conv4_2 '][1], trainable=false) net[' conv4_2 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv4_1 '), Weight4_2, [ 1, 1, 1, 1], ' SAME ') + bias4_2) # Conv4_3 Weight4_3 = tf. Variable (wdict[' Conv4_3 '][0], trainable=false) Bias4_3 = tf. Variable (wdict[' Conv4_3 '][1], trainable=false) net[' conv4_3 '] = Tf.nn.relu (tf.nn.conv2d (net[' conv4_2 '), Weight4_3, [ 1, 1, 1, 1], ' SAME ') + bias4_3) # conv4_4 weight4_4 = tf. Variable (wdict[' Conv4_4 '][0], trainable=false) Bias4_4 = tf. Variable (wdict[' Conv4_4 '][1], trainable=false) net[' conv4_4 '] = Tf.nn.relu (tf.nn.conv2d (net[' Conv4_3 '), Weight4_4, [  1, 1, 1, 1], ' SAME ') + bias4_4) # pool4 net[' pool4 '] = Tf.nn.avg_pool (net[' conv4_4 '), [1, 2, 2, 1], [1, 2, 2, 1], ' SAME ') # Conv5_1 Weight5_1 = tf. Variable (wdict[' Conv5_1 '][0], trainable=false) bias5_1 = TF. Variable (wdict[' Conv5_1 '][1], trainable=false) net[' conv5_1 '] = Tf.nn.relu (tf.nn.conv2d (net[' Pool4 '), Weight5_1, [1, 1, 1, 1], ' SAME ') + bias5_1) # Conv5_2 weight5_2 = tf. Variable (wdict[' Conv5_2 '][0], trainable=false) Bias5_2 = tf. Variable (wdict[' Conv5_2 '][1], trainable=false) net[' conv5_2 '] = Tf.nn.relu (tf.nn.conv2d (net[' Conv5_1 '), Weight5_2, [ 1, 1, 1, 1], ' SAME ') + bias5_2) # Conv5_3 Weight5_3 = tf. Variable (wdict[' Conv5_3 '][0], trainable=false) Bias5_3 = tf. Variable (wdict[' Conv5_3 '][1], trainable=false) net[' conv5_3 '] = Tf.nn.relu (tf.nn.conv2d (net[' Conv5_2 '), Weight5_3, [ 1, 1, 1, 1], ' SAME ') + bias5_3) # Conv5_4 Weight5_4 = tf. Variable (wdict[' Conv5_4 '][0], trainable=false) Bias5_4 = tf. Variable (wdict[' Conv5_4 '][1], trainable=false) net[' conv5_4 '] = Tf.nn.relu (tf.nn.conv2d (net[' Conv5_3 '), Weight5_4, [  1, 1, 1, 1], ' SAME ') + bias5_4) # pool5 net[' pool5 '] = Tf.nn.avg_pool (net[' Conv5_4 '), [1, 2, 2, 1], [1, 2, 2, 1], ' SAME ') return net Def gram_matrix (tensor, length, depth): "':p Aram tensor:the tensor to convert, W
    Hich could be a numpy array or a tensorflow tensor.
    :p Aram length:the Length you need to convert to.
    :p Aram Depth:the Depth you need to convert to.
    : Return:a tensor. ' tensor = Tf.reshape (tensor, (length, depth)) return Tf.matmul (Tf.transpose (tensor), tensor) pass Def Bui Ld_content_loss (combination, content): "':p Aram Combination:the Network which is the combination of the style NE Twork and content network, this is the style-transfered network:p Aram Content:the network of T He content image:return:The loss between the combination and the content "' Content_sum = 0.0 for I, l
        N Enumerate (content_layers): shape = Combination[l[0]].get_shape () M = Shape[1].value * Shape[2].value N = Shape[3].value Content_sum + + l[1] * 0.25/(M * * 0.5 + N * * 0.5) * TF.Reduce_sum (Tf.pow (combination[l[0]]-content[l[0]], 2)) return Content_sum pass def build_style_loss (combinatio
                       N, Style): ':p Aram Combination:the Network which is the combination of the style network and content network, This is the style-transfered network:p Aram Style:the Network of the style Image:return:
        The loss between the combination and the style ' style_sum = 0.0 for I, L in enumerate (style_layers): Shape = Combination[l[0]].get_shape () M = Shape[1].value * Shape[2].value N = shape[3].value par
        A1 = Combination[l[0]] Para2 = style[l[0]] Sub = Gram_matrix (Para1, M, N)-Gram_matrix (Para2, M, N) sum = Tf.reduce_sum (Tf.pow (sub, 2)) Pre = l[1] * 1.0/(4 * N * * 2 * M * * 2) Style_sum + = Tf.multiply (Pre, sum) return Style_sum pass def Main (): # Define a placeholder myinput = Tf.placeholder (dtype=tf.f Loat32, Shape=[1, Image_H, Image_w, 3]) # Read the style IMAGE raw_styleimg = Cv2.imread (style_img) raw_styleimg = Cv2.resize (raw_sty Leimg, (Image_h, image_w)) styleimg = Np.expand_dims (raw_styleimg, 0) # The normalization method of th style ima
    Ge.
    # Actually, I have tried many methods, and this one is the most useful and powerful. Styleimg[0][0]-= 123 Styleimg[0][1] = 117 styleimg[0][2]-= STYLEIMG = tf.  Variable (styleimg, Dtype=tf.float32, trainable=false) raw_contentimg = Cv2.imread (content_img) # Store the ratio
    The content image. Ratio = Raw_contentimg.shape raw_contentimg = Cv2.resize (raw_contentimg, (Image_h, image_w)) contentimg = Np.expa Nd_dims (raw_contentimg, 0) contentimg[0][0] = 123 contentimg[0][1]-= 117 contentimg[0][2]-= img = tf. Variable (contentimg, Dtype=tf.float32, Trainable=false) # The combination image, which is consisted to noise image an
    D content image. Combination = Ini_noisE_ratio*np.random.uniform ( -20, (1, Image_h, Image_w, 3)). Astype (' float32 ') \ + \ ( 1.-ini_noise_ratio) * contentimg combination = tf. Variable (combination, Dtype=tf.float32, trainable=true) # build all Networks Stylenet = VGG19 (Myinput * style IMG) contentnet = vgg19 (Myinput * contentimg) combinationnet = vgg19 (Myinput * combination) # Define the lo

    SS Function Loss = * Build_style_loss (combinationnet, stylenet) + Build_content_loss (combinationnet, contentnet)

    # here, Adamoptimizer are used, and the learning rate is 2.0 train = Tf.train.AdamOptimizer (2). Minimize (loss)
    # Input Image, consisted of 1s. img = Np.ones (Dtype=np.float32, shape=[1, Image_h, Image_w, 3]) with TF. Session () as Sess:sess.run (Tf.global_variables_initializer ()) to I in range (iteration): PRI

NT (Sess.run (loss, feed_dict={myinput:img}) Sess.run (Train, feed_dict={myinput:img})            # Actually, the combination is the final output. Pic = Sess.run (combination, feed_dict={myinput:img}) [0] pic[0] + = 123 Pic[1] + = 117 P IC[2] + + cv2.imwrite (' results/%d.jpg '% i, cv2.resize (pic, (ratio[1), ratio[0])) If __name__ = ' __main __ ': Main ()

Three, Results
1 iterations:

500 iterations:

1000 iterations:

1500 iterations: