# Multi-layer Perceptron with tensorflow¶

The 8th KIAS CAC Summer School

Machine Learning Practice II - Multi-layer Neural Networks example

Author: Yung-Kyun Noh, Ph.D.

Many parts are borrowed from Jiseob Kim's Github.

#### We added one more layer from the logistic regression example.¶

In [1]:

    import tensorflow as tf

    import numpy as np
    import matplotlib.pyplot as plt

    %pylab inline

    Populating the interactive namespace from numpy and matplotlib

In [2]:

    n_data = 50
    dim_data = 2
    ## Gaussian mixture data
    data1_1 = np.random.multivariate_normal([4,0], [[1,0],[0,1]], n_data/2)
    data1_2 = np.random.multivariate_normal([3,5], [[1,0],[0,1]], n_data/2)
    data1 = concatenate((data1_1,data1_2), axis=0)
    data2_1 = np.random.multivariate_normal([0,4], [[1,0],[0,1]], n_data/2)
    data2_2 = np.random.multivariate_normal([2,4], [[1,0],[0,1]], n_data/2)
    data2 = concatenate((data2_1,data2_2), axis=0)

    data_x = np.vstack([data1,data2])
    data_y = np.hstack([np.ones((n_data,)), -np.ones((n_data,))])

In [3]:

    print np.ones([2,3])
    n_hiddn = 10
    b1_temp = np.zeros(n_hiddn)
    print b1_temp
    b_temp = np.zeros(1)
    print b_temp
    print np.random.randn(2,5)

    [[ 1.  1.  1.]
     [ 1.  1.  1.]]
    [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
    [ 0.]
    [[-2.38573394 -1.39713717 -0.72588676 -0.79057646  0.47921442]
     [-0.330139   -1.54081108 -0.35276147  0.9526559   1.20514698]]

In [4]:

    def draw_state(n_hiddn, W1val, b1val, wval, bval):
        plt.rcParams['figure.figsize']=(5,5)
        plt.scatter(data1[:,0],data1[:,1],30,'r')
        plt.scatter(data2[:,0],data2[:,1],30,'b')
        [x1min,x1max,x2min,x2max] = plt.axis()
        x1val = np.arange(x1min,x1max,0.1)
        for iHNode in range(n_hiddn):
            plt.plot(x1val, -(W1val[0,iHNode]*x1val+b1val[iHNode])/W1val[1,iHNode], 'k')
            plt.axis([x1min,x1max,x2min,x2max])
        plt.show()

    #n_hiddn = 3
    n_hiddn = 10
    #W1_temp = np.ones([dim_data,n_hiddn])
    #b1_temp = np.zeros(n_hiddn)
    #w_temp = np.ones([n_hiddn,1])
    #b_temp = np.zeros(1)
    W1_temp = np.random.randn(dim_data,n_hiddn)
    b1_temp = np.random.randn(n_hiddn)
    w_temp = np.random.randn(n_hiddn,1)
    b_temp = np.random.randn(1)
    draw_state(n_hiddn, W1_temp, b1_temp, w_temp, b_temp)

## Multilayer Perceptron¶

\begin{eqnarray} P(y|\mathbf{x};\mathbf{w},W_1, b, \mathbf{b}_1) &=& \frac{1}{1 + \exp(-y(\mathbf{w}^\top\phi(\mathbf{x}; W_1,\mathbf{b}_1) + b))}, \quad y\in\{-1,1\}, \\ \phi(\mathbf{x}; W_1,\mathbf{b}) &=& \left(\begin{array}{c} \phi_1(\mathbf{x}; W_1,\mathbf{b}) \\ \vdots \\ \phi_H(\mathbf{x}; W_1,\mathbf{b}) \end{array}\right), \\ \phi_h(\mathbf{x}; W_1,\mathbf{b}) &=& \frac{1}{1 + \exp(-[W_1^\top\mathbf{x} + \mathbf{b}_1]_h)}, \quad h\in\{1,\ldots,H\}\\\\ \end{eqnarray}

The objective function $L$ with $\mathcal{D}=\{\mathbf{x}_i,y_i\}_{i = 1}^N$ is \begin{eqnarray} P(y_1,\ldots,y_N|\mathbf{x}_1,\ldots,\mathbf{x}_N; \mathbf{w}, W_1, b, \mathbf{b}1) = \prod{i = 1}^N P(y = y_i|\mathbf{x} = \mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}1) \end{eqnarray} \begin{eqnarray} \min{\mathbf{w}, W_1, b, \mathbf{b}1} L = -\sum{i = 1}^N \ln P(y_i|\mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}_1) + \lambda (||\mathbf{w}||^2 + ||\mathbf{W_1}||^2) \end{eqnarray}

In [5]:

    # Create the model

    x = tf.placeholder(tf.float32, [None, dim_data])
    W1 = tf.Variable(tf.random_normal([dim_data,n_hiddn]))
    b1 = tf.Variable(tf.random_normal([n_hiddn]))
    W = tf.Variable(tf.random_normal([n_hiddn,1]))
    b = tf.Variable(tf.random_normal([1]))
    y_true = tf.placeholder(tf.float32, [None])

    a1 = tf.matmul(x, W1) + b1
    p_H1 = tf.sigmoid(a1)
    a = tf.matmul(p_H1, W) + b
    p_y = tf.sigmoid(tf.mul(a, tf.reshape(y_true, [-1,1]) ))

    loss = tf.reduce_mean(-tf.log(p_y))

In [6]:

    train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)

In [7]:

    config = tf.ConfigProto()
    config.gpu_options.allow_growth=True
    sess = tf.Session(config=config)
    sess.run(tf.initialize_all_variables())

In [8]:

    print sess.run(W1)
    draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))

    [[ 0.30718651  0.13353711  1.19592011  1.06949592 -0.18721274 -1.85314798
       0.9595266   0.39773613  2.34892774  0.59797508]
     [ 1.16737843  0.06495925  1.06536782  1.21652699  1.08880055  0.18309812
       0.20474984 -0.34250903  1.3155055   0.0371066 ]]

In [9]:

    # Train
    loss_history = []
    for i in range(500):
        sess.run(train_step, feed_dict={x: data_x, y_true: data_y})
        loss_history.append( sess.run(loss, feed_dict={x: data_x, y_true: data_y}) )
        #print sess.run( p_y, feed_dict={x: data_x, y_true: data_y})
        #print sess.run(W), sess.run(b)

    plt.plot(loss_history)

Out[9]:

    [<matplotlib.lines.Line2D at 0x7fe79c5b5b50>]

In [10]:

    print sess.run(W1)

    [[-0.30231872  0.66152674  1.20453513  1.06579745 -0.54890591 -1.79353869
       0.92923242 -1.153615    2.32039809  1.55813456]
     [ 1.30747318  0.31365266  1.04095554  1.22093582  1.29814148 -1.04103601
       0.4476124   0.20019771  1.4619391  -0.23008688]]

In [11]:

    draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))

In [12]:

    import matplotlib.pyplot as plt
    delta = 0.025

    meshx = np.arange(-4.0, 8.0, delta)
    meshy = np.arange(-2.5, 6.5, delta)
    meshX, meshY = np.meshgrid(meshx, meshy)
    contourShape = meshX.shape

    meshgridData = np.concatenate((meshX.reshape((-1,1)), meshY.reshape((-1,1))), axis=1)
    #meshgridData = np.hstack([meshX.reshape((-1,1)), meshY.reshape((-1,1))])

    testp_y = tf.sigmoid(a)
    Z = sess.run(testp_y, feed_dict={x: meshgridData}).reshape(contourShape)

    levels = [0,.5,1]
    CS = plt.contourf(meshX, meshY, Z, levels, colors=('r', 'g'))

    draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))

In [ ]:

In [ ]:

In [ ]:

In [ ]: