Multi-layer Perceptron with tensorflow

The 8th KIAS CAC Summer School

Machine Learning Practice II - Multi-layer Neural Networks example

Author: Yung-Kyun Noh, Ph.D.

Many parts are borrowed from Jiseob Kim's Github.

We added one more layer from the logistic regression example.

In [1]:
import tensorflow as tf

import numpy as np
import matplotlib.pyplot as plt

%pylab inline

Populating the interactive namespace from numpy and matplotlib
In [2]:
n_data = 50
dim_data = 2
## Gaussian mixture data
data1_1 = np.random.multivariate_normal([4,0], [[1,0],[0,1]], n_data/2)
data1_2 = np.random.multivariate_normal([3,5], [[1,0],[0,1]], n_data/2)
data1 = concatenate((data1_1,data1_2), axis=0)
data2_1 = np.random.multivariate_normal([0,4], [[1,0],[0,1]], n_data/2)
data2_2 = np.random.multivariate_normal([2,4], [[1,0],[0,1]], n_data/2)
data2 = concatenate((data2_1,data2_2), axis=0)

data_x = np.vstack([data1,data2])
data_y = np.hstack([np.ones((n_data,)), -np.ones((n_data,))])
In [3]:
print np.ones([2,3])
n_hiddn = 10
b1_temp = np.zeros(n_hiddn)
print b1_temp
b_temp = np.zeros(1)
print b_temp
print np.random.randn(2,5)

[[ 1.  1.  1.]
 [ 1.  1.  1.]]
[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
[ 0.]
[[-2.38573394 -1.39713717 -0.72588676 -0.79057646  0.47921442]
 [-0.330139   -1.54081108 -0.35276147  0.9526559   1.20514698]]
In [4]:
def draw_state(n_hiddn, W1val, b1val, wval, bval):
    plt.rcParams['figure.figsize']=(5,5)
    plt.scatter(data1[:,0],data1[:,1],30,'r')
    plt.scatter(data2[:,0],data2[:,1],30,'b')
    [x1min,x1max,x2min,x2max] = plt.axis()
    x1val = np.arange(x1min,x1max,0.1)
    for iHNode in range(n_hiddn):
        plt.plot(x1val, -(W1val[0,iHNode]*x1val+b1val[iHNode])/W1val[1,iHNode], 'k')
        plt.axis([x1min,x1max,x2min,x2max])
    plt.show()

#n_hiddn = 3
n_hiddn = 10
#W1_temp = np.ones([dim_data,n_hiddn])
#b1_temp = np.zeros(n_hiddn)
#w_temp = np.ones([n_hiddn,1])
#b_temp = np.zeros(1)
W1_temp = np.random.randn(dim_data,n_hiddn)
b1_temp = np.random.randn(n_hiddn)
w_temp = np.random.randn(n_hiddn,1)
b_temp = np.random.randn(1)
draw_state(n_hiddn, W1_temp, b1_temp, w_temp, b_temp)

Multilayer Perceptron

\begin{eqnarray} P(y|\mathbf{x};\mathbf{w},W_1, b, \mathbf{b}_1) &=& \frac{1}{1 + \exp(-y(\mathbf{w}^\top\phi(\mathbf{x}; W_1,\mathbf{b}_1) + b))}, \quad y\in\{-1,1\}, \\ \phi(\mathbf{x}; W_1,\mathbf{b}) &=& \left(\begin{array}{c} \phi_1(\mathbf{x}; W_1,\mathbf{b}) \\ \vdots \\ \phi_H(\mathbf{x}; W_1,\mathbf{b}) \end{array}\right), \\ \phi_h(\mathbf{x}; W_1,\mathbf{b}) &=& \frac{1}{1 + \exp(-[W_1^\top\mathbf{x} + \mathbf{b}_1]_h)}, \quad h\in\{1,\ldots,H\}\\\\ \end{eqnarray}

The objective function $L$ with $\mathcal{D}=\{\mathbf{x}_i,y_i\}_{i = 1}^N$ is \begin{eqnarray} P(y_1,\ldots,y_N|\mathbf{x}_1,\ldots,\mathbf{x}_N; \mathbf{w}, W_1, b, \mathbf{b}1) = \prod{i = 1}^N P(y = y_i|\mathbf{x} = \mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}1) \end{eqnarray} \begin{eqnarray} \min{\mathbf{w}, W_1, b, \mathbf{b}1} L = -\sum{i = 1}^N \ln P(y_i|\mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}_1) + \lambda (||\mathbf{w}||^2 + ||\mathbf{W_1}||^2) \end{eqnarray}

In [5]:
# Create the model

x = tf.placeholder(tf.float32, [None, dim_data])
W1 = tf.Variable(tf.random_normal([dim_data,n_hiddn]))
b1 = tf.Variable(tf.random_normal([n_hiddn]))
W = tf.Variable(tf.random_normal([n_hiddn,1]))
b = tf.Variable(tf.random_normal([1]))
y_true = tf.placeholder(tf.float32, [None])

a1 = tf.matmul(x, W1) + b1
p_H1 = tf.sigmoid(a1)
a = tf.matmul(p_H1, W) + b
p_y = tf.sigmoid(tf.mul(a, tf.reshape(y_true, [-1,1]) ))

loss = tf.reduce_mean(-tf.log(p_y))
In [6]:
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
In [7]:
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
sess.run(tf.initialize_all_variables())
In [8]:
print sess.run(W1)
draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))

[[ 0.30718651  0.13353711  1.19592011  1.06949592 -0.18721274 -1.85314798
   0.9595266   0.39773613  2.34892774  0.59797508]
 [ 1.16737843  0.06495925  1.06536782  1.21652699  1.08880055  0.18309812
   0.20474984 -0.34250903  1.3155055   0.0371066 ]]
In [9]:
# Train
loss_history = []
for i in range(500):
    sess.run(train_step, feed_dict={x: data_x, y_true: data_y})
    loss_history.append( sess.run(loss, feed_dict={x: data_x, y_true: data_y}) )
    #print sess.run( p_y, feed_dict={x: data_x, y_true: data_y})
    #print sess.run(W), sess.run(b)
    
plt.plot(loss_history)
Out[9]:
[<matplotlib.lines.Line2D at 0x7fe79c5b5b50>]
In [10]:
print sess.run(W1)

[[-0.30231872  0.66152674  1.20453513  1.06579745 -0.54890591 -1.79353869
   0.92923242 -1.153615    2.32039809  1.55813456]
 [ 1.30747318  0.31365266  1.04095554  1.22093582  1.29814148 -1.04103601
   0.4476124   0.20019771  1.4619391  -0.23008688]]
In [11]:
draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))
In [12]:
import matplotlib.pyplot as plt
delta = 0.025

meshx = np.arange(-4.0, 8.0, delta)
meshy = np.arange(-2.5, 6.5, delta)
meshX, meshY = np.meshgrid(meshx, meshy)
contourShape = meshX.shape

meshgridData = np.concatenate((meshX.reshape((-1,1)), meshY.reshape((-1,1))), axis=1)
#meshgridData = np.hstack([meshX.reshape((-1,1)), meshY.reshape((-1,1))])

testp_y = tf.sigmoid(a)
Z = sess.run(testp_y, feed_dict={x: meshgridData}).reshape(contourShape)

levels = [0,.5,1]
CS = plt.contourf(meshX, meshY, Z, levels, colors=('r', 'g'))

draw_state(n_hiddn, sess.run(W1), sess.run(b1), sess.run(W), sess.run(b))
In [ ]:
In [ ]:
In [ ]:
In [ ]: