Multi-layer Perceptron with theano¶
The 8th KIAS CAC Summer School
Machine Learning Practice II - Multi-layer Neural Networks example
Author: Yung-Kyun Noh, Ph.D.
We added one more layer from the logistic regression example.¶
In [1]:
import theano
import theano.tensor as T
import numpy as np
import matplotlib.pyplot as plt
%pylab inline
floatX = theano.config.floatX
In [2]:
n_data = 50
## Gaussian data
#data1 = np.random.multivariate_normal([4,0], [[1,0],[0,1]], n_data)
#data2 = np.random.multivariate_normal([0,4], [[1,0],[0,1]], n_data)
## Gaussian mixture data
data1_1 = np.random.multivariate_normal([4,0], [[1,0],[0,1]], n_data/2)
data1_2 = np.random.multivariate_normal([3,5], [[1,0],[0,1]], n_data/2)
data1 = concatenate((data1_1,data1_2), axis=0)
data2_1 = np.random.multivariate_normal([0,4], [[1,0],[0,1]], n_data/2)
data2_2 = np.random.multivariate_normal([2,4], [[1,0],[0,1]], n_data/2)
data2 = concatenate((data2_1,data2_2), axis=0)
data_x = np.vstack([data1,data2])
data_y = np.hstack([np.ones((n_data,)), -np.ones((n_data,))])
shared_x = theano.shared(np.asarray(data_x, dtype=floatX), name='data_x')
shared_y = theano.shared(np.asarray(data_y, dtype=floatX), name='data_y')
#n_hiddn = 3
n_hiddn = 10
W1 = theano.shared(np.random.randn(2,n_hiddn), name='W1')
b1 = theano.shared(np.random.randn(n_hiddn), name='b1')
w = theano.shared(np.ones((n_hiddn,1), dtype=floatX), name='w')
b = theano.shared(np.zeros((1,), dtype=floatX), name='b')
def draw_state(n_hiddn):
plt.rcParams['figure.figsize']=(5,5)
plt.scatter(data1[:,0],data1[:,1],30,'r')
plt.scatter(data2[:,0],data2[:,1],30,'b')
[x1min,x1max,x2min,x2max] = plt.axis()
x1val = np.arange(x1min,x1max,0.1)
W1val = W1.get_value(borrow=True)
b1val = b1.get_value(borrow=True)
for iHNode in range(n_hiddn):
plt.plot(x1val, -(W1val[0,iHNode]*x1val+b1val[iHNode])/W1val[1,iHNode], 'k')
plt.axis([x1min,x1max,x2min,x2max])
plt.show()
draw_state(n_hiddn)
Multilayer Perceptron¶
\begin{eqnarray}
P(y|\mathbf{x};\mathbf{w},W_1, b, \mathbf{b}_1) &=& \frac{1}{1 + \exp(-y(\mathbf{w}^\top\phi(\mathbf{x}; W_1,\mathbf{b}_1) + b))}, \quad y\in\{-1,1\}, \\
\phi(\mathbf{x}; W_1,\mathbf{b}) &=& \left(\begin{array}{c}
\phi_1(\mathbf{x}; W_1,\mathbf{b}) \\
\vdots \\
\phi_H(\mathbf{x}; W_1,\mathbf{b})
\end{array}\right), \\
\phi_h(\mathbf{x}; W_1,\mathbf{b}) &=& \frac{1}{1 + \exp(-[W_1^\top\mathbf{x} + \mathbf{b}_1]_h)}, \quad h\in\{1,\ldots,H\}\\\\
\end{eqnarray}
The objective function $L$ with $\mathcal{D}=\{\mathbf{x}_i,y_i\}_{i = 1}^N$ is \begin{eqnarray} P(y_1,\ldots,y_N|\mathbf{x}_1,\ldots,\mathbf{x}_N; \mathbf{w}, W_1, b, \mathbf{b}1) = \prod{i = 1}^N P(y = y_i|\mathbf{x} = \mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}1) \end{eqnarray} \begin{eqnarray} \min{\mathbf{w}, W_1, b, \mathbf{b}1} L = -\sum{i = 1}^N \ln P(y_i|\mathbf{x}_i; \mathbf{w}, W_1, b, \mathbf{b}_1) + \lambda (||\mathbf{w}||^2 + ||\mathbf{W_1}||^2) \end{eqnarray}
In [3]:
x = T.matrix('x')
y = T.vector('y')
p_H = 1/(1+T.exp(-(T.dot(x, W1) + b1)))
p_y = 1/(1+T.exp(-(T.dot(p_H, w) + b)*T.reshape(y,(-1,1))))
loss = T.mean(-T.log(p_y)) + 10**-3*(w.norm(2) + W1.norm(2))
Gradient Descent¶
In [4]:
wgrad = T.grad(loss, w)
bgrad = T.grad(loss, b)
W1grad = T.grad(loss, W1)
b1grad = T.grad(loss, b1)
lr = 0.1
train = theano.function([], loss, givens=[(x,shared_x), (y,shared_y)],
updates=[(W1,W1-lr*W1grad), (b1,b1-lr*b1grad), (w,w-lr*wgrad), (b,b-lr*bgrad)])
In [5]:
for epoch in xrange(100):
loss_val = train()
draw_state(n_hiddn)
print('loss: {}, w norm: {}, W1.norm: {}'.format(loss_val,
np.sqrt(np.sum(w.get_value()**2)),
np.sqrt(np.sum(np.sum(W1.get_value()**2, axis=0)))))
In [6]:
p_y_pred = 1/(1+T.exp(-(T.dot(p_H, w) + b)))
predict = theano.function([x], p_y_pred, allow_input_downcast=True)
print predict([[0,3],[4,-2],[3,6],[10,-10]])
print predict(data1).T
print predict(data2).T
In [7]:
import matplotlib.pyplot as plt
delta = 0.025
x = np.arange(-4.0, 8.0, delta)
y = np.arange(-2.5, 6.5, delta)
X, Y = np.meshgrid(x, y)
contourShape = X.shape
# print X.reshape((-1,1))
meshgridData = np.concatenate((X.reshape((-1,1)), Y.reshape((-1,1))), axis=1)
Z = predict(meshgridData).reshape(contourShape)
levels = [0,.5,1]
CS = plt.contourf(X, Y, Z, levels, colors=('r', 'g'))
draw_state(n_hiddn)
