Machine learning in python programming

{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Homework 3: Back-propagation Implementation\n", "\n", "1. Implement the sofmax function. 5 pints\n", "\n", "2. Implement the Layer class. 10 points.\n", "\n", "3. Complete the NN class. 70 points.\n", "\n", " 3.1: implement the 'add' function. 5 points\n", " \n", " 3.2: implement the cross-entropy loss. 5 points \n", " \n", " 3.3: implement the forward propagation process. 5 points.\n", " \n", " 3.4: implement the prediction function. 5 points\n", " \n", " 3.5: complete 'train' function. 10 points.\n", " \n", " 3.6: implement the BP algorithm. 30 points\n", " \n", " 3.7: update all weights and bias. 5 points\n", " \n", " 3.8: calculate the accuracy. 5 points \n", " \n", "4. Evaluation. 15 points." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from matplotlib import pyplot as plt\n", "import keras\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "## load the digits dataset\n", "def load_digits(show_sample = True):\n", " (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", " \n", " #show first 100 images\n", " if show_sample == True:\n", " nImg = 4\n", " for i in range(nImg*nImg):\n", " plt.subplot(nImg, nImg, i+1)\n", " plt.imshow(x_train[i], cmap = 'Greys_r')\n", " plt.show()\n", " \n", " x_train_1 = np.reshape(x_train, [x_train.shape[0], x_train.shape[1] * x_train.shape[2]])\n", " x_test_1 = np.reshape(x_test, [x_test.shape[0], x_test.shape[1] * x_test.shape[2]])\n", " \n", " return x_train_1, y_train, x_test_1, y_test\n", "\n", "# x_train, y_train, x_test, y_test = load_wine()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "## Task 1: Activation functions: implement the softmax function. 5 points\n", "def sigm(z):\n", " return 1/(1 + np.exp(-z))\n", "\n", "def dsigm(z):\n", " return sigm(z)*(1 - sigm(z))\n", "\n", "def softmax(z):\n", " ''' softmax function for the output layer\n", "\n", " z: is a vector\n", " '''\n", " #\n", " # " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "## Task 2: create the layer class. 10 points\n", "class Layer:\n", " \"\"\" Regular densely-connected NN layer \n", " \"\"\"\n", " \n", " def __init__(self, units, input_dim, activation = None): # 5 points\n", " '''initialize weights and bias\n", " \n", " units: the number of hidden nodes\n", " '''\n", " self.units = units\n", " self.activation = activation\n", " self.input_dim = input_dim\n", " \n", " np.random.seed(0)\n", " #self.W = \n", " #self.bias = \n", " \n", " #self.gW = # gradient of wights\n", " #self.gb = # gradient of bias\n", " \n", " def run(self, inputs): # 5 points\n", " ''' calculate the net input and activation output of the current layer\n", " \n", " inputs=(n_sample * n_features)\n", " \n", " return the activation output\n", " '''\n", " # self.net = \n", " \n", " if self.activation is 'sigm':\n", " #self.output = \n", " \n", " if self.activation is 'softmax':\n", " #self.output = \n", " \n", " return self.output" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "## Task 3: complete the following NN class. 60 points\n", "class NeuralNetwork:\n", " \n", " def __init__(self):\n", " self.layers=[]\n", " \n", " # Task 3.1: implement the 'add' function. 5 points \n", " def add(self, units, input_dim, activation = 'sigm'):\n", " '''add one layer to neural network\n", " \n", " units: the number of nodes of current layer\n", " input_dim: input dimension (the number of nodes of the previous layer)\n", " activation: the name of the activation function\n", " '''\n", " \n", " #layer = \n", " #\n", " \n", " # Task 3.2: implement the cross-entropy loss. 5 points \n", " def loss(self, y_pred, y):\n", " '''loss function: 1/n_samples*sum_samples(sum_output(-y_i*log(y_pred_i)))\n", " \n", " y_pred: predictions(n_samples * 10)\n", " y: target(one-hot vectors: n_samples * 10)\n", " '''\n", " \n", " m = y.shape[0] # the number of samples\n", " #y_pred = \n", " #loss = \n", " return loss\n", " \n", " # Task 3.3: implement the forward propagation process. 5 points.\n", " def forward_prop(self, inputs):\n", " '''forward propagation calculates net input and output for each layer\n", " \n", " inputs: input data(n_samples * n_features)\n", " return the output of the last layer\n", " \n", " '''\n", " \n", " nLayers = len(self.layers)\n", " \n", " for i in range(nLayers):\n", " #\n", " #\n", " \n", " return out\n", " \n", " # Task 3.4: implement the prediction function. 5 points\n", " def predict_class(self, x):\n", " '''predict class lables (0, 1, 2, 3, ..., 9) for data samples\n", " \n", " x: input(n_samples * n_features) \n", " return class labels\n", " '''\n", " \n", " #\n", " # \n", "\n", " # Task 3.5: complete the following 'train' function. 10 points.\n", " def train(self, inputs, targets, lr = 0.001, batch_size = 32, epochs = 50):\n", " '''implement the SGD process and use Back-Propagation algorithm to calculate gradients \n", " \n", " inputs: training samples\n", " targets: training targets\n", " lr: learning rate\n", " batch_size: batch size\n", " epochs: max number of epochs\n", " '''\n", " \n", " m = len(targets) \n", " #print(m, targets.shape)\n", " loss_hist = np.zeros(epochs)\n", " \n", " for i in range(epochs):\n", " #shuffle the data\n", " idx = np.arange(m)\n", " np.random.shuffle(idx)\n", " inputs = inputs[idx]\n", " targets = targets[idx]\n", " \n", " for b in range(int(m/batch_size)):\n", " b_start= b*batch_size\n", " b_end = min((b+1)*batch_size, m)\n", " \n", " x_batch = inputs[b_start:b_end, :]\n", " y_batch = targets[b_start:b_end, :]\n", " \n", " # 1. run forward propagation\n", " \n", " # 2. call BP to calculate all gradients\n", " \n", " # 3. update all weights and bias\n", " \n", " \n", " lr = lr*0.9 \n", " # 4. calculate and record the loss of current epoch\n", " #\n", " #loss_hist[i] = \n", " \n", " # 5. print out the loss of current epoch\n", " #\n", " \n", " return loss_hist\n", " \n", " # Task 3.6: implement the BP algorithm. 30 points\n", " def BP(self, x, y):\n", " ''' Back-propagation algorithm\n", " \n", " x: input samples (n_samples * n_features)\n", " y: ont-hot targets (n_samples * 10)\n", " '''\n", " \n", " nLayers = len(self.layers)\n", " m_batch = x.shape[0]\n", " \n", " # 1. calculate gradients for the hidden-to-output layer. 15 points\n", " # idx >= 1\n", " idx = nLayers - 1\n", " #h = \n", " #net = \n", " #o = \n", "\n", " \n", " # 2. calculate gradients for the input-to-hidden layers. 15 points\n", " \n", " \n", " # Task 3.7: update all weights and bias. 5 points \n", " def updateWeights(self, lr):\n", " nLayers = len(self.layers)\n", " for i in range(nLayers):\n", " #self.layers[i].W \n", " #self.layers[i].bias \n", " \n", " # Task 3.8: calculate the accuracy. 5 points \n", " def Acc(self, y, y_pred):\n", " '''accuracy\n", " \n", " y: target: categorical values (0, 1, ...9). n_samples * 1\n", " y_pred: prediction: 0,1,2, ..9. n_samples *1\n", " '''\n", " # " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Task 4: Evaluation. 15 points\n", " \n", "# 1. create a one-hidden layer NN. \n", "# The hidden layer activation is sigmoid and the outputlayer activation is the softmax. \n", "# 5 points\n", "#nn = \n", "#nn.add()\n", "#nn.add()\n", "\n", "# 2. train the NN. \n", "x_train, y_train, x_test, y_test = load_digits(show_sample = True)\n", "y_train_onehot = keras.utils.to_categorical(y_train)\n", "# \n", "\n", "# 3. calculte and print out the test and training accuracy. 10 points\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }