Create a Simple Neural Network from scratch:

Let’s build a simple binary classifier using a neural network with backpropagation. The learning algorithm we’ll implement is known as full-gradient descent. The neural network has two hidden layers, and the first layer uses the ReLU activation function

import numpy as np


class SimpleNeuralNet:

    def __init__(self, alpha=0.01, max_iters=25):
        self.alpha = alpha
        self.max_iters= max_iters

    def fit(self, X, y):
        np.random.seed(123)
        self.hidden_layer_size_0_1 = 2
        self.hidden_layer_size_1_2 = 1
        self.weight_0_1 = np.random.random((X.shape[1], self.hidden_layer_size_0_1))
        self.weight_1_2 = np.random.random(
            (self.hidden_layer_size_0_1, self.hidden_layer_size_1_2)
        )
        self.errors = 0

        for _ in range(self.max_iters):
            for i in range(len(X)):
                input = X[i : i + 1]
                target = y[i : i + 1]
                layer_0_1 = self.relu(input @ self.weight_0_1)
                layer_1_2 = layer_0_1 @ self.weight_1_2

                self.errors += np.sum((layer_1_2 - target) ** 2)

                layer_2_delta = layer_1_2 - target
                layer_1_delta = (
                    layer_2_delta @ self.weight_1_2.T
                ) * self.derivative_of_relu(layer_0_1)

                self.weight_1_2 -= self.alpha * (layer_0_1.T @ layer_2_delta)
                self.weight_0_1 -= self.alpha * (input.T @ layer_1_delta)
            print(f"total error: {self.errors}")
            self.errors = 0

    def relu(self, x):
        return x * (x > 0)

    def derivative_of_relu(self, x):
        return 1 * (x > 0)

    def predict(self, X, y):
        accuracy = 0.0

        for i in range(len(X)):
            layer_0_prediction = self.relu(X[i : i + 1] @ self.weight_0_1)
            layer_1_prediction = layer_0_prediction @ self.weight_1_2
            accuracy += int(layer_1_prediction) == y[i]
        accuracy = accuracy / len(X)
        print(f"accuracy: {accuracy}")

Let’s train our tiny SNN model on the Iris dataset

import pandas as pd

url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'
df = pd.read_csv(url,header=None,encoding='utf-8')
y = df.iloc[0:100, 4].values
y = np.where(y == 'Iris-setosa', 0, 1)
X = df.iloc[0:100, [0,2]].values

snn = SimpleNeuralNet()
snn.fit(X, y.T)

Now, it’s time to test our tiny SNN model on the test dataset

y = df.iloc[100:150, 4].values
y = np.where(y == 'Iris-setosa', 0, 1)
X = df.iloc[100:150, [0,2]].values

snn.predict(X, y)

Now, train our tiny SNN model on the MNIST dataset and measure its performance on the test dataset.

from sklearn.datasets import fetch_openml

mnist = fetch_openml("mnist_784", version=1)
X, y = mnist["data"], mnist["target"]
y = y.astype(np.uint8)
X_train, X_test, y_train, y_test = (
    X[:60000].values,
    X[60000:].values,
    y[:60000].values,
    y[60000:].values,
)
y_train_5 = 1 * (y_train == 5)
y_test_5 = 1 * (y_test == 5)

snn = SimpleNeuralNet()
snn.fit(X_train, y_train_5.T)

snn.predict(X_test, y_test_5)