Skip to content
Snippets Groups Projects
Commit 756eda29 authored by Elkhadri Doha's avatar Elkhadri Doha
Browse files

Upload New File

parent 80d13b17
No related branches found
No related tags found
No related merge requests found
mlp.py 0 → 100644
import numpy as np
import matplotlib.pyplot as plt
from read_cifar import read_cifar_batch, read_cifar, split_dataset
N = 30 # number of input data
d_in = 3 # input dimension
d_h = 3 # number of neurons in the hidden layer
d_out = 2 # output dimension (number of neurons of the output layer)
# Random initialization of the network weights and biaises
w1 = 2 * np.random.rand(d_in, d_h) - 1 # first layer weights
b1 = np.zeros((1, d_h)) # first layer biaises
w2 = 2 * np.random.rand(d_h, d_out) - 1 # second layer weights
b2 = np.zeros((1, d_out)) # second layer biaises
data = np.random.rand(N, d_in) # create a random data
targets = np.random.rand(N, d_out) # create a random targets
def sigmoid(x):
return 1 / (1 + np.exp(-np.clip(x, -500, 500)))
def learn_once_mse(w1, b1, w2, b2, data, targets, learning_rate):
# Forward pass
a0 = data # the data are the input of the first layer
z1 = np.matmul(a0, w1) + b1 # input of the hidden layer
a1 = 1 / (1 + np.exp(-z1)) # output of the hidden layer (sigmoid activation function)
z2 = np.matmul(a1, w2) + b2 # input of the output layer
a2 = sigmoid(z2) # output of the output layer (sigmoid activation function)
predictions = a2 # the predicted values are the outputs of the output layer
# Compute loss (MSE)
loss = np.mean(np.square(predictions - targets))
print(loss)
# Backpropagation
dC_da2 = 2 * np.sum(predictions - targets) / N
dC_dz2 = dC_da2 * predictions * (1 - predictions)
dC_dw2 = np.matmul(a1.T, dC_dz2)
dC_db2 = np.sum(dC_dz2, axis=0)
dC_da1 = np.matmul(dC_dz2, w2.T)
dC_dz1 = dC_da1 * a1 * (1 - a1)
dC_dw1 = np.matmul(a0.T, dC_dz1)
dC_db1 = np.sum(dC_dz1, axis=0)
# Update weights and biases
w2 = w2 - learning_rate * dC_dw2
b2 = b2 - learning_rate * dC_db2
w1 = w1 - learning_rate * dC_dw1
b1 = b1 - learning_rate * dC_db1
return w1, b1, w2, b2, loss
#Convert a list into a matrix
def one_hot(labels):
n_samples = len(labels)
n_unique = len(np.unique(labels))
one_hot_matrix = np.zeros((n_samples, n_unique))
one_hot_matrix[np.arange(n_samples), labels] = 1
return one_hot_matrix
def softmax(z):
exp_z = np.exp(z)
sum = exp_z.sum()
softmax_z = exp_z / sum
return softmax_z
def learn_once_cross_entropy(w1, b1, w2, b2, data, labels_train, learning_rate):
a0 = data # the data are the input of the first layer
z1 = np.matmul(a0, w1) + b1 # input of the hidden layer
a1 = 1 / (1 + np.exp(-z1)) # output of the hidden layer (sigmoid activation function)
z2 = np.matmul(a1, w2) + b2 # input of the output layer
a2 = softmax(z2) #Softmax activation layer
predict = a2 # the predicted values are the outputs of the output layer
targets = one_hot(labels_train)
# Compute the loss
loss = loss = -np.sum(targets * np.log(predict))
# Backpropagation
dC_da2 = 2 * np.sum(predict - targets) / N
dC_dz2 = (2 / N) * (predict - targets) * a2 * (1 - predict)
dC_dw2 = np.dot(a1.T, dC_dz2)
dC_db2 = dC_dz2
dC_da1 = np.dot(dC_dz2, w2.T)
dC_dz1 = dC_da1 * a1 * (1 - a1)
dC_dw1 = np.dot(a0.T, dC_dz1)
dC_db1 = np.sum(dC_dz1, axis=0)
# Update weights and biases
w2 = w2 - learning_rate * dC_dw2
b2 = b2 - learning_rate * dC_db2
w1 = w1 - learning_rate * dC_dw1
b1 = b1 - learning_rate * dC_db1
return w1, b1, w2, b2, loss
def accuracy(labels, predictions):
return np.mean(labels == predictions)
def train_mlp(w1, b1, w2, b2, data_train, labels_train, learning_rate, num_epoch):
train_accuracies = []
batch_size, d_in = data_train.shape
train_accuracies = []
N = len(labels_train) # Number of samples in the training set
# Ensure num_epoch is an integer
num_epoch = int(np.isscalar(num_epoch))
for i in range(num_epoch):
# Forward propagation
z1 = np.dot(data_train, w1) + b1
a1 = sigmoid(z1)
z2 = np.dot(a1, w2) + b2
a2 = softmax(z2)
predicted_labels = np.argmax(a2, axis = 1) # the predicted labels
# Compute the accuracy on the training set
train_accuracy = accuracy(labels_train, predicted_labels)
train_accuracies.append(train_accuracy)
# train_accurary = np.mean(np.array(predicted_labels) == np.array(labels_train))
return w1, b1, w2, b2, train_accuracies
def test_mlp(w1, b1, w2, b2, data_test, labels_test):
predictions = []
# Forward propagation
a0 = data_test
z1 = np.dot(a0, w1) + b1
a1 = sigmoid(z1)
z2 = np.dot(a1, w2) + b2
a2 = sigmoid(z2)
predicted_labels_test = np.argmax(a2, axis=1)
test_accuracy = np.mean(predicted_labels_test == labels_test)
return test_accuracy
def run_mlp_training(data_train, labels_train, data_test, labels_test, d_h, learning_rate, num_epochs):
d_in = data_train.shape[1]
d_out = len(np.unique(labels_train))
# Initialize weights and biases
w1 = np.random.randn(d_in, d_h)
b1 = np.zeros((1, d_h))
w2 = np.random.randn(d_h, d_out)
b2 = np.zeros((1, d_out))
# Train MLP
w1, b1, w2, b2, train_accuracies = train_mlp(w1, b1, w2, b2, data_train, labels_train, learning_rate, num_epochs)
# Test MLP
final_test_accuracy = test_mlp(w1, b1, w2, b2, data_test, labels_test)
return train_accuracies, final_test_accuracy
if __name__ == "__main__":
# Create train and test datasets
data, labels = read_cifar(r"C:\Users\etulyon1\OneDrive\Desktop\ECL\Apprentissage profond & Intelligence Artificielle\BE1\image-classification\data\cifar-10-batches-py")
split_factor = 0.9
a, b, c, d = split_dataset(data, labels, split_factor)
# Define the network hyper-parameters and train it
d_h = 64
learning_rate = 0.1
num_epoch = 100
train_accuracies, final_accuracy = run_mlp_training(a, b, c, d, d_h, learning_rate, num_epoch)
print("The accuracy of the network on the test dataset is " + str(final_accuracy) + "%")
# Plot the evolution of the accuracy with the number training steps
plt.plot(train_accuracies)
plt.xlabel("Number of training steps")
plt.ylabel("Accuracy (in %)")
plt.title("Accuracy of the Neural Network depending on the number of iterations of training")
plt.show()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment