feat: add all functions in utils and workflow in a notebook

.gitignore

+/data
+*.pyc
+tests.ipynb
+

utils/binary_cross_entropy.py

+import numpy as np
+
+def binary_cross_entropy(predictions: np.ndarray, targets: np.ndarray):
+    # compute the binary cross-entropy loss (1e-7 is added to avoid log(0))
+    predictions = np.clip(predictions, 1e-7, 1 - 1e-7)
+    term_0 = (1-targets) * np.log(1-predictions)
+    term_1 = targets * np.log(predictions)
+    return -np.mean(term_0+term_1)
\ No newline at end of file

utils/distance_matrix.py

+import numpy as np
+
+def distance_matrix(X: np.ndarray, Y: np.ndarray):
+    # compute the distance matrix between two sets of samples
+    x2 = np.sum(X**2, axis=1, keepdims=True)
+    y2 = np.sum(Y**2, axis=1, keepdims=True)
+    xy = X.dot(Y.T)
+    return np.sqrt(x2 - 2 * xy + y2.T)
\ No newline at end of file

utils/evaluate_knn.py

+from utils.knn_predict import knn_predict
+from utils.distance_matrix import distance_matrix
+import numpy as np
+
+def evaluate_knn(data_train, labels_train, data_test, labels_test, k):
+    # compute the accuracy of the kNN model
+    dists = distance_matrix(data_test, data_train)
+    y_pred = knn_predict(dists, labels_train, k)
+    accuracy = np.mean(y_pred == labels_test)
+    return accuracy
\ No newline at end of file

utils/forward_pass.py

+from utils.sigmoid import sigmoid
+import numpy as np
+
+def forward_pass(w1, b1, w2, b2, data):
+    # compute the forward pass of the MLP with sigmoid activations
+    z1 = np.matmul(data, w1) + b1
+    a1 = sigmoid(z1)
+    z2 = np.matmul(a1, w2) + b2
+    a2 = sigmoid(z2)
+    return a1, a2
\ No newline at end of file

utils/knn_predict.py

+import numpy as np
+
+def knn_predict(dists: np.ndarray, labels_train, k):
+    # predict labels based on k nearest neighbors
+    n = dists.shape[0]
+    y_pred = np.zeros(n, dtype=np.int64)
+    for i in range(n):
+        closest_y = []
+        closest_y = labels_train[np.argsort(dists[i])[:k]]
+        y_pred[i] = np.argmax(np.bincount(closest_y))
+    return y_pred
\ No newline at end of file

utils/learn_once_cross_entropy.py

+import numpy as np
+from utils.forward_pass import forward_pass
+from utils.binary_cross_entropy import binary_cross_entropy
+
+def adjust_weights_binary_cross_entropy(a1, a2, w1, b1, w2, b2, data, targets, learning_rate):
+    batch_size = data.shape[0]
+    dCdZ2 = a2 - targets
+    dCdW2 = np.matmul(a1.T, dCdZ2)  / batch_size
+    dCdB2 = np.sum(dCdZ2, axis=0, keepdims=True) / batch_size
+    dCdA1 = np.matmul(dCdZ2, w2.T)
+    dCdZ1 = dCdA1 * a1 * (1 - a1)
+    dCdW1 = np.matmul(data.T, dCdZ1) / batch_size
+    dCdB1 = np.sum(dCdZ1, axis=0, keepdims=True) / batch_size
+
+    w2 -= learning_rate * dCdW2
+    w1 -= learning_rate * dCdW1
+    b1 -= learning_rate * dCdB1
+    b2 -= learning_rate * dCdB2
+    return w1, b1, w2, b2
+
+def learn_once_cross_entropy(w1,b1,w2,b2,data,targets,learning_rate):
+    a1, a2 = forward_pass(w1, b1, w2, b2, data)
+    loss = binary_cross_entropy(a2, targets)
+    w1, b1, w2, b2 = adjust_weights_binary_cross_entropy(a1, a2, w1, b1, w2, b2, data, targets, learning_rate)
+    return w1, b1, w2, b2, loss
\ No newline at end of file

utils/learn_once_mse.py

+import numpy as np
+from utils.forward_pass import forward_pass
+
+def adjust_weights_mse(a1, a2, w1, b1, w2, b2, data, targets, learning_rate):
+    batch_size = data.shape[0]
+    N_out = targets.shape[1]
+    dCdA2 = 2 * (a2 - targets) / N_out
+    dCdZ2 = dCdA2 * a2 * (1 - a2)
+    dCdW2 = np.matmul(a1.T, dCdZ2) / batch_size 
+    dCdB2 = np.sum(dCdZ2, axis=0, keepdims=True) / batch_size
+    dCdA1 = np.matmul(dCdZ2, w2.T) 
+    dCdZ1 = dCdA1 * a1 * (1 - a1)
+    dCdW1 = np.matmul(data.T, dCdZ1) / batch_size
+    dCdB1 = np.sum(dCdZ1, axis=0, keepdims=True) / batch_size
+
+    w2 -= learning_rate * dCdW2
+    w1 -= learning_rate * dCdW1
+    b1 -= learning_rate * dCdB1
+    b2 -= learning_rate * dCdB2
+    return w1, b1, w2, b2
+
+
+def learn_once_mse(w1: np.ndarray, b1: np.ndarray, w2: np.ndarray, b2: np.ndarray, data: np.ndarray, targets: np.ndarray, learning_rate: float):
+    a1, a2 = forward_pass(w1, b1, w2, b2, data)
+    loss = np.mean(np.square(a2 - targets))
+    w1, b1, w2, b2 = adjust_weights_mse(a1, a2, w1, b1, w2, b2, data, targets, learning_rate)
+    return w1, b1, w2, b2, loss
\ No newline at end of file

utils/mlp_training.py

+import tqdm
+import numpy as np
+from utils.forward_pass import forward_pass
+from utils.one_hot import one_hot
+from utils.learn_once_cross_entropy import learn_once_cross_entropy
+
+
+def train_mlp(w1, b1, w2, b2, data_train, labels_train, learning_rate, num_epochs, batch_size, n_classes):
+    # train the MLP for num_epochs epochs, using batches of size batch_size
+    losses = []
+    for epoch in range(num_epochs):
+        for i in tqdm.tqdm(range(0, data_train.shape[0], batch_size)):
+            data = data_train[i:i+batch_size]
+            targets = one_hot(labels_train[i:i+batch_size], n_classes)
+            w1, b1, w2, b2, loss = learn_once_cross_entropy(w1, b1, w2, b2, data, targets, learning_rate)
+        losses.append(loss)
+        print(f'epoch={epoch}, loss={loss}')
+    return losses, w1, b1, w2, b2
+
+def test_mlp(w1, b1, w2, b2, data_test, labels_test):
+    # test the MLP on data_test, and return the accuracy
+    _, a2 = forward_pass(w1, b1, w2, b2, data_test)
+    predictions = np.argmax(a2, axis=1)
+    test_accuracy = np.mean(predictions == labels_test)
+    return test_accuracy
+
+def initialize_mlp(d_in, d_h, d_out):
+    # initialize the weights and biases of the MLP
+    w1 = 2 * np.random.rand(d_in, d_h) - 1
+    b1 = np.zeros((1, d_h))
+    w2 = 2 * np.random.rand(d_h, d_out) - 1
+    b2 = np.zeros((1, d_out))
+    return w1, b1, w2, b2
+
+def run_mlp_training(data_train, labels_train, data_test, labels_test, d_h, learning_rate, num_epochs, batch_size, n_classes = 10):
+    # run the training and testing of the MLP on data_train and data_test
+    d_in = data_train.shape[1]
+    d_out = np.max(labels_train) + 1
+    w1, b1, w2, b2 = initialize_mlp(d_in, d_h, d_out)
+    losses, w1, b1, w2, b2 = train_mlp(w1, b1, w2, b2, data_train, labels_train, learning_rate, num_epochs, batch_size, n_classes)
+    test_accuracy = test_mlp(w1, b1, w2, b2, data_test, labels_test)
+    return losses, test_accuracy
\ No newline at end of file

utils/one_hot.py

+import numpy as np
+
+def one_hot(labels: np.ndarray, n_classes: int):
+    # convert an array of labels to a one-hot representation
+    n = labels.shape[0]
+    one_hot_labels = np.zeros((n, n_classes))
+    one_hot_labels[np.arange(n), labels] = 1
+    return one_hot_labels
\ No newline at end of file

utils/process_image.py

+import matplotlib.pyplot as plt
+def plot_image_with_label(img, label):
+    # plot image with label
+    plt.imshow(img)
+    plt.title(label)
+    plt.show()
+
+def save_plot_as_image(X, Y, y_label, x_label, save_path):
+    # plot and save image as png
+    plt.figure(figsize=(10,5))
+    plt.plot(X, Y)
+    plt.ylabel(y_label)
+    plt.xlabel(x_label)
+    plt.savefig(save_path)
+    plt.show()
+    plt.close()
+    
\ No newline at end of file

utils/read_cifar.py

+import glob
+import numpy as np
+import pickle
+
+def read_cifar_batch(batch_path):
+    # read a batch of cifar data
+    with open(batch_path, 'rb') as f:
+        batch = pickle.load(f, encoding='bytes')
+    data=np.array(batch[b'data'],dtype=np.float32)/255.0
+    labels=np.array(batch[b'labels'],dtype=np.int64)
+  
+    return data, labels
+
+def read_cifar(directory):
+    # read all cifar data in a directory
+    files = glob.glob(f'{directory}/*_batch*')
+    data = np.empty((0, 3072), dtype=np.float32)
+    labels = np.empty((0), dtype=np.int64)
+    for file in files:
+        batch_data, batch_labels = read_cifar_batch(file)
+        data = np.vstack((data, batch_data))
+        labels = np.hstack((labels, batch_labels))
+    #print(data.shape, labels.shape)
+    return data, labels

utils/sigmoid.py

+import numpy as np
+def sigmoid(x):
+    return 1 / (1 + np.exp(-x))
\ No newline at end of file

utils/split_dataset.py

+import numpy as np
+
+def split_dataset(data, labels, split: float)-> (np.ndarray, np.ndarray, np.ndarray, np.ndarray):
+    # split dataset into train and test
+    assert data.shape[0] == labels.shape[0]
+    # shuffle data
+    indices = np.arange(data.shape[0])
+    np.random.shuffle(indices)
+    data = data[indices]
+    labels = labels[indices]
+    # split data
+    split_index = int(data.shape[0] * split)
+    train_data = data[:split_index]
+    train_labels = labels[:split_index]
+    test_data = data[split_index:]
+    test_labels = labels[split_index:]
+    return train_data, train_labels, test_data, test_labels
\ No newline at end of file