Practical_sessions/Project/chromadb_example.ipynb

+%% Cell type:markdown id: tags:
+
+# Chroma database
+
+Chroma is an open-source vector database that is similar to Milvus and can be used with Windows systems. Here is an example of code illustrating its use.
+
+%% Cell type:code id: tags:
+
+``` python
+# Installing the chromadb package
+!pip install chromadb
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Importing the necessary module
+from chromadb import PersistentClient
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Creating a database client stored in the "ragdb" folder, or loading it if it already exists
+client = PersistentClient(path="./ragdb")
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Creating or loading a collection in ChromaDB
+collection_name = "my_rag_collection"
+try:
+    collection = client.get_collection(name=collection_name)
+except:
+    collection = client.create_collection(name=collection_name)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+from sentence_transformers import SentenceTransformer
+
+# Load an embedding model
+embedding_model = SentenceTransformer("BAAI/bge-small-en-v1.5")
+
+# Define an embedding function
+def text_embedding(text):
+    return embedding_model.encode(text).tolist()
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Adding documents with their metadata and unique identifiers
+documents = [
+    "The sun rises in the east and sets in the west.",
+    "Raindrops create soothing sounds as they hit the ground.",
+    "Stars twinkle brightly in the clear night sky.",
+    "The ocean waves crash gently against the shore.",
+    "Mountains stand tall and majestic, covered in snow.",
+    "Birds chirp melodiously during the early morning hours.",
+    "The forest is alive with the sounds of rustling leaves and wildlife.",
+    "A gentle breeze flows through the meadow, carrying the scent of flowers."
+]
+embeddings = [text_embedding(document) for document in documents]
+ids = [f"{i}" for i in range(len(documents))]
+
+collection.add(
+    documents=documents,
+    embeddings=embeddings,
+    ids=ids
+)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Querying to find the documents most similar to a given phrase
+query = "What happens in the forest during the day?"
+# query = "Describe how stars appear in a clear night sky."
+
+query_embedding = text_embedding(query)
+
+results = collection.query(
+    query_embeddings=[query_embedding],
+    n_results=2  # Number of desired similar results
+)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Displaying the results
+for result in results['documents']:
+    print("Similar document:", result)
+```

Practical_sessions/Session_2/nn_classification-completed.py

+import matplotlib.pyplot as plt
+import numpy as np
+
+def load_data(file_name, delimiter=','):
+    """ Reads the file containing the data and returns the matrices corresponding to it
+
+    Parameters
+    ----------
+    file_name : name of the file containing the data
+    delimiter : character that separates the columns in the file (default is ",")
+
+    Returns
+    -------
+    x : data matrix of dimension [N, nb_var]
+    d : matrix containing the target variable values of dimension [N, nb_target]
+    N : number of elements
+    nb_var : number of predictor variables
+    nb_target : number of target variables
+
+    """
+    
+    data = np.loadtxt(file_name, delimiter=delimiter)
+    
+    nb_target = 1
+    nb_var = data.shape[1] - nb_target
+    N = data.shape[0]
+
+    x = data[:, :nb_var]
+    d = data[:, nb_var:].reshape(N,1)
+    
+    return x, d, N, nb_var, nb_target
+
+
+def normalization(x):
+    """ Normalizes the data by centering and scaling the predictor variables
+    
+    Parameters
+    ----------
+    X : data matrix of dimension [N, nb_var]
+    
+    with N : number of elements and nb_var : number of predictor variables
+
+    Returns
+    -------
+    X_norm : normalized data matrix of dimension [N, nb_var]
+    mu : mean of the variables of dimension [1, nb_var]
+    sigma : standard deviation of the variables of dimension [1, nb_var]
+    
+    """
+    
+    mu = np.mean(x, 0)
+    sigma = np.std(x, 0)
+    x_norm = (x - mu) / sigma
+
+    return x_norm, mu, sigma
+
+def split_data(x,d,prop_val=0.2, prop_test=0.2):
+    """ Splits the original data into three distinct subsets: training, validation, and test
+    
+    Parameters
+    ----------
+    x : data matrix of dimension [N, nb_var]
+    d : target values matrix [N, nb_target]
+    prop_val : proportion of validation data in the total data (between 0 and 1)
+    prop_test : proportion of test data in the total data (between 0 and 1)
+    
+    with N : number of elements, nb_var : number of predictor variables, nb_target : number of target variables
+
+    Returns
+    -------
+    x_train : training data matrix
+    d_train : target values matrix for training
+    x_val : validation data matrix
+    d_val : target values matrix for validation
+    x_test : test data matrix
+    d_test : target values matrix for test
+
+    """
+    assert prop_val + prop_test < 1.0
+
+    N = x.shape[0]
+    indices = np.arange(N)
+    np.random.shuffle(indices)
+    nb_val = int(N*prop_val)
+    nb_test = int(N*prop_test)
+    nb_train = N - nb_val - nb_test
+
+    x = x[indices,:]
+    d = d[indices,:]
+
+    x_train = x[:nb_train,:]
+    d_train = d[:nb_train,:]
+
+    x_val = x[nb_train:nb_train+nb_val,:]
+    d_val = d[nb_train:nb_train+nb_val,:]
+
+    x_test = x[N-nb_test:,:]
+    d_test = d[N-nb_test:,:]
+
+    return x_train, d_train, x_val, d_val, x_test, d_test
+
+def compute_cross_entropy_cost(y, d):
+    """ Computes the value of the cross-entropy cost function
+    
+    Parameters
+    ----------
+    y : predicted data matrix (softmax)
+    d : actual data matrix encoded by 1
+    
+    Returns
+    -------
+    cost : value corresponding to the cost function
+
+    """
+
+    N = y.shape[1]
+    
+    cost = - np.sum(d*np.log(y)) / N
+
+    return cost
+
+def forward_pass(x, W, b, activation):
+    """ Performs a forward pass in the neural network
+    
+    Parameters
+    ----------
+    x : input matrix, dimension nb_var x N
+    W : list containing the weight matrices of the network
+    b : list containing the bias matrices of the network
+    activation : list containing the activation functions of the network layers
+
+    with N : number of elements, nb_var : number of predictor variables 
+
+    Returns
+    -------
+    a : list containing the input potentials of the network layers
+    h : list containing the outputs of the network layers
+
+    """
+    h = [x]
+    a = []
+    for i in range(len(b)):
+        a.append( W[i].dot(h[i]) + b[i] )
+        h.append( activation[i](a[i]) ) 
+
+    return a, h
+
+def backward_pass(delta_h, a, h, W, activation):
+    """ Performs a backward pass in the neural network (backpropagation)
+    
+    Parameters
+    ----------
+    delta_h : matrix containing the gradient of the cost with respect to the network output
+    a : list containing the input potentials of the network layers
+    h : list containing the outputs of the network layers
+    W : list containing the weight matrices of the network
+    activation : list containing the activation functions of the network layers
+
+    Returns
+    -------
+    delta_W : list containing the gradient matrices of the network's weight layers
+    delta_b : list containing the gradient matrices of the network's bias layers
+
+    """
+
+    delta_b = []
+    delta_W = []
+
+    for i in range(len(W)-1,-1,-1):
+
+        delta_a = delta_h * activation[i](a[i], True)
+     
+        delta_b.append( delta_a.mean(1).reshape(-1,1) ) 
+        delta_W.append( delta_a.dot(h[i].T) ) 
+
+        delta_h = (W[i].T).dot(delta_a) 
+
+    delta_b = delta_b[::-1]
+    delta_W = delta_W[::-1]
+
+    return delta_W, delta_b
+
+def sigmoid(z, deriv=False):
+    """ Computes the value of the sigmoid function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the sigmoid function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the sigmoid function applied to z or its derivative. Same dimension as z
+
+    """
+
+    s = 1 / (1 + np.exp(-z))
+    if deriv:
+        return s * (1 - s)
+    else :
+        return s
+
+def linear(z, deriv=False):
+    """ Computes the value of the linear function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the linear function, if True returns its derivative
+
+
+    Returns
+    -------
+    s : value of the linear function applied to z or its derivative. Same dimension as z
+
+    """
+    if deriv:       
+        return 1     
+    else :
+        return z
+
+
+def relu(z, deriv=False):
+    """ Computes the value of the ReLU function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the ReLU function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the ReLU function applied to z or its derivative. Same dimension as z
+
+    """
+
+    r = np.zeros(z.shape)
+    if deriv:
+        pos = np.where(z>=0)
+        r[pos] = 1.0
+        return r
+    else :    
+        return np.maximum(r,z)
+
+def softmax(z, deriv=False):
+    """ Computes the value of the softmax function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : data matrix
+    deriv : boolean. If False returns the value of the softmax function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the softmax function applied to z or its derivative. Same dimension as z
+
+    """
+    
+    if deriv:
+        return 1
+    else :        
+        return np.exp(z) / np.sum(np.exp(z),axis=0)
+
+def one_hot_encoding(d):
+    """ Performs a one-hot encoding: for the output neurons of the network, only 1 will have the value 1, all others will be 0
+    
+    Parameters
+    ----------
+    d : matrix containing the values of the target variable (class of the elements) of dimension [N, 1]
+    with N : number of elements
+
+    Returns
+    -------
+    e : encoded data matrix of dimension [N, nb_classes]
+    with N : number of elements and nb_classes the number of classes (maximum+1) of the values in d
+
+    """    
+
+    d = d.astype(int).flatten()
+    N = d.shape[0]
+    nb_classes = d.max() + 1
+
+    e = np.zeros((N,nb_classes))
+    e[range(N),d] = 1
+
+    return e
+
+def classification_accuracy(y,d):
+    """ Computes the classification accuracy (proportion of correctly classified elements)
+    
+    Parameters
+    ----------
+    y : network outputs matrix of dimension [nb_output_neurons x N]
+    d : true values matrix [nb_output_neurons x N]
+    
+    with N : number of elements and nb_output_neurons : number of neurons in the output layer
+
+    Returns
+    -------
+    t : classification accuracy
+
+    """
+
+    ind_y = np.argmax(y,axis=0)
+    ind_d = np.argmax(d,axis=0)
+    t = np.mean(ind_y == ind_d)
+    
+    return t
+
+
+# ===================== Part 1: Reading and Normalizing the Data =====================
+print("Reading the data ...")
+
+x, d, N, nb_var, nb_target = load_data("iris.txt")
+# x, d, N, nb_var, nb_target = load_data("scores.txt")
+
+# Display the first 10 examples of the dataset
+print("Displaying the first 10 examples of the dataset: ")
+for i in range(0, 10):
+    print(f"x = {x[i,:]}, d = {d[i]}")
+    
+# Normalization of the variables (centering and scaling)
+print("Normalizing the variables ...")
+x, mu, sigma = normalization(x)
+d = one_hot_encoding(d)
+
+# Split the data into training, validation, and test subsets
+x_train, d_train, x_val, d_val, x_test, d_test = split_data(x,d)
+
+# ===================== Part 2: Training =====================
+
+# Learning rate and number of iterations
+alpha = 0.0001
+nb_iters = 10000
+training_costs = np.zeros(nb_iters)
+validation_costs = np.zeros(nb_iters)
+
+# Network dimensions
+D_c = [nb_var, 15, 15,  d_train.shape[1]] # list containing the number of neurons for each layer 
+activation = [relu, relu, softmax] # list containing the activation functions for hidden layers and output layer
+
+# Random initialization of network weights
+W = []
+b = []
+for i in range(len(D_c)-1):    
+    W.append(2 * np.random.random((D_c[i+1], D_c[i])) - 1)
+    b.append(np.zeros((D_c[i+1],1)))
+
+x_train = x_train.T # Data is presented as column vectors at the network input
+d_train = d_train.T 
+
+x_val = x_val.T # Data is presented as column vectors at the network input
+d_val = d_val.T 
+
+x_test = x_test.T # Data is presented as column vectors at the network input
+d_test = d_test.T 
+
+for t in range(nb_iters):
+
+    #############################################################################
+    # Forward pass: calculate predicted output y on validation data #
+    #############################################################################
+    a, h = forward_pass(x_val, W, b, activation)
+    y_val = h[-1] # Predicted output
+
+    ###############################################################################
+    # Forward pass: calculate predicted output y on training data #
+    ###############################################################################
+    a, h = forward_pass(x_train, W, b, activation)
+    y_train = h[-1] # Predicted output
+
+    ###########################################
+    # Compute Mean Squared Error loss function #
+    ###########################################
+    training_costs[t] = compute_cross_entropy_cost(y_train,d_train)
+    validation_costs[t] = compute_cross_entropy_cost(y_val,d_val)
+
+    ####################################
+    # Backward pass: backpropagation #
+    ####################################
+    delta_h = (y_train-d_train) # For the last layer 
+    delta_W, delta_b = backward_pass(delta_h, a, h, W, activation)
+  
+    #############################################
+    # Update weights and biases  ##### #
+    ############################################# 
+    for i in range(len(b)-1,-1,-1):
+        b[i] -= alpha * delta_b[i]
+        W[i] -= alpha * delta_W[i]
+
+print("Final cost on the training set: ", training_costs[-1])
+print("Classification accuracy on the training set: ", classification_accuracy(y_train, d_train))
+
+print("Final cost on the validation set: ", validation_costs[-1])
+print("Classification accuracy on the validation set: ", classification_accuracy(y_val, d_val))
+
+# Display cost function evolution during backpropagation
+plt.figure(0)
+plt.title("Cost function evolution during backpropagation")
+plt.plot(np.arange(training_costs.size), training_costs, label="Training")
+plt.plot(np.arange(validation_costs.size), validation_costs, label="Validation")
+plt.legend(loc="upper left")
+plt.xlabel("Number of iterations")
+plt.ylabel("Cost")
+plt.show()
+
+# ===================== Part 3: Evaluation on the test set =====================
+
+#######################################################################
+# Forward pass: calculate predicted output y on test data #
+#######################################################################
+a, h = forward_pass(x_test, W, b, activation)
+y_test = h[-1] # Predicted output
+
+cost = compute_cross_entropy_cost(y_test,d_test)
+print("Cost on the test set: ", cost)
+print("Classification accuracy on the test set: ", classification_accuracy(y_test, d_test))

Practical_sessions/Session_2/nn_regression-completed.py

+
+import matplotlib.pyplot as plt
+import numpy as np
+
+def read_data(file_name, delimiter=','):
+    """ Reads the file containing the data and returns the corresponding matrices
+
+    Parameters
+    ----------
+    file_name : name of the file containing the data
+    delimiter : character separating columns in the file ("," by default)
+
+    Returns
+    -------
+    x : data matrix of size [N, num_vars]
+    d : matrix containing the target variable values of size [N, num_targets]
+    N : number of elements
+    num_vars : number of predictor variables
+    num_targets : number of target variables
+
+    """
+    
+    data = np.loadtxt(file_name, delimiter=delimiter)
+    
+    num_targets = 1
+    num_vars = data.shape[1] - num_targets
+    N = data.shape[0]
+
+    x = data[:, :num_vars]
+    d = data[:, num_vars:].reshape(N,1)
+    
+    return x, d, N, num_vars, num_targets
+
+def normalization(x):
+    """ Normalizes the data by centering and scaling the predictor variables
+    
+    Parameters
+    ----------
+    X : data matrix of size [N, num_vars]
+    
+    with N : number of elements and num_vars : number of predictor variables
+
+    Returns
+    -------
+    X_norm : centered-scaled data matrix of size [N, num_vars]
+    mu : mean of the variables of size [1, num_vars]
+    sigma : standard deviation of the variables of size [1, num_vars]
+    
+    """
+    
+    mu = np.mean(x, 0)
+    sigma = np.std(x, 0)
+    x_norm = (x - mu) / sigma
+
+    return x_norm, mu, sigma
+
+def split_data(x, d, val_prop=0.2, test_prop=0.2):
+    """ Splits the initial data into three distinct subsets for training, validation, and testing
+    
+    Parameters
+    ----------
+    x : data matrix of size [N, num_vars]
+    d : matrix of target values [N, num_targets]
+    val_prop : proportion of validation data over the entire dataset (between 0 and 1)
+    test_prop : proportion of test data over the entire dataset (between 0 and 1)
+    
+    with N : number of elements, num_vars : number of predictor variables, num_targets : number of target variables
+
+    Returns
+    -------
+    x_train : training data matrix
+    d_train : training target values matrix
+    x_val : validation data matrix
+    d_val : validation target values matrix
+    x_test : test data matrix
+    d_test : test target values matrix
+
+    """
+    assert val_prop + test_prop < 1.0
+
+    N = x.shape[0]
+    indices = np.arange(N)
+    np.random.shuffle(indices)
+    num_val = int(N*val_prop)
+    num_test = int(N*test_prop)
+    num_train = N - num_val - num_test
+
+    x = x[indices,:]
+    d = d[indices,:]
+
+    x_train = x[:num_train,:]
+    d_train = d[:num_train,:]
+
+    x_val = x[num_train:num_train+num_val,:]
+    d_val = d[num_train:num_train+num_val,:]
+
+    x_test = x[N-num_test:,:]
+    d_test = d[N-num_test:,:]
+
+    return x_train, d_train, x_val, d_val, x_test, d_test
+
+def calculate_mse_cost(y, d):
+    """ Calculates the value of the MSE (mean squared error) cost function
+    
+    Parameters
+    ----------
+    y : matrix of predicted data 
+    d : matrix of actual data 
+    
+    Returns
+    -------
+    cost : value corresponding to the MSE cost function (mean squared error)
+
+    """
+
+    N = y.shape[1]    
+    cost = np.square(y - d).sum() / 2 / N
+
+    return cost
+
+def forward_pass(x, W, b, activation):
+    """ Performs a forward pass in the neural network
+    
+    Parameters
+    ----------
+    x : input matrix, of size num_vars x N
+    W : list containing the weight matrices of the network
+    b : list containing the bias matrices of the network
+    activation : list containing the activation functions of the network layers
+
+    with N : number of elements, num_vars : number of predictor variables 
+
+    Returns
+    -------
+    a : list containing the input potentials of the network layers
+    h : list containing the outputs of the network layers
+
+    """
+    h = [x]
+    a = []
+    for i in range(len(b)):
+        a.append( W[i].dot(h[i]) + b[i] )
+        h.append( activation[i](a[i]) ) 
+
+    return a, h
+
+def backward_pass(delta_h, a, h, W, activation):
+    """ Performs a backward pass in the neural network (backpropagation)
+    
+    Parameters
+    ----------
+    delta_h : matrix containing the gradient of the cost with respect to the output of the network
+    a : list containing the input potentials of the network layers
+    h : list containing the outputs of the network layers
+    W : list containing the weight matrices of the network
+    activation : list containing the activation functions of the network layers
+
+    Returns
+    -------
+    delta_W : list containing the gradient matrices of the network layer weights
+    delta_b : list containing the gradient matrices of the network layer biases
+
+    """
+
+    delta_b = []
+    delta_W = []
+
+    for i in range(len(W)-1,-1,-1):
+
+        delta_a = delta_h * activation[i](a[i], True)
+     
+        delta_b.append( delta_a.mean(1).reshape(-1,1) ) 
+        delta_W.append( delta_a.dot(h[i].T) ) 
+
+        delta_h = (W[i].T).dot(delta_a) 
+
+    delta_b = delta_b[::-1]
+    delta_W = delta_W[::-1]
+
+    return delta_W, delta_b
+
+def sigmoid(z, deriv=False):
+    """ Calculates the value of the sigmoid function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the sigmoid function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the sigmoid function applied to z or its derivative. Same dimension as z
+
+    """
+
+    s = 1 / (1 + np.exp(-z))
+    if deriv:
+        return s * (1 - s)
+    else :
+        return s
+
+def linear(z, deriv=False):
+    """ Calculates the value of the linear function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the linear function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the linear function applied to z or its derivative. Same dimension as z
+
+    """
+    if deriv:       
+        return 1     
+    else :
+        return z
+
+def relu(z, deriv=False):
+    """ Calculates the value of the relu function or its derivative applied to z
+    
+    Parameters
+    ----------
+    z : can be a scalar or a matrix
+    deriv : boolean. If False returns the value of the relu function, if True returns its derivative
+
+    Returns
+    -------
+    s : value of the relu function applied to z or its derivative. Same dimension as z
+
+    """
+
+    r = np.zeros(z.shape)
+    if deriv:
+        pos = np.where(z>=0)
+        r[pos] = 1.0
+        return r
+    else :    
+        return np.maximum(r,z)
+
+
+# ===================== Part 1: Data Reading and Normalization =====================
+print("Reading data ...")
+
+x, d, N, num_vars, num_targets = read_data("food_truck.txt")
+# x, d, N, num_vars, num_targets = read_data("houses.txt")
+
+# Displaying the first 10 examples from the dataset
+print("Displaying the first 10 examples from the dataset: ")
+for i in range(0, 10):
+    print(f"x = {x[i,:]}, d = {d[i]}")
+    
+# Normalizing the variables (centering and scaling)
+print("Normalizing the variables ...")
+x, mu, sigma = normalization(x)
+dmax = d.max()
+d = d / dmax
+
+# Splitting the data into training, validation, and test subsets
+x_train, d_train, x_val, d_val, x_test, d_test = split_data(x, d)
+
+# ===================== Part 2: Training =====================
+
+# Choosing the learning rate and number of iterations
+alpha = 0.001
+num_iters = 500
+train_costs = np.zeros(num_iters)
+val_costs = np.zeros(num_iters)
+
+# Network dimensions
+D_c = [num_vars, 5, 10, num_targets] # list containing the number of neurons for each layer 
+activation = [relu, sigmoid, linear] # list containing the activation functions for the hidden layers and the output layer 
+
+# Random initialization of the network weights
+W = []
+b = []
+for i in range(len(D_c)-1):    
+    W.append(2 * np.random.random((D_c[i+1], D_c[i])) - 1)
+    b.append(np.zeros((D_c[i+1],1)))
+
+x_train = x_train.T # Data is presented as column vectors at the input of the network
+d_train = d_train.T 
+
+x_val = x_val.T # Data is presented as column vectors at the input of the network
+d_val = d_val.T 
+
+x_test = x_test.T # Data is presented as column vectors at the input of the network
+d_test = d_test.T 
+
+for t in range(num_iters):
+
+    #############################################################################
+    # Forward pass: calculating predicted output y on validation data #
+    #############################################################################
+    a, h = forward_pass(x_val, W, b, activation)
+    y_val = h[-1] # Predicted output
+
+    ###############################################################################
+    # Forward pass: calculating predicted output y on training data #
+    ###############################################################################
+    a, h = forward_pass(x_train, W, b, activation)
+    y_train = h[-1] # Predicted output
+
+    ###########################################
+    # Calculating the MSE loss function #
+    ###########################################
+    train_costs[t] = calculate_mse_cost(y_train, d_train)
+    val_costs[t] = calculate_mse_cost(y_val, d_val)
+
+    ####################################
+    # Backward pass: backpropagation #
+    ####################################
+    delta_h = (y_train-d_train) # For the last layer 
+    delta_W, delta_b = backward_pass(delta_h, a, h, W, activation)
+  
+    #############################################
+    # Updating weights and biases #
+    ############################################# 
+    for i in range(len(b)-1,-1,-1):
+        b[i] -= alpha * delta_b[i]
+        W[i] -= alpha * delta_W[i]
+
+print("Final cost on the training set: ", train_costs[-1])
+print("Final cost on the validation set: ", val_costs[-1])
+
+# Plotting the evolution of the cost function during backpropagation
+plt.figure(0)
+plt.title("Evolution of the cost function during backpropagation")
+plt.plot(np.arange(train_costs.size), train_costs, label="Training")
+plt.plot(np.arange(val_costs.size), val_costs, label="Validation")
+plt.legend(loc="upper left")
+plt.xlabel("Number of iterations")
+plt.ylabel("Cost")
+plt.show()
+
+# ===================== Part 3: Evaluation on the test set =====================
+
+#######################################################################
+# Forward pass: calculating predicted output y on test data #
+#######################################################################
+a, h = forward_pass(x_test, W, b, activation)
+y_test = h[-1] # Predicted output
+
+cost = calculate_mse_cost(y_test, d_test)
+print("Test set cost: ", cost)

Practical_sessions/Session_3/food_truck.txt

+6.1101,17.592
+5.5277,9.1302
+8.5186,13.662
+7.0032,11.854
+5.8598,6.8233
+8.3829,11.886
+7.4764,4.3483
+8.5781,12
+6.4862,6.5987
+5.0546,3.8166
+5.7107,3.2522
+14.164,15.505
+5.734,3.1551
+8.4084,7.2258
+5.6407,0.71618
+5.3794,3.5129
+6.3654,5.3048
+5.1301,0.56077
+6.4296,3.6518
+7.0708,5.3893
+6.1891,3.1386
+20.27,21.767
+5.4901,4.263
+6.3261,5.1875
+5.5649,3.0825
+18.945,22.638
+12.828,13.501
+10.957,7.0467
+13.176,14.692
+22.203,24.147
+5.2524,-1.22
+6.5894,5.9966
+9.2482,12.134
+5.8918,1.8495
+8.2111,6.5426
+7.9334,4.5623
+8.0959,4.1164
+5.6063,3.3928
+12.836,10.117
+6.3534,5.4974
+5.4069,0.55657
+6.8825,3.9115
+11.708,5.3854
+5.7737,2.4406
+7.8247,6.7318
+7.0931,1.0463
+5.0702,5.1337
+5.8014,1.844
+11.7,8.0043
+5.5416,1.0179
+7.5402,6.7504
+5.3077,1.8396
+7.4239,4.2885
+7.6031,4.9981
+6.3328,1.4233
+6.3589,-1.4211
+6.2742,2.4756
+5.6397,4.6042
+9.3102,3.9624
+9.4536,5.4141
+8.8254,5.1694
+5.1793,-0.74279
+21.279,17.929
+14.908,12.054
+18.959,17.054
+7.2182,4.8852
+8.2951,5.7442
+10.236,7.7754
+5.4994,1.0173
+20.341,20.992
+10.136,6.6799
+7.3345,4.0259
+6.0062,1.2784
+7.2259,3.3411
+5.0269,-2.6807
+6.5479,0.29678
+7.5386,3.8845
+5.0365,5.7014
+10.274,6.7526
+5.1077,2.0576
+5.7292,0.47953
+5.1884,0.20421
+6.3557,0.67861
+9.7687,7.5435
+6.5159,5.3436
+8.5172,4.2415
+9.1802,6.7981
+6.002,0.92695
+5.5204,0.152
+5.0594,2.8214
+5.7077,1.8451
+7.6366,4.2959
+5.8707,7.2029
+5.3054,1.9869
+8.2934,0.14454
+13.394,9.0551
+5.4369,0.61705

Practical_sessions/Session_3/nn_regression-pytorch-completed.py

+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+import matplotlib.pyplot as plt
+
+def read_data(file_name, delimiter=','):
+    """ Reads the file containing the data and returns the corresponding tensors """
+    data = np.loadtxt(file_name, delimiter=delimiter)
+    num_targets = 1
+    num_vars = data.shape[1] - num_targets
+    N = data.shape[0]
+
+    x = torch.tensor(data[:, :num_vars], dtype=torch.float32)
+    d = torch.tensor(data[:, num_vars:], dtype=torch.float32).view(N, 1)
+    
+    return x, d, N, num_vars, num_targets
+
+def normalization(x):
+    """ Normalizes the data by centering and scaling the predictor variables """
+    mu = x.mean(0)
+    sigma = x.std(0)
+    x_norm = (x - mu) / sigma
+
+    return x_norm, mu, sigma
+
+def split_data(x, d, val_prop=0.2, test_prop=0.2):
+    """ Splits the initial data into training, validation, and testing subsets """
+    assert val_prop + test_prop < 1.0
+
+    N = x.size(0)
+    indices = torch.randperm(N)
+    num_val = int(N * val_prop)
+    num_test = int(N * test_prop)
+    num_train = N - num_val - num_test
+
+    x = x[indices]
+    d = d[indices]
+
+    x_train = x[:num_train]
+    d_train = d[:num_train]
+
+    x_val = x[num_train:num_train + num_val]
+    d_val = d[num_train:num_train + num_val]
+
+    x_test = x[num_train + num_val:]
+    d_test = d[num_train + num_val:]
+
+    return x_train, d_train, x_val, d_val, x_test, d_test
+
+# Define the neural network class
+class NeuralNetwork(nn.Module):
+    def __init__(self, layer_dims, activations):
+        super(NeuralNetwork, self).__init__()
+        layers = []
+        for i in range(len(layer_dims) - 1):
+            layers.append(nn.Linear(layer_dims[i], layer_dims[i + 1]))
+            if activations[i] == 'relu':
+                layers.append(nn.ReLU())
+            elif activations[i] == 'sigmoid':
+                layers.append(nn.Sigmoid())
+            elif activations[i] == 'linear':
+                pass  # Linear activation is implicit
+        self.model = nn.Sequential(*layers)
+
+    def forward(self, x):
+        return self.model(x)
+
+def calculate_mse_cost(y, d):
+    """ Calculates the MSE loss function """
+    return ((y - d) ** 2).mean() / 2
+
+# ===================== Part 1: Data Reading and Normalization =====================
+print("Reading data ...")
+
+x, d, N, num_vars, num_targets = read_data("food_truck.txt")
+# x, d, N, num_vars, num_targets = read_data("houses.txt")
+
+# Displaying the first 10 examples from the dataset
+print("Displaying the first 10 examples from the dataset: ")
+for i in range(10):
+    print(f"x = {x[i]}, d = {d[i]}")
+
+# Normalizing the variables
+print("Normalizing the variables ...")
+x, mu, sigma = normalization(x)
+dmax = d.max()
+d = d / dmax
+
+# Splitting the data
+x_train, d_train, x_val, d_val, x_test, d_test = split_data(x, d)
+
+# ===================== Part 2: Training =====================
+
+# Hyperparameters
+alpha = 0.001
+num_iters = 500
+layer_dims = [num_vars, 5, 10, num_targets]
+activations = ['relu', 'sigmoid', 'linear']
+
+# Model, loss, and optimizer
+model = NeuralNetwork(layer_dims, activations)
+criterion = nn.MSELoss()
+optimizer = optim.SGD(model.parameters(), lr=alpha)
+
+train_costs = []
+val_costs = []
+
+for t in range(num_iters):
+    # Training forward pass
+    model.train()
+    y_train = model(x_train)
+    train_loss = criterion(y_train, d_train)
+
+    # Backpropagation
+    optimizer.zero_grad()
+    train_loss.backward()
+    optimizer.step()
+
+    # Validation forward pass
+    model.eval()
+    with torch.no_grad():
+        y_val = model(x_val)
+        val_loss = criterion(y_val, d_val)
+
+    train_costs.append(train_loss.item())
+    val_costs.append(val_loss.item())
+
+print("Final cost on the training set: ", train_costs[-1])
+print("Final cost on the validation set: ", val_costs[-1])
+
+# Plotting the evolution of the cost function
+plt.figure()
+plt.title("Evolution of the cost function during training")
+plt.plot(range(num_iters), train_costs, label="Training")
+plt.plot(range(num_iters), val_costs, label="Validation")
+plt.legend(loc="upper left")
+plt.xlabel("Number of iterations")
+plt.ylabel("Cost")
+plt.show()
+
+# ===================== Part 3: Evaluation on the Test Set =====================
+model.eval()
+with torch.no_grad():
+    y_test = model(x_test)
+    test_loss = criterion(y_test, d_test)
+
+print("Test set cost: ", test_loss.item())

Practical_sessions/Session_4/imagenet-simple-labels.json

+["tench",
+"goldfish",
+"great white shark",
+"tiger shark",
+"hammerhead shark",
+"electric ray",
+"stingray",
+"cock",
+"hen",
+"ostrich",
+"brambling",
+"goldfinch",
+"house finch",
+"junco",
+"indigo bunting",
+"American robin",
+"bulbul",
+"jay",
+"magpie",
+"chickadee",
+"American dipper",
+"kite",
+"bald eagle",
+"vulture",
+"great grey owl",
+"fire salamander",
+"smooth newt",
+"newt",
+"spotted salamander",
+"axolotl",
+"American bullfrog",
+"tree frog",
+"tailed frog",
+"loggerhead sea turtle",
+"leatherback sea turtle",
+"mud turtle",
+"terrapin",
+"box turtle",
+"banded gecko",
+"green iguana",
+"Carolina anole",
+"desert grassland whiptail lizard",
+"agama",
+"frilled-necked lizard",
+"alligator lizard",
+"Gila monster",
+"European green lizard",
+"chameleon",
+"Komodo dragon",
+"Nile crocodile",
+"American alligator",
+"triceratops",
+"worm snake",
+"ring-necked snake",
+"eastern hog-nosed snake",
+"smooth green snake",
+"kingsnake",
+"garter snake",
+"water snake",
+"vine snake",
+"night snake",
+"boa constrictor",
+"African rock python",
+"Indian cobra",
+"green mamba",
+"sea snake",
+"Saharan horned viper",
+"eastern diamondback rattlesnake",
+"sidewinder",
+"trilobite",
+"harvestman",
+"scorpion",
+"yellow garden spider",
+"barn spider",
+"European garden spider",
+"southern black widow",
+"tarantula",
+"wolf spider",
+"tick",
+"centipede",
+"black grouse",
+"ptarmigan",
+"ruffed grouse",
+"prairie grouse",
+"peacock",
+"quail",
+"partridge",
+"grey parrot",
+"macaw",
+"sulphur-crested cockatoo",
+"lorikeet",
+"coucal",
+"bee eater",
+"hornbill",
+"hummingbird",
+"jacamar",
+"toucan",
+"duck",
+"red-breasted merganser",
+"goose",
+"black swan",
+"tusker",
+"echidna",
+"platypus",
+"wallaby",
+"koala",
+"wombat",
+"jellyfish",
+"sea anemone",
+"brain coral",
+"flatworm",
+"nematode",
+"conch",
+"snail",
+"slug",
+"sea slug",
+"chiton",
+"chambered nautilus",
+"Dungeness crab",
+"rock crab",
+"fiddler crab",
+"red king crab",
+"American lobster",
+"spiny lobster",
+"crayfish",
+"hermit crab",
+"isopod",
+"white stork",
+"black stork",
+"spoonbill",
+"flamingo",
+"little blue heron",
+"great egret",
+"bittern",
+"crane",
+"limpkin",
+"common gallinule",
+"American coot",
+"bustard",
+"ruddy turnstone",
+"dunlin",
+"common redshank",
+"dowitcher",
+"oystercatcher",
+"pelican",
+"king penguin",
+"albatross",
+"grey whale",
+"killer whale",
+"dugong",
+"sea lion",
+"Chihuahua",
+"Japanese Chin",
+"Maltese",
+"Pekingese",
+"Shih Tzu",
+"King Charles Spaniel",
+"Papillon",
+"toy terrier",
+"Rhodesian Ridgeback",
+"Afghan Hound",
+"Basset Hound",
+"Beagle",
+"Bloodhound",
+"Bluetick Coonhound",
+"Black and Tan Coonhound",
+"Treeing Walker Coonhound",
+"English foxhound",
+"Redbone Coonhound",
+"borzoi",
+"Irish Wolfhound",
+"Italian Greyhound",
+"Whippet",
+"Ibizan Hound",
+"Norwegian Elkhound",
+"Otterhound",
+"Saluki",
+"Scottish Deerhound",
+"Weimaraner",
+"Staffordshire Bull Terrier",
+"American Staffordshire Terrier",
+"Bedlington Terrier",
+"Border Terrier",
+"Kerry Blue Terrier",
+"Irish Terrier",
+"Norfolk Terrier",
+"Norwich Terrier",
+"Yorkshire Terrier",
+"Wire Fox Terrier",
+"Lakeland Terrier",
+"Sealyham Terrier",
+"Airedale Terrier",
+"Cairn Terrier",
+"Australian Terrier",
+"Dandie Dinmont Terrier",
+"Boston Terrier",
+"Miniature Schnauzer",
+"Giant Schnauzer",
+"Standard Schnauzer",
+"Scottish Terrier",
+"Tibetan Terrier",
+"Australian Silky Terrier",
+"Soft-coated Wheaten Terrier",
+"West Highland White Terrier",
+"Lhasa Apso",
+"Flat-Coated Retriever",
+"Curly-coated Retriever",
+"Golden Retriever",
+"Labrador Retriever",
+"Chesapeake Bay Retriever",
+"German Shorthaired Pointer",
+"Vizsla",
+"English Setter",
+"Irish Setter",
+"Gordon Setter",
+"Brittany",
+"Clumber Spaniel",
+"English Springer Spaniel",
+"Welsh Springer Spaniel",
+"Cocker Spaniels",
+"Sussex Spaniel",
+"Irish Water Spaniel",
+"Kuvasz",
+"Schipperke",
+"Groenendael",
+"Malinois",
+"Briard",
+"Australian Kelpie",
+"Komondor",
+"Old English Sheepdog",
+"Shetland Sheepdog",
+"collie",
+"Border Collie",
+"Bouvier des Flandres",
+"Rottweiler",
+"German Shepherd Dog",
+"Dobermann",
+"Miniature Pinscher",
+"Greater Swiss Mountain Dog",
+"Bernese Mountain Dog",
+"Appenzeller Sennenhund",
+"Entlebucher Sennenhund",
+"Boxer",
+"Bullmastiff",
+"Tibetan Mastiff",
+"French Bulldog",
+"Great Dane",
+"St. Bernard",
+"husky",
+"Alaskan Malamute",
+"Siberian Husky",
+"Dalmatian",
+"Affenpinscher",
+"Basenji",
+"pug",
+"Leonberger",
+"Newfoundland",
+"Pyrenean Mountain Dog",
+"Samoyed",
+"Pomeranian",
+"Chow Chow",
+"Keeshond",
+"Griffon Bruxellois",
+"Pembroke Welsh Corgi",
+"Cardigan Welsh Corgi",
+"Toy Poodle",
+"Miniature Poodle",
+"Standard Poodle",
+"Mexican hairless dog",
+"grey wolf",
+"Alaskan tundra wolf",
+"red wolf",
+"coyote",
+"dingo",
+"dhole",
+"African wild dog",
+"hyena",
+"red fox",
+"kit fox",
+"Arctic fox",
+"grey fox",
+"tabby cat",
+"tiger cat",
+"Persian cat",
+"Siamese cat",
+"Egyptian Mau",
+"cougar",
+"lynx",
+"leopard",
+"snow leopard",
+"jaguar",
+"lion",
+"tiger",
+"cheetah",
+"brown bear",
+"American black bear",
+"polar bear",
+"sloth bear",
+"mongoose",
+"meerkat",
+"tiger beetle",
+"ladybug",
+"ground beetle",
+"longhorn beetle",
+"leaf beetle",
+"dung beetle",
+"rhinoceros beetle",
+"weevil",
+"fly",
+"bee",
+"ant",
+"grasshopper",
+"cricket",
+"stick insect",
+"cockroach",
+"mantis",
+"cicada",
+"leafhopper",
+"lacewing",
+"dragonfly",
+"damselfly",
+"red admiral",
+"ringlet",
+"monarch butterfly",
+"small white",
+"sulphur butterfly",
+"gossamer-winged butterfly",
+"starfish",
+"sea urchin",
+"sea cucumber",
+"cottontail rabbit",
+"hare",
+"Angora rabbit",
+"hamster",
+"porcupine",
+"fox squirrel",
+"marmot",
+"beaver",
+"guinea pig",
+"common sorrel",
+"zebra",
+"pig",
+"wild boar",
+"warthog",
+"hippopotamus",
+"ox",
+"water buffalo",
+"bison",
+"ram",
+"bighorn sheep",
+"Alpine ibex",
+"hartebeest",
+"impala",
+"gazelle",
+"dromedary",
+"llama",
+"weasel",
+"mink",
+"European polecat",
+"black-footed ferret",
+"otter",
+"skunk",
+"badger",
+"armadillo",
+"three-toed sloth",
+"orangutan",
+"gorilla",
+"chimpanzee",
+"gibbon",
+"siamang",
+"guenon",
+"patas monkey",
+"baboon",
+"macaque",
+"langur",
+"black-and-white colobus",
+"proboscis monkey",
+"marmoset",
+"white-headed capuchin",
+"howler monkey",
+"titi",
+"Geoffroy's spider monkey",
+"common squirrel monkey",
+"ring-tailed lemur",
+"indri",
+"Asian elephant",
+"African bush elephant",
+"red panda",
+"giant panda",
+"snoek",
+"eel",
+"coho salmon",
+"rock beauty",
+"clownfish",
+"sturgeon",
+"garfish",
+"lionfish",
+"pufferfish",
+"abacus",
+"abaya",
+"academic gown",
+"accordion",
+"acoustic guitar",
+"aircraft carrier",
+"airliner",
+"airship",
+"altar",
+"ambulance",
+"amphibious vehicle",
+"analog clock",
+"apiary",
+"apron",
+"waste container",
+"assault rifle",
+"backpack",
+"bakery",
+"balance beam",
+"balloon",
+"ballpoint pen",
+"Band-Aid",
+"banjo",
+"baluster",
+"barbell",
+"barber chair",
+"barbershop",
+"barn",
+"barometer",
+"barrel",
+"wheelbarrow",
+"baseball",
+"basketball",
+"bassinet",
+"bassoon",
+"swimming cap",
+"bath towel",
+"bathtub",
+"station wagon",
+"lighthouse",
+"beaker",
+"military cap",
+"beer bottle",
+"beer glass",
+"bell-cot",
+"bib",
+"tandem bicycle",
+"bikini",
+"ring binder",
+"binoculars",
+"birdhouse",
+"boathouse",
+"bobsleigh",
+"bolo tie",
+"poke bonnet",
+"bookcase",
+"bookstore",
+"bottle cap",
+"bow",
+"bow tie",
+"brass",
+"bra",
+"breakwater",
+"breastplate",
+"broom",
+"bucket",
+"buckle",
+"bulletproof vest",
+"high-speed train",
+"butcher shop",
+"taxicab",
+"cauldron",
+"candle",
+"cannon",
+"canoe",
+"can opener",
+"cardigan",
+"car mirror",
+"carousel",
+"tool kit",
+"carton",
+"car wheel",
+"automated teller machine",
+"cassette",
+"cassette player",
+"castle",
+"catamaran",
+"CD player",
+"cello",
+"mobile phone",
+"chain",
+"chain-link fence",
+"chain mail",
+"chainsaw",
+"chest",
+"chiffonier",
+"chime",
+"china cabinet",
+"Christmas stocking",
+"church",
+"movie theater",
+"cleaver",
+"cliff dwelling",
+"cloak",
+"clogs",
+"cocktail shaker",
+"coffee mug",
+"coffeemaker",
+"coil",
+"combination lock",
+"computer keyboard",
+"confectionery store",
+"container ship",
+"convertible",
+"corkscrew",
+"cornet",
+"cowboy boot",
+"cowboy hat",
+"cradle",
+"crane",
+"crash helmet",
+"crate",
+"infant bed",
+"Crock Pot",
+"croquet ball",
+"crutch",
+"cuirass",
+"dam",
+"desk",
+"desktop computer",
+"rotary dial telephone",
+"diaper",
+"digital clock",
+"digital watch",
+"dining table",
+"dishcloth",
+"dishwasher",
+"disc brake",
+"dock",
+"dog sled",
+"dome",
+"doormat",
+"drilling rig",
+"drum",
+"drumstick",
+"dumbbell",
+"Dutch oven",
+"electric fan",
+"electric guitar",
+"electric locomotive",
+"entertainment center",
+"envelope",
+"espresso machine",
+"face powder",
+"feather boa",
+"filing cabinet",
+"fireboat",
+"fire engine",
+"fire screen sheet",
+"flagpole",
+"flute",
+"folding chair",
+"football helmet",
+"forklift",
+"fountain",
+"fountain pen",
+"four-poster bed",
+"freight car",
+"French horn",
+"frying pan",
+"fur coat",
+"garbage truck",
+"gas mask",
+"gas pump",
+"goblet",
+"go-kart",
+"golf ball",
+"golf cart",
+"gondola",
+"gong",
+"gown",
+"grand piano",
+"greenhouse",
+"grille",
+"grocery store",
+"guillotine",
+"barrette",
+"hair spray",
+"half-track",
+"hammer",
+"hamper",
+"hair dryer",
+"hand-held computer",
+"handkerchief",
+"hard disk drive",
+"harmonica",
+"harp",
+"harvester",
+"hatchet",
+"holster",
+"home theater",
+"honeycomb",
+"hook",
+"hoop skirt",
+"horizontal bar",
+"horse-drawn vehicle",
+"hourglass",
+"iPod",
+"clothes iron",
+"jack-o'-lantern",
+"jeans",
+"jeep",
+"T-shirt",
+"jigsaw puzzle",
+"pulled rickshaw",
+"joystick",
+"kimono",
+"knee pad",
+"knot",
+"lab coat",
+"ladle",
+"lampshade",
+"laptop computer",
+"lawn mower",
+"lens cap",
+"paper knife",
+"library",
+"lifeboat",
+"lighter",
+"limousine",
+"ocean liner",
+"lipstick",
+"slip-on shoe",
+"lotion",
+"speaker",
+"loupe",
+"sawmill",
+"magnetic compass",
+"mail bag",
+"mailbox",
+"tights",
+"tank suit",
+"manhole cover",
+"maraca",
+"marimba",
+"mask",
+"match",
+"maypole",
+"maze",
+"measuring cup",
+"medicine chest",
+"megalith",
+"microphone",
+"microwave oven",
+"military uniform",
+"milk can",
+"minibus",
+"miniskirt",
+"minivan",
+"missile",
+"mitten",
+"mixing bowl",
+"mobile home",
+"Model T",
+"modem",
+"monastery",
+"monitor",
+"moped",
+"mortar",
+"square academic cap",
+"mosque",
+"mosquito net",
+"scooter",
+"mountain bike",
+"tent",
+"computer mouse",
+"mousetrap",
+"moving van",
+"muzzle",
+"nail",
+"neck brace",
+"necklace",
+"nipple",
+"notebook computer",
+"obelisk",
+"oboe",
+"ocarina",
+"odometer",
+"oil filter",
+"organ",
+"oscilloscope",
+"overskirt",
+"bullock cart",
+"oxygen mask",
+"packet",
+"paddle",
+"paddle wheel",
+"padlock",
+"paintbrush",
+"pajamas",
+"palace",
+"pan flute",
+"paper towel",
+"parachute",
+"parallel bars",
+"park bench",
+"parking meter",
+"passenger car",
+"patio",
+"payphone",
+"pedestal",
+"pencil case",
+"pencil sharpener",
+"perfume",
+"Petri dish",
+"photocopier",
+"plectrum",
+"Pickelhaube",
+"picket fence",
+"pickup truck",
+"pier",
+"piggy bank",
+"pill bottle",
+"pillow",
+"ping-pong ball",
+"pinwheel",
+"pirate ship",
+"pitcher",
+"hand plane",
+"planetarium",
+"plastic bag",
+"plate rack",
+"plow",
+"plunger",
+"Polaroid camera",
+"pole",
+"police van",
+"poncho",
+"billiard table",
+"soda bottle",
+"pot",
+"potter's wheel",
+"power drill",
+"prayer rug",
+"printer",
+"prison",
+"projectile",
+"projector",
+"hockey puck",
+"punching bag",
+"purse",
+"quill",
+"quilt",
+"race car",
+"racket",
+"radiator",
+"radio",
+"radio telescope",
+"rain barrel",
+"recreational vehicle",
+"reel",
+"reflex camera",
+"refrigerator",
+"remote control",
+"restaurant",
+"revolver",
+"rifle",
+"rocking chair",
+"rotisserie",
+"eraser",
+"rugby ball",
+"ruler",
+"running shoe",
+"safe",
+"safety pin",
+"salt shaker",
+"sandal",
+"sarong",
+"saxophone",
+"scabbard",
+"weighing scale",
+"school bus",
+"schooner",
+"scoreboard",
+"CRT screen",
+"screw",
+"screwdriver",
+"seat belt",
+"sewing machine",
+"shield",
+"shoe store",
+"shoji",
+"shopping basket",
+"shopping cart",
+"shovel",
+"shower cap",
+"shower curtain",
+"ski",
+"ski mask",
+"sleeping bag",
+"slide rule",
+"sliding door",
+"slot machine",
+"snorkel",
+"snowmobile",
+"snowplow",
+"soap dispenser",
+"soccer ball",
+"sock",
+"solar thermal collector",
+"sombrero",
+"soup bowl",
+"space bar",
+"space heater",
+"space shuttle",
+"spatula",
+"motorboat",
+"spider web",
+"spindle",
+"sports car",
+"spotlight",
+"stage",
+"steam locomotive",
+"through arch bridge",
+"steel drum",
+"stethoscope",
+"scarf",
+"stone wall",
+"stopwatch",
+"stove",
+"strainer",
+"tram",
+"stretcher",
+"couch",
+"stupa",
+"submarine",
+"suit",
+"sundial",
+"sunglass",
+"sunglasses",
+"sunscreen",
+"suspension bridge",
+"mop",
+"sweatshirt",
+"swimsuit",
+"swing",
+"switch",
+"syringe",
+"table lamp",
+"tank",
+"tape player",
+"teapot",
+"teddy bear",
+"television",
+"tennis ball",
+"thatched roof",
+"front curtain",
+"thimble",
+"threshing machine",
+"throne",
+"tile roof",
+"toaster",
+"tobacco shop",
+"toilet seat",
+"torch",
+"totem pole",
+"tow truck",
+"toy store",
+"tractor",
+"semi-trailer truck",
+"tray",
+"trench coat",
+"tricycle",
+"trimaran",
+"tripod",
+"triumphal arch",
+"trolleybus",
+"trombone",
+"tub",
+"turnstile",
+"typewriter keyboard",
+"umbrella",
+"unicycle",
+"upright piano",
+"vacuum cleaner",
+"vase",
+"vault",
+"velvet",
+"vending machine",
+"vestment",
+"viaduct",
+"violin",
+"volleyball",
+"waffle iron",
+"wall clock",
+"wallet",
+"wardrobe",
+"military aircraft",
+"sink",
+"washing machine",
+"water bottle",
+"water jug",
+"water tower",
+"whiskey jug",
+"whistle",
+"wig",
+"window screen",
+"window shade",
+"Windsor tie",
+"wine bottle",
+"wing",
+"wok",
+"wooden spoon",
+"wool",
+"split-rail fence",
+"shipwreck",
+"yawl",
+"yurt",
+"website",
+"comic book",
+"crossword",
+"traffic sign",
+"traffic light",
+"dust jacket",
+"menu",
+"plate",
+"guacamole",
+"consomme",
+"hot pot",
+"trifle",
+"ice cream",
+"ice pop",
+"baguette",
+"bagel",
+"pretzel",
+"cheeseburger",
+"hot dog",
+"mashed potato",
+"cabbage",
+"broccoli",
+"cauliflower",
+"zucchini",
+"spaghetti squash",
+"acorn squash",
+"butternut squash",
+"cucumber",
+"artichoke",
+"bell pepper",
+"cardoon",
+"mushroom",
+"Granny Smith",
+"strawberry",
+"orange",
+"lemon",
+"fig",
+"pineapple",
+"banana",
+"jackfruit",
+"custard apple",
+"pomegranate",
+"hay",
+"carbonara",
+"chocolate syrup",
+"dough",
+"meatloaf",
+"pizza",
+"pot pie",
+"burrito",
+"red wine",
+"espresso",
+"cup",
+"eggnog",
+"alp",
+"bubble",
+"cliff",
+"coral reef",
+"geyser",
+"lakeshore",
+"promontory",
+"shoal",
+"seashore",
+"valley",
+"volcano",
+"baseball player",
+"bridegroom",
+"scuba diver",
+"rapeseed",
+"daisy",
+"yellow lady's slipper",
+"corn",
+"acorn",
+"rose hip",
+"horse chestnut seed",
+"coral fungus",
+"agaric",
+"gyromitra",
+"stinkhorn mushroom",
+"earth star",
+"hen-of-the-woods",
+"bolete",
+"ear",
+"toilet paper"]

Practical_sessions/Session_5/Subject_5_Training_monitoring.ipynb

+%% Cell type:markdown id: tags:
+
+### **_Deep Learning  - Bsc Data Science for Responsible Business - Centrale Lyon_**
+
+2024-2025
+
+Emmanuel Dellandréa
+
+%% Cell type:markdown id: tags:
+
+# Practical Session 5 – Monitoring the training with Weights & Biases
+
+The objective of this short tutorial is to learn how to monitor a CNN training with [Weights and Biases](https://wandb.ai/site/). With W&B, you can track and compare your experiments, visualize your model training and performance.
+
+#### Installation
+
+You'll need to install `wand`.
+
+```shell
+pip install wandb
+```
+
+
+Have a look at the documentation of for integrating [Weights & Biases into Pytorch](https://docs.wandb.ai/guides/integrations/pytorch/).
+
+Then, study the code below and the informations registered in W&B.
+
+As the computation is heavy, particularly during training, we encourage you to use a GPU. If your laptob is not equiped, you may use one of these remote jupyter servers, where you can select the execution on GPU :
+
+1) [jupyter.mi90.ec-lyon.fr](https://jupyter.mi90.ec-lyon.fr/)
+
+This server is accessible within the campus network. If outside, you need to use a VPN. Before executing the notebook, select the kernel "Python PyTorch" to run it on GPU and have access to PyTorch module.
+
+2) [Google Colaboratory](https://colab.research.google.com/)
+
+Before executing the notebook, select the execution on GPU : "Exécution" Menu -> "Modifier le type d'exécution" and select "T4 GPU".
+
+%% Cell type:code id: tags:
+
+``` 
+import wandb
+
+# Initialize wandb
+wandb.init(
+    project="cnn_cifar10",  # Set your project name
+    config={                       # Define hyperparameters
+        "epochs": 5,
+        "batch_size": 64,
+        "learning_rate": 0.01,
+        "optimizer": "Adam"
+    }
+)
+```
+
+%% Cell type:code id: tags:
+
+``` 
+import torch
+import numpy as np
+from torchvision import datasets, transforms
+from torch.utils.data.sampler import SubsetRandomSampler
+import torch.optim as optim
+
+# number of subprocesses to use for data loading
+num_workers = 0
+# how many samples per batch to load
+batch_size = wandb.config.batch_size
+# percentage of training set to use as validation
+valid_size = 0.2
+
+# convert data to a normalized torch.FloatTensor
+transform = transforms.Compose(
+    [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
+)
+
+# choose the training and test datasets
+train_data = datasets.CIFAR10("data", train=True, download=True, transform=transform)
+test_data = datasets.CIFAR10("data", train=False, download=True, transform=transform)
+
+# obtain training indices that will be used for validation
+num_train = len(train_data)
+indices = list(range(num_train))
+np.random.shuffle(indices)
+split = int(np.floor(valid_size * num_train))
+train_idx, valid_idx = indices[split:], indices[:split]
+
+# define samplers for obtaining training and validation batches
+train_sampler = SubsetRandomSampler(train_idx)
+valid_sampler = SubsetRandomSampler(valid_idx)
+
+# prepare data loaders (combine dataset and sampler)
+train_loader = torch.utils.data.DataLoader(
+    train_data, batch_size=wandb.config.batch_size, sampler=train_sampler, num_workers=num_workers
+)
+valid_loader = torch.utils.data.DataLoader(
+    train_data, batch_size=wandb.config.batch_size, sampler=valid_sampler, num_workers=num_workers
+)
+test_loader = torch.utils.data.DataLoader(
+    test_data, batch_size=wandb.config.batch_size, num_workers=num_workers
+)
+
+# specify the image classes
+classes = [
+    "airplane",
+    "automobile",
+    "bird",
+    "cat",
+    "deer",
+    "dog",
+    "frog",
+    "horse",
+    "ship",
+    "truck",
+]
+```
+
+%% Cell type:code id: tags:
+
+``` 
+import torch.nn as nn
+import torch.nn.functional as F
+
+# define the CNN architecture
+
+
+class Net(nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(3, 6, 5)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv2 = nn.Conv2d(6, 16, 5)
+        self.fc1 = nn.Linear(16 * 5 * 5, 120)
+        self.fc2 = nn.Linear(120, 84)
+        self.fc3 = nn.Linear(84, 10)
+
+    def forward(self, x):
+        x = self.pool(F.relu(self.conv1(x)))
+        x = self.pool(F.relu(self.conv2(x)))
+        x = x.view(-1, 16 * 5 * 5)
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = self.fc3(x)
+        return x
+```
+
+%% Cell type:code id: tags:
+
+``` 
+# Define model, loss, and optimizer
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = Net().to(device)
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.Adam(model.parameters(), lr=wandb.config.learning_rate)
+```
+
+%% Cell type:code id: tags:
+
+``` 
+valid_loss_min = np.Inf
+
+# Training loop
+for epoch in range(wandb.config.epochs):
+    epoch_loss_train = 0
+    correct_train = 0
+    total_train = 0
+
+    # Training
+    model.train()
+    for images, labels in train_loader:
+        images, labels = images.to(device), labels.to(device)
+
+        optimizer.zero_grad()
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+        loss.backward()
+        optimizer.step()
+
+        epoch_loss_train += loss.item()
+        _, predicted = torch.max(outputs, 1)
+        total_train += labels.size(0)
+        correct_train += (predicted == labels).sum().item()
+
+    # Validation
+    epoch_loss_valid = 0
+    correct_valid = 0
+    total_valid = 0
+
+    model.eval()
+    for images, labels in valid_loader:
+        images, labels = images.to(device), labels.to(device)
+
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+
+        epoch_loss_valid += loss.item()
+        _, predicted = torch.max(outputs, 1)
+        total_valid += labels.size(0)
+        correct_valid += (predicted == labels).sum().item()
+
+    # Save model if validation loss has decreased
+    if epoch_loss_valid <= valid_loss_min:
+        print(
+            "Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...".format(
+                valid_loss_min, epoch_loss_valid
+            )
+        )
+        torch.save(model.state_dict(), "model_cifar.pt")
+        valid_loss_min = epoch_loss_valid
+
+
+    accuracy_train = 100 * correct_train / total_train
+    avg_loss_train = epoch_loss_train / len(train_loader)
+    accuracy_valid = 100 * correct_valid / total_valid
+    avg_loss_valid = epoch_loss_valid / len(valid_loader)
+
+    # Log metrics to wandb
+    wandb.log({"epoch": epoch+1, "train_loss": avg_loss_train, "train_accuracy": accuracy_train})
+    wandb.log({"valid_loss": avg_loss_valid, "valid_accuracy": accuracy_valid})
+
+    print(f"Epoch {epoch+1}, Loss: {avg_loss_train:.4f}, Accuracy: {accuracy_train:.2f}%")
+
+```
+
+%% Cell type:code id: tags:
+
+``` 
+torch.save(model.state_dict(), "model_cifar.pth")
+wandb.save("model_cifar.pth")
+```
+
+%% Cell type:code id: tags:
+
+``` 
+# Log an example image
+wandb.log({"example_image": [wandb.Image(images[0].cpu())]})
+
+# Log gradients
+wandb.watch(model, log="all")
+```
+
+%% Cell type:code id: tags:
+
+``` 
+# Finish the wandb run
+wandb.finish()
+```
+
+%% Cell type:markdown id: tags:
+
+## Experiments
+
+Run several trainings with different tuning of the hyperparameters and check the result in W&B.