K-NN algorithm from Scratch: Breast Cancer Dataset, Part IV

# k-fold Cross validation
def k_fold_cross_validation(x_train,y_train,number_of_folds):
    start = 0
    size = len(x_train)
    size_of_fold = int(len(x_train)/number_of_folds)
    index_matrix = np.random.randint(start,size,size=(number_of_folds,size_of_fold))
    x_split = x_train[index_matrix]
    y_split = y_train[index_matrix]
    return x_split,y_split
  
# Compute Accuracy 
def find_accuracy(y_pred,y_test):
    acc = np.mean(y_pred == y_test)
    return acc

# Euclidean metrics
def euclidean_distance(x_row,y_row):
    output_distance = np.sqrt(np.sum((x_row-y_row)**2)) # Element-wise sum and square root
    return output_distance
  
# Find the First Neighbors
def find_neighbors(data_test,data_train,y_train,number_of_neighbours):
    new_distance_list = np.array([])
    distance_list = [] # List to append euclidian distances along with the row variables
    for idx,value in enumerate(data_train):
        distance = euclidean_distance(data_test,value) # Euclidian distances or #distance_list.append((data_test,row,distance))
        distance_list.append((value,distance,y_train[idx])) # Append the distances and respective rows to a list
    
    distance_list = np.array(distance_list)
    new_distance_list = distance_list[distance_list[:,1].argsort(kind='mergesort')] # Sort rows based on distance
    
    # Select the first neighbours
    x_first_neighbours = new_distance_list[:number_of_neighbours,:new_distance_list.shape[1]-2] # Select only the first neighbours 
    y_first_neighbours = new_distance_list[:number_of_neighbours,new_distance_list.shape[1]-1]

    
    return x_first_neighbours,y_first_neighbours
  
# Predict() runs find_neighbors() and output the most common label
def predict(data_test, data_train,y_train,number_of_neighbors):
    x_neighbors,y_neighbors = find_neighbors(data_test,data_train,y_train,number_of_neighbors)
    y_neighbors = list(y_neighbors)
    output = max(y_neighbors,key=y_neighbors.count)
    return output
  
# knn() is responsible to make the predictions
def knn(x_test,x_train,y_train,number_of_neighbors):
    x_test_prediction = []
    if(len(x_test)==9):
        out_predict = predict(x_test,x_train,y_train,number_of_neighbors)
        return(out_predict)
    else:
        for idx,value in enumerate(x_test):
            out_predict = predict(value,x_train,y_train,number_of_neighbors)
            x_test_prediction.append(out_predict)
        return x_test_prediction
     
# Evaluate an algorithm using a cross validation split
def cross_validation(x_train,y_train, number_of_folds, number_of_neighbors):
    accuracy = []
    average_acc = np.array([])
    x_cross,y_cross = k_fold_cross_validation(x_train,y_train,number_of_folds)
    inv_matr_identity = np.where(np.identity(number_of_folds,dtype=int)==1, False, True)
    
    for idx in range(number_of_folds):
        x_fold_train = x_cross[inv_matr_identity[idx]]
        y_fold_train = y_cross[inv_matr_identity[idx]]
        x_t = np.reshape(x_fold_train,(-1,x_cross.shape[2]))
        y_t = np.reshape(y_fold_train,((-1)))
        forecast = knn(x_cross[idx],x_t,y_t,number_of_neighbors)
        print('Prediction Fold:',idx,forecast)
        acc = find_accuracy(forecast,y_cross[idx])
        accuracy.append(acc)
    average_acc = np.average(np.asarray(accuracy))
    print('Average Prediction',average_acc)
    return x_cross,y_cross,accuracy,average_acc