diff --git a/.gitignore b/.gitignore index 956a40e..d6316d1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ data/mnist_train.csv data/mnist_outputs.npy data/mnist_inputs.npy other/mlp.ipynb + +data/mnist_2_outputs.npy + +data/mnist_2_inputs.npy diff --git a/data/ci_2_inputs.npy b/data/ci_2_inputs.npy new file mode 100644 index 0000000..efd9624 Binary files /dev/null and b/data/ci_2_inputs.npy differ diff --git a/data/ci_2_outputs.npy b/data/ci_2_outputs.npy new file mode 100644 index 0000000..d79ff56 Binary files /dev/null and b/data/ci_2_outputs.npy differ diff --git a/main.py b/main.py index 5ceff63..653a207 100644 --- a/main.py +++ b/main.py @@ -4,23 +4,25 @@ import time np.set_printoptions(linewidth=200) -data_input = np.load("data/ci_inputs.npy") -data_output = np.load("data/ci_outputs.npy") +data_input = np.load("data/mnist_inputs.npy") +data_output = np.load("data/mnist_outputs.npy") print(data_input.shape) print(data_output.shape) print("Begin compiling!") +np.random.seed(420) begin_time = time.time_ns() compiled_nn_values = nn.make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=[h.softmax]) -nn.train_auto(data_input[:100], data_output[:100], data_input[100: 140], data_output[100: 140], compiled_nn_values) +print("Created Neural network lists:", (time.time_ns()-begin_time) / 1e9) +nn.train_auto(data_input[:100], data_output[:100], data_input[100: 140], data_output[100: 140], 10, compiled_nn_values) end_time = time.time_ns() print("Compile time:", (end_time-begin_time) / 1e9) -np.random.seed(420) total_accuracy = 0.0 -begin_total = time.time_ns() -n = 10 +total_time = 0.0 +n = 5 +max_epochs = 200 for i in range(n): random_seed = np.random.randint(10, 1010) @@ -29,16 +31,17 @@ train_input, validate_input, test_input = h.kfold(7, data_input, random_seed) train_output, validate_output, test_output = h.kfold(7, data_output, random_seed) - nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=[h.sigmoid, h.softmax]) + nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], train_output.shape[1]], layer_activations=[h.softmax]) begin_time = time.time_ns() - epochs, current_mse = nn.train_auto(train_input, train_output, validate_input, validate_output, nn_values) + epochs, current_mse = nn.train_auto(train_input, train_output, validate_input, validate_output, max_epochs, nn_values) end_time = time.time_ns() train_mse = nn.calculate_MSE(train_input, train_output, nn_values) test_mse = nn.calculate_MSE(test_input, test_output, nn_values) - accuracy_test = nn.evaluate(test_input, test_output, nn_values) + total_accuracy += accuracy_test + total_time += (end_time-begin_time)/1e9 print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse) -print("Average Accuracy:", total_accuracy / n, "Average Time:", ((time.time_ns()-begin_total)/1e9) / n) +print("Average Accuracy:", total_accuracy / n, "Average Time:", total_time / n) diff --git a/numba_neural_network.py b/numba_neural_network.py index 0e7861d..6b9bdb0 100644 --- a/numba_neural_network.py +++ b/numba_neural_network.py @@ -70,7 +70,7 @@ def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low= def calculate_output(input_data, nn): assert len(input_data) == nn.layer_sizes[0] y = input_data - for i in prange(len(nn.weights)): + for i in range(len(nn.weights)): y = nn.layer_activations[i](np.dot(nn.weights[i].T, y) + nn.biases[i], False) return y @@ -79,7 +79,7 @@ def calculate_output(input_data, nn): def feed_forward_layers(input_data, nn): assert len(input_data) == nn.layer_sizes[0] nn.layer_outputs[0] = input_data - for i in prange(len(nn.weights)): + for i in range(len(nn.weights)): nn.layer_outputs[i+1] = nn.layer_activations[i](np.dot(nn.weights[i].T, nn.layer_outputs[i]) + nn.biases[i], False) @@ -94,8 +94,8 @@ def train_single(input_data, desired_output_data, nn): nn.biases[-1] += nn.learning_rate * error length_weights = len(nn.weights) - for i in prange(1, length_weights): - i = length_weights - i - 1 + for p in range(1, length_weights): + i = length_weights - p - 1 error = np.dot(nn.weights[i+1], error) * nn.layer_activations[i](nn.layer_outputs[i+1], True) nn.weights[i] += nn.learning_rate * nn.layer_outputs[i] * error.T nn.biases[i] += nn.learning_rate * error @@ -124,11 +124,11 @@ def train_epoch(train_input_data, train_desired_output_data, validate_input_data @njit -def train_auto(train_input_data, train_desired_output_data, validate_input_data, validate_output_data, nn): +def train_auto(train_input_data, train_desired_output_data, validate_input_data, validate_output_data, max_epochs, nn): previous_mse = 1.0 current_mse = 0.0 epochs = 0 - while(current_mse < previous_mse): + while(current_mse < previous_mse and epochs < max_epochs): epochs += 1 previous_mse = calculate_MSE(validate_input_data, validate_output_data, nn) for i in range(len(train_input_data)): diff --git a/numpy_main_2.py b/numpy_main_2.py new file mode 100644 index 0000000..9b629b6 --- /dev/null +++ b/numpy_main_2.py @@ -0,0 +1,46 @@ +import numpy as np +import numpy_neural_network_2 as nn +import numpy_z_helper_2 as h +import time +np.set_printoptions(linewidth=200) + +data_input = np.load("data/mnist_2_inputs.npy") +data_output = np.load("data/mnist_2_outputs.npy") + +print(data_input.shape) +print(data_output.shape) + +print("Begin compiling!") +np.random.seed(420) +begin_time = time.time_ns() +compiled_nn_values = nn.make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=[h.sigmoid]) +print("Created Neural network lists:", (time.time_ns()-begin_time) / 1e9) +nn.train_auto(data_input[:10], data_output[:10], data_input[10: 15], data_output[10: 15], 5, 500, compiled_nn_values) +print("Compile time:", (time.time_ns()-begin_time) / 1e9) + +total_accuracy = 0.0 +total_time = 0.0 +n = 5 +batch_size = 16 +max_epochs = 200 +for i in range(n): + + random_seed = np.random.randint(10, 1010) + np.random.seed(random_seed) + + train_input, validate_input, test_input = h.kfold(7, data_input, random_seed) + train_output, validate_output, test_output = h.kfold(7, data_output, random_seed) + nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], train_output.shape[1]], layer_activations=[h.softmax_2]) + + begin_time = time.time_ns() + epochs, current_mse = nn.train_auto(train_input, train_output, validate_input, validate_output, batch_size, max_epochs, nn_values) + end_time = time.time_ns() + + train_mse = nn.calculate_MSE(train_input, train_output, nn_values) + test_mse = nn.calculate_MSE(test_input, test_output, nn_values) + accuracy_test = nn.evaluate(test_input, test_output, nn_values) + + total_accuracy += accuracy_test + total_time += (end_time-begin_time)/1e9 + print("Seed:", random_seed, "Batch_size:", batch_size, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse) +print("Average Accuracy:", total_accuracy / n, "Average Time:", total_time / n) diff --git a/numpy_neural_network_2.py b/numpy_neural_network_2.py new file mode 100644 index 0000000..c00bc0b --- /dev/null +++ b/numpy_neural_network_2.py @@ -0,0 +1,164 @@ +import numpy as np +from numba.experimental import jitclass +from numba import njit, types, typed, prange, typeof +import numpy_z_helper_2 as h +import time + +from numba.core.errors import NumbaTypeSafetyWarning +import warnings + +warnings.simplefilter('ignore', category=NumbaTypeSafetyWarning) + +weights_type = types.float64[:, ::1] +biases_type = types.float64[::1] +spec = [ + ("layer_sizes", types.ListType(types.int64)), + ("layer_activations", types.ListType(types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean)))), + ("weights", types.ListType(weights_type)), + ("biases", types.ListType(biases_type)), + ("layer_outputs", types.ListType(types.float64[:, ::1])), + ("learning_rate", types.float64), +] +@jitclass(spec) +class NeuralNetwork: + def __init__(self, layer_sizes, layer_activations, weights, biases, layer_outputs, learning_rate): + self.layer_sizes = layer_sizes + self.layer_activations = layer_activations + self.weights = weights + self.biases = biases + self.layer_outputs = layer_outputs + self.learning_rate = learning_rate + + +def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=-2, high=2): + for size in layer_sizes: + assert size > 0 + + # Initialize typed layer sizes list. + typed_layer_sizes = typed.List() + for size in layer_sizes: + typed_layer_sizes.append(size) + + # Initialie typed layer activation method strings list. + prototype = types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean)) + typed_layer_activations = typed.List.empty_list(prototype) + for activation in layer_activations: + typed_layer_activations.append(activation) + + # Initialize weights between every neuron in all adjacent layers. + typed_weights = typed.List() + for i in range(1, len(layer_sizes)): + typed_weights.append(np.random.uniform(low, high, (layer_sizes[i-1], layer_sizes[i]))) + + # Initialize biases for every neuron in all layers + typed_biases = typed.List() + for i in range(1, len(layer_sizes)): + typed_biases.append(np.random.uniform(low, high, (layer_sizes[i],))) + + # Initialize empty list of output of every neuron in all layers. + typed_layer_outputs = typed.List() + for i in range(len(layer_sizes)): + typed_layer_outputs.append(np.zeros((layer_sizes[i], 1))) + + typed_learning_rate = learning_rate + return NeuralNetwork(typed_layer_sizes, typed_layer_activations, typed_weights, typed_biases, typed_layer_outputs, typed_learning_rate) + + +@njit +def update_parameters(weights, biases, nn): + # temp_weights = typed.List() + # temp_biases = typed.List() + # for i in range(len(weights)): + # temp_weights.append(weights[i]) + # temp_biases.append(biases[i]) + # nn.weights = temp_weights + # nn.biases = temp_biases + + nn.weights = weights + nn.biases = biases + + +@njit +def calculate_output(input_data, nn): + assert input_data.shape[1] == nn.layer_sizes[0] + y = input_data + for i in prange(len(nn.weights)): + y = nn.layer_activations[i](np.dot(y, nn.weights[i]) + nn.biases[i], False) + return y + + +@njit +def feed_forward_layers(input_data, nn): + assert input_data.shape[1] == nn.layer_sizes[0] + nn.layer_outputs[0] = input_data + for i in prange(len(nn.weights)): + nn.layer_outputs[i+1] = nn.layer_activations[i](np.dot(nn.layer_outputs[i], nn.weights[i]) + nn.biases[i], False) + + +@njit +def propogate_back(input_data, desired_output_data, nn): + + length_weights = len(nn.weights) + temp_weights = typed.List.empty_list(item_type=weights_type) + temp_biases = typed.List.empty_list(item_type=biases_type) + # temp_weights = typed.Dict.empty(key_type=types.int64, value_type=weights_type) + # temp_biases = typed.Dict.empty(key_type=types.int64, value_type=biases_type) + + error = (desired_output_data - nn.layer_outputs[-1]) * nn.layer_activations[-1](nn.layer_outputs[-1], True) + temp_weights.insert(0, nn.weights[-1] + nn.learning_rate * np.dot(nn.layer_outputs[-2].T, error) / input_data.shape[0]) + temp_biases.insert(0, nn.biases[-1] + nn.learning_rate * h.np_mean(0, error)) + # temp_weights[length_weights - 1] = nn.weights[-1] + nn.learning_rate * np.dot(nn.layer_outputs[-2].T, error) / input_data.shape[0] + # temp_biases[length_weights - 1] = nn.biases[-1] + nn.learning_rate * h.np_mean(0, error) + + for p in range(1, length_weights): + i = length_weights - p - 1 + error = np.dot(error, nn.weights[i+1].T) * nn.layer_activations[i](nn.layer_outputs[i+1], True) + temp_weights.insert(0, nn.weights[i] + nn.learning_rate * np.dot(nn.layer_outputs[i].T, error) / input_data.shape[0]) + temp_biases.insert(0, nn.biases[i] + nn.learning_rate * h.np_mean(0, error)) + # temp_weights[i] = nn.weights[i] + nn.learning_rate * np.dot(nn.layer_outputs[i].T, error) / input_data.shape[0] + # temp_biases[i] = nn.biases[i] + nn.learning_rate * h.np_mean(0, error) + + return temp_weights, temp_biases + + +@njit(parallel=True) +def calculate_MSE(input_data, desired_output_data, nn): + assert input_data.shape[0] == desired_output_data.shape[0] + sum_error = np.sum(np.power(desired_output_data - calculate_output(input_data, nn), 2)) + return sum_error / input_data.shape[0] + + +@njit +def train_auto(train_input_data, train_desired_output_data, validate_input_data, validate_output_data, batch_size, max_epochs, nn): + previous_mse = 1.0 + current_mse = 0.0 + epochs = 0 + while(current_mse < previous_mse and epochs < max_epochs): + epochs += 1 + previous_mse = calculate_MSE(validate_input_data, validate_output_data, nn) + b, e = 0, batch_size + while(e <= len(train_input_data)): + feed_forward_layers(train_input_data[b:e], nn) + temp_weights, temp_biases = propogate_back(train_input_data[b:e], train_desired_output_data[b:e], nn) + update_parameters(temp_weights, temp_biases, nn) + b += batch_size + e += batch_size + current_mse = calculate_MSE(validate_input_data, validate_output_data, nn) + return epochs, current_mse + + +@njit(parallel=True) +def evaluate(input_data, desired_output_data, nn): + output_max = h.np_argmax(1, calculate_output(input_data, nn)) + desired_output_max = h.np_argmax(1, desired_output_data) + difference_output_max = output_max - desired_output_max + correct = np.count_nonzero(difference_output_max == 0) + return correct / input_data.shape[0] + + +@njit +def print_weights_and_biases(nn): + weights = np.clip(nn.weights[0], 0.001, 0.999) + biases = np.clip(nn.biases[0], 0.001, 0.999) + print(weights) + print(biases) diff --git a/numpy_z_helper_2.py b/numpy_z_helper_2.py new file mode 100644 index 0000000..6fe2757 --- /dev/null +++ b/numpy_z_helper_2.py @@ -0,0 +1,89 @@ +import numpy as np +from numba import njit + + +def import_from_csv(path, data_type): + return np.genfromtxt(path, dtype=data_type, delimiter=',') + + +def class_to_array(maximum_class, x): + data = np.zeros(maximum_class) + 0.01 + data[x-1] = 0.99 + return data + + +def kfold(k, data, seed=99): + np.random.seed(seed) + data = np.random.permutation(data) + fold_size = int(len(data) / k) + return data[fold_size*2:], data[:fold_size], data[fold_size:fold_size*2] + + +@njit +def np_func(npfunc, axis, arr): + assert arr.ndim == 2 + assert axis in [0, 1] + if axis == 0: + result = np.empty(arr.shape[1]) + for i in range(len(result)): + result[i] = npfunc(arr[:, i]) + else: + result = np.empty(arr.shape[0]) + for i in range(len(result)): + result[i] = npfunc(arr[i, :]) + return result + + +@njit +def np_argmax(axis, arr): + # return arr.argmax(axis=axis) + return np_func(np.argmax, axis, arr) + + +@njit +def np_max(axis, arr): + # return arr.max(axis=axis) + return np_func(np.max, axis, arr) + + +@njit +def np_mean(axis, arr): + # return arr.mean(axis=axis) + return np_func(np.mean, axis, arr) + +@njit('float64[:, ::1](float64[:, ::1], boolean)') +def sigmoid(x, derivative): + if derivative: + return x * (1.0 - x) + else: + return 1.0 / (1.0 + np.exp(-x)) + + +@njit('float64[:, ::1](float64[:, ::1], boolean)') +def relu(x, derivative): + if derivative: + return np.where(x <= 0.0, 0.0, 1.0) + else: + return np.maximum(0.0, x) + + +@njit('float64[:, ::1](float64[:, ::1], boolean)') +def leaky_relu(x, derivative): + if derivative: + return np.where(x <= 0.0, -0.01*x, 1.0) + else: + return np.maximum(-0.01*x, x) + + +@njit('float64[:, ::1](float64[:, ::1], boolean)') +def softmax(x, derivative): + e_x = np.exp(x - np.max(x)) + result = e_x / e_x.sum() + return result + + +@njit('float64[:, ::1](float64[:, ::1], boolean)') +def softmax_2(x, derivative): + tmp = x - np_max(1, x).reshape(-1, 1) + exp_tmp = np.exp(tmp) + return exp_tmp / exp_tmp.sum(axis=1).reshape(-1, 1) diff --git a/prepare_data.py b/prepare_data.py index 65414a8..7ccd171 100644 --- a/prepare_data.py +++ b/prepare_data.py @@ -15,13 +15,13 @@ def prepare_mnist_data(): data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output]) print(6) - data_input = data_input.reshape((len(data_input), -1, 1)) - print(7) - data_output = data_output.reshape((len(data_output), -1, 1)) - print(8) + # data_input = data_input.reshape((len(data_input), -1, 1)) + # print(7) + # data_output = data_output.reshape((len(data_output), -1, 1)) + # print(8) - np.save("data/mnist_inputs", data_input) - np.save("data/mnist_outputs", data_output) + np.save("data/mnist_2_inputs", data_input) + np.save("data/mnist_2_outputs", data_output) def prepare_ci_data(): @@ -29,9 +29,13 @@ def prepare_ci_data(): data_output = h.import_from_csv("data/ci_targets.txt", int) data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output]) - data_input = data_input.reshape((len(data_input), -1, 1)) - data_output = data_output.reshape((len(data_output), -1, 1)) - np.save("data/ci_inputs", data_input) - np.save("data/ci_outputs", data_output) + + # print(data_input.shape) + # print(data_output.shape) + + # data_input = data_input.reshape((len(data_input), -1, 1)) + # data_output = data_output.reshape((len(data_output), -1, 1)) + np.save("data/ci_2_inputs", data_input) + np.save("data/ci_2_outputs", data_output) prepare_mnist_data() \ No newline at end of file diff --git a/z_helper.py b/z_helper.py index 024a6ed..93b5511 100644 --- a/z_helper.py +++ b/z_helper.py @@ -1,5 +1,5 @@ import numpy as np -from numba import njit +from numba import njit, typeof def import_from_csv(path, data_type): @@ -46,4 +46,5 @@ def leaky_relu(x, derivative): @njit('float64[:, ::1](float64[:, ::1], boolean)') def softmax(x, derivative): e_x = np.exp(x - np.max(x)) - return e_x / e_x.sum() + result = e_x / e_x.sum() + return result pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy