new attempt at batching training

Capsar · Capsar · commit 2bf0f7b2bfad · 2020-05-02T23:36:31.000+02:00
diff --git a/data/ci_2_inputs.npy b/data/ci_2_inputs.npy
diff --git a/data/ci_2_outputs.npy b/data/ci_2_outputs.npy
diff --git a/numpy_main_2.py b/numpy_main_2.py
@@ -0,0 +1,43 @@
+import numpy as np
+import numpy_neural_network_2 as nn
+import z_helper as h
+import time
+np.set_printoptions(linewidth=200)
+
+data_input = np.load("data/ci_2_inputs.npy")
+data_output = np.load("data/ci_2_outputs.npy")
+
+print(data_input.shape)
+print(data_output.shape)
+
+print("Begin compiling!")
+begin_time = time.time_ns()
+compiled_nn_values = nn.make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=[h.sigmoid])
+nn.train_auto(data_input[:10], data_output[:10], data_input[10: 15], data_output[10: 15], compiled_nn_values)
+end_time = time.time_ns()
+print("Compile time:", (end_time-begin_time) / 1e9)
+
+np.random.seed(420)
+total_accuracy = 0.0
+begin_total = time.time_ns()
+n = 10
+for i in range(n):
+
+    random_seed = np.random.randint(10, 1010)
+    np.random.seed(random_seed)
+
+    train_input, validate_input, test_input = h.kfold(7, data_input, random_seed)
+    train_output, validate_output, test_output = h.kfold(7, data_output, random_seed)
+    nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=[h.sigmoid, h.softmax_2])
+
+    begin_time = time.time_ns()
+    epochs, current_mse = nn.train_auto(train_input, train_output, validate_input, validate_output, nn_values)
+    end_time = time.time_ns()
+
+    train_mse = nn.calculate_MSE(train_input, train_output, nn_values)
+    test_mse = nn.calculate_MSE(test_input, test_output, nn_values)
+
+    accuracy_test = nn.evaluate(test_input, test_output, nn_values)
+    total_accuracy += accuracy_test
+    print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse)
+print("Average Accuracy:", total_accuracy / n, "Average Time:", ((time.time_ns()-begin_total)/1e9) / n)
diff --git a/numpy_neural_network_2.py b/numpy_neural_network_2.py
@@ -0,0 +1,156 @@
+import numpy as np
+from numba.experimental import jitclass
+from numba import njit, types, typed, prange
+import z_helper as h
+import time
+
+from numba.core.errors import NumbaTypeSafetyWarning
+import warnings
+
+warnings.simplefilter('ignore', category=NumbaTypeSafetyWarning)
+
+# spec = [
+#     ("layer_sizes", types.ListType(types.int64)),
+#     ("layer_activations", types.ListType(types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean)))),
+#     ("weights", types.ListType(types.float64[:, ::1])),
+#     ("biases", types.ListType(types.float64[:, ::1])),
+#     ("layer_outputs", types.ListType(types.float64[:, ::1])),
+#     ("learning_rate", types.float64),
+# ]
+# @jitclass(spec)
+
+
+class NeuralNetwork:
+    def __init__(self, layer_sizes, layer_activations, weights, biases, layer_outputs, learning_rate):
+        self.layer_sizes = layer_sizes
+        self.layer_activations = layer_activations
+        self.weights = weights
+        self.biases = biases
+        self.layer_outputs = layer_outputs
+        self.learning_rate = learning_rate
+
+
+def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=-2, high=2):
+    for size in layer_sizes:
+        assert size > 0
+
+    # Initialize typed layer sizes list.
+    # typed_layer_sizes = typed.List()
+    # for size in layer_sizes:
+    #     typed_layer_sizes.append(size)
+    # print(typeof(typed_layer_sizes))
+    typed_layer_sizes = layer_sizes
+
+    # Initialie typed layer activation method strings list.
+    # prototype = types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean))
+    # typed_layer_activations = typed.List.empty_list(prototype)
+    # for activation in layer_activations:
+    #     typed_layer_activations.append(activation)
+    # print(typedof(typed_layer_activations))
+    typed_layer_activations = layer_activations
+
+    # Initialize weights between every neuron in all adjacent layers.
+    # typed_weights = typed.List()
+    # for i in range(1, len(layer_sizes)):
+    #     typed_weights.append(np.random.uniform(low, high, (layer_sizes[i-1], layer_sizes[i])))
+    # print(typeof(typed_weights))
+    typed_weights = [np.random.uniform(low, high, (layer_sizes[i-1], layer_sizes[i])) for i in range(1, len(layer_sizes))]
+
+    # Initialize biases for every neuron in all layers
+    # typed_biases = typed.List()
+    # for i in range(1, len(layer_sizes)):
+    #     typed_biases.append(np.random.uniform(low, high, (layer_sizes[i], 1)))
+    # print(typeof(typed_biases))
+    typed_biases = [np.random.uniform(low, high, (layer_sizes[i],)) for i in range(1, len(layer_sizes))]
+
+    # Initialize empty list of output of every neuron in all layers.
+    # typed_layer_outputs = typed.List()
+    # for i in range(len(layer_sizes)):
+    #     typed_layer_outputs.append(np.zeros((layer_sizes[i], 1)))
+    # print(typeof(typed_layer_outputs))
+    typed_layer_outputs = [np.zeros((layer_sizes[i],1)) for i in range(len(layer_sizes))]
+
+    typed_learning_rate = learning_rate
+    return NeuralNetwork(typed_layer_sizes, typed_layer_activations, typed_weights, typed_biases, typed_layer_outputs, typed_learning_rate)
+
+
+# @njit
+def calculate_output(input_data, nn):
+    assert input_data.shape[1] == nn.layer_sizes[0]
+    y = input_data
+    for i in prange(len(nn.weights)):
+        y = nn.layer_activations[i](np.dot(y, nn.weights[i]) + nn.biases[i], False)
+    return y
+
+
+# @njit
+def feed_forward_layers(input_data, nn):
+    assert input_data.shape[1] == nn.layer_sizes[0]
+    nn.layer_outputs[0] = input_data
+    for i in prange(len(nn.weights)):
+        ac = np.dot(nn.layer_outputs[i], nn.weights[i]) + nn.biases[i]
+        nn.layer_outputs[i+1] = nn.layer_activations[i](ac, False)
+
+def train_batch(input_data, desired_output_data, nn):
+    feed_forward_layers(input_data, nn)
+    error = (desired_output_data - nn.layer_outputs[-1]) * nn.layer_activations[-1](nn.layer_outputs[-1], True)
+
+    temp_weights = []
+    temp_biases = []
+
+    temp_weights.insert(0, nn.weights[-1] + nn.learning_rate * np.dot(nn.layer_outputs[-2].T, error) / input_data.shape[0])
+    temp_biases.insert(0, nn.biases[-1] + nn.learning_rate * np.average(error, axis=0))
+
+    length_weights = len(nn.weights)
+    for i in range(1, length_weights):
+        i = length_weights - i - 1
+        error = np.dot(error, nn.weights[i+1].T) * nn.layer_activations[i](nn.layer_outputs[i+1], True)
+        temp_weights.insert(0, nn.weights[i] + nn.learning_rate * np.dot(nn.layer_outputs[i].T, error) / input_data.shape[0])
+        temp_biases.insert(0, nn.biases[i] + nn.learning_rate * np.average(error, axis=0))
+
+    nn.weights = temp_weights
+    nn.biases = temp_biases
+
+
+# @njit(parallel=True)
+def calculate_MSE(input_data, desired_output_data, nn):
+    assert input_data.shape[0] == desired_output_data.shape[0]
+    sum_error = np.sum(np.power(desired_output_data - calculate_output(input_data, nn), 2))
+    return sum_error / len(input_data)
+
+
+# @njit
+def train_auto(train_input_data, train_desired_output_data, validate_input_data, validate_output_data, nn):
+    previous_mse = 1.0
+    current_mse = 0.0
+    epochs = 0
+    batch_size = 8
+
+
+    while(current_mse < previous_mse):
+        epochs += 1
+        previous_mse = calculate_MSE(validate_input_data, validate_output_data, nn)
+        b,e = 0, batch_size
+        while(e < len(train_input_data) + 1):
+            train_batch(train_input_data[b:e], train_desired_output_data[b:e], nn)
+            b += batch_size
+            e += batch_size
+        current_mse = calculate_MSE(validate_input_data, validate_output_data, nn)
+    return epochs, current_mse
+
+
+# @njit(parallel=True)
+def evaluate(input_data, desired_output_data, nn):
+    output_max = calculate_output(input_data, nn).argmax(axis=1)
+    desired_output_max = desired_output_data.argmax(axis=1)
+    difference_output_max = output_max - desired_output_max
+    correct = np.count_nonzero(difference_output_max == 0)
+    return correct / input_data.shape[0]
+
+
+# @njit
+def print_weights_and_biases(nn):
+    weights = np.clip(nn.weights[0], 0.001, 0.999)
+    biases = np.clip(nn.biases[0], 0.001, 0.999)
+    print(weights)
+    print(biases)
diff --git a/prepare_data.py b/prepare_data.py
@@ -29,9 +29,13 @@ def prepare_ci_data():
     data_output = h.import_from_csv("data/ci_targets.txt", int)
 
     data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output])
-    data_input = data_input.reshape((len(data_input), -1, 1))
-    data_output = data_output.reshape((len(data_output), -1, 1))
-    np.save("data/ci_inputs", data_input)
-    np.save("data/ci_outputs", data_output)
 
-prepare_mnist_data()
+    # print(data_input.shape)
+    # print(data_output.shape)
+
+    # data_input = data_input.reshape((len(data_input), -1, 1))
+    # data_output = data_output.reshape((len(data_output), -1, 1))
+    np.save("data/ci_2_inputs", data_input)
+    np.save("data/ci_2_outputs", data_output)
+
+prepare_ci_data()
diff --git a/z_helper.py b/z_helper.py
@@ -1,5 +1,5 @@
 import numpy as np
-from numba import njit
+from numba import njit, typeof
 
 
 def import_from_csv(path, data_type):
@@ -19,31 +19,69 @@ def kfold(k, data, seed=99):
     return data[fold_size*2:], data[:fold_size], data[fold_size:fold_size*2]
 
 
-@njit('float64[:, ::1](float64[:, ::1], boolean)')
+@njit
+def np_func(npfunc, axis, arr):
+    assert arr.ndim == 2
+    assert axis in [0, 1]
+    if axis == 0:
+        result = np.empty(arr.shape[1])
+        for i in range(len(result)):
+            result[i] = npfunc(arr[:, i])
+    else:
+        result = np.empty(arr.shape[0])
+        for i in range(len(result)):
+            result[i] = npfunc(arr[i, :])
+    return result
+
+
+@njit
+def np_argmax(axis, arr):
+    return np_func(np.argmax, axis, arr)
+
+
+@njit
+def np_max(axis, arr):
+    return np_func(np.max, axis, arr)
+
+
+@njit
+def np_mean(axis, arr):
+    return np_func(np.mean, axis, arr)
+
+
+# @njit('float64[:, ::1](float64[:, ::1], boolean)')
 def sigmoid(x, derivative):
     if derivative:
         return x * (1.0 - x)
     else:
         return 1.0 / (1.0 + np.exp(-x))
 
 
-@njit('float64[:, ::1](float64[:, ::1], boolean)')
+# @njit('float64[:, ::1](float64[:, ::1], boolean)')
 def relu(x, derivative):
     if derivative:
         return np.where(x <= 0.0, 0.0, 1.0)
     else:
         return np.maximum(0.0, x)
 
 
-@njit('float64[:, ::1](float64[:, ::1], boolean)')
+# @njit('float64[:, ::1](float64[:, ::1], boolean)')
 def leaky_relu(x, derivative):
     if derivative:
         return np.where(x <= 0.0, -0.01*x, 1.0)
     else:
         return np.maximum(-0.01*x, x)
 
 
-@njit('float64[:, ::1](float64[:, ::1], boolean)')
+# @njit('float64[:, ::1](float64[:, ::1], boolean)')
 def softmax(x, derivative):
     e_x = np.exp(x - np.max(x))
-    return e_x / e_x.sum()
+    result = e_x / e_x.sum()
+    return result
+
+
+# @njit('float64[:, ::1](float64[:, ::1], boolean)')
+def softmax_2(x, derivative):
+    tmp = x - np_max(1, x).reshape(-1, 1)
+    exp_tmp = np.exp(tmp)
+    return exp_tmp / exp_tmp.sum(axis=1).reshape(-1, 1)
