Now using jitclass instead of tuple

Capsar · Capsar · commit f9317d047906 · 2020-05-02T13:57:01.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@ __pycache__/
 data/mnist_train.csv
 data/mnist_outputs.npy
 data/mnist_inputs.npy
+other/mlp.ipynb
diff --git a/main.py b/main.py
@@ -2,22 +2,22 @@
 import numba_neural_network as nn
 import z_helper as h
 import time
-
 np.set_printoptions(linewidth=200)
 
 data_input = np.load("data/ci_inputs.npy")
 data_output = np.load("data/ci_outputs.npy")
 
 print("Begin compiling!")
 begin_time = time.time_ns()
-compiled_nn_values = nn.make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=["sigmoid"])
+compiled_nn_values = nn.make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=[h.softmax])
 nn.train_auto(data_input[:1], data_output[:1], data_input[1: 2], data_output[1: 2], compiled_nn_values)
 end_time = time.time_ns()
 print("Compile time:", (end_time-begin_time) / 1e9)
 
 np.random.seed(420)
-total_time = 0.0
-n = 1
+total_accuracy = 0.0
+begin_total = time.time_ns()
+n = 10
 for i in range(n):
 
     random_seed = np.random.randint(10, 1010)
@@ -26,14 +26,7 @@
     train_input, validate_input, test_input = h.kfold(7, data_input, random_seed)
     train_output, validate_output, test_output = h.kfold(7, data_output, random_seed)
 
-    nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=["sigmoid", "sigmoid"])
-    # n2 = 100
-    # begin_time = time.time_ns()
-    # for ii in range(n2):
-    #     nn.calculate_MSE(validate_input, validate_output, nn_values)
-    #     # nn.train_single(validate_input[ii], validate_output[ii], nn_values)
-    # end_time = time.time_ns()
-    # print("Average 1:", ((end_time-begin_time)/1e9) / n2)
+    nn_values = nn.make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=[h.sigmoid, h.softmax])
 
     begin_time = time.time_ns()
     epochs, current_mse = nn.train_auto(train_input, train_output, validate_input, validate_output, nn_values)
@@ -43,6 +36,6 @@
     test_mse = nn.calculate_MSE(test_input, test_output, nn_values)
 
     accuracy_test = nn.evaluate(test_input, test_output, nn_values)
-    total_time += (end_time-begin_time)/1e9
+    total_accuracy += accuracy_test
     print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse)
-print("Average 2:", total_time / n)
+print("Average Accuracy:", total_accuracy / n, "Average Time:", ((time.time_ns()-begin_total)/1e9) / n)
diff --git a/numba_neural_network.py b/numba_neural_network.py
@@ -1,11 +1,36 @@
 import numpy as np
-# import cupy as cp
+from numba.experimental import jitclass
 from numba import njit, types, typed, prange
 import z_helper as h
 import time
 
+from numba.core.errors import NumbaTypeSafetyWarning
+import warnings
+
+warnings.simplefilter('ignore', category=NumbaTypeSafetyWarning)
+
+spec = [
+    ("layer_sizes", types.ListType(types.int64)),
+    ("layer_activations", types.ListType(types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean)))),
+    ("weights", types.ListType(types.float64[:, ::1])),
+    ("biases", types.ListType(types.float64[:, ::1])),
+    ("layer_outputs", types.ListType(types.float64[:, ::1])),
+    ("learning_rate", types.float64),
+]
+@jitclass(spec)
+class NeuralNetwork:
+    def __init__(self, layer_sizes, layer_activations, weights, biases, layer_outputs, learning_rate):
+        self.layer_sizes = layer_sizes
+        self.layer_activations = layer_activations
+        self.weights = weights
+        self.biases = biases
+        self.layer_outputs = layer_outputs
+        self.learning_rate = learning_rate
+
 
 def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=-2, high=2):
+    for size in layer_sizes:
+        assert size > 0
 
     # Initialize typed layer sizes list.
     typed_layer_sizes = typed.List()
@@ -14,10 +39,10 @@ def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=
     # print(typeof(typed_layer_sizes))
 
     # Initialie typed layer activation method strings list.
-    typed_layer_activations = typed.List()
+    prototype = types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean))
+    typed_layer_activations = typed.List.empty_list(prototype)
     for activation in layer_activations:
         typed_layer_activations.append(activation)
-    # print(typeof(typed_layer_activations))
 
     # Initialize weights between every neuron in all adjacent layers.
     typed_weights = typed.List()
@@ -38,49 +63,42 @@ def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=
     # print(typeof(typed_layer_outputs))
 
     typed_learning_rate = learning_rate
-    return (typed_layer_sizes, typed_layer_activations, typed_weights, typed_biases, typed_layer_outputs, typed_learning_rate)
-
-# typed_layer_sizes = 0
-# typed_layer_activations = 1
-# typed_weights = 2
-# typed_biases = 3
-# typed_layer_outputs = 4
-# typed_learning_rate = 5
+    return NeuralNetwork(typed_layer_sizes, typed_layer_activations, typed_weights, typed_biases, typed_layer_outputs, typed_learning_rate)
 
 
 @njit
 def calculate_output(input_data, nn):
-    assert len(input_data) == nn[0][0]
+    assert len(input_data) == nn.layer_sizes[0]
     y = input_data
-    for i in prange(len(nn[2])):
-        y = h.activation(np.dot(nn[2][i].T, y) + nn[3][i], nn[1][i], False)
+    for i in prange(len(nn.weights)):
+        y = nn.layer_activations[i](np.dot(nn.weights[i].T, y) + nn.biases[i], False)
     return y
 
 
 @njit
 def feed_forward_layers(input_data, nn):
-    assert len(input_data) == nn[0][0]
-    nn[4][0] = input_data
-    for i in range(len(nn[2])):
-        nn[4][i+1] = h.activation(np.dot(nn[2][i].T, nn[4][i]) + nn[3][i], nn[1][i], False)
+    assert len(input_data) == nn.layer_sizes[0]
+    nn.layer_outputs[0] = input_data
+    for i in prange(len(nn.weights)):
+        nn.layer_outputs[i+1] = nn.layer_activations[i](np.dot(nn.weights[i].T, nn.layer_outputs[i]) + nn.biases[i], False)
 
 
 @njit
 def train_single(input_data, desired_output_data, nn):
-    assert len(input_data) == nn[0][0]
-    assert len(desired_output_data) == nn[0][-1]
+    assert len(input_data) == nn.layer_sizes[0]
+    assert len(desired_output_data) == nn.layer_sizes[-1]
     feed_forward_layers(input_data, nn)
 
-    error = (desired_output_data - nn[4][-1]) * h.activation(nn[4][-1], nn[1][-1], True)
-    nn[2][-1] += (nn[5] * nn[4][-2] * error.T)
-    nn[3][-1] += nn[5] * error
+    error = (desired_output_data - nn.layer_outputs[-1]) * nn.layer_activations[-1](nn.layer_outputs[-1], True)
+    nn.weights[-1] += nn.learning_rate * nn.layer_outputs[-2] * error.T
+    nn.biases[-1] += nn.learning_rate * error
 
-    length_weights = len(nn[2])
-    for i in range(1, length_weights):
+    length_weights = len(nn.weights)
+    for i in prange(1, length_weights):
         i = length_weights - i - 1
-        error = np.dot(nn[2][i+1], error) * h.activation(nn[4][i+1], nn[1][i], True)
-        nn[2][i] += (nn[5] * nn[4][i] * error.T)
-        nn[3][i] += nn[5] * error
+        error = np.dot(nn.weights[i+1], error) * nn.layer_activations[i](nn.layer_outputs[i+1], True)
+        nn.weights[i] += nn.learning_rate * nn.layer_outputs[i] * error.T
+        nn.biases[i] += nn.learning_rate * error
     return nn
 
 
@@ -135,5 +153,5 @@ def evaluate(input_data, desired_output_data, nn):
 
 @njit
 def print_weights_and_biases(nn):
-    print(nn[2])
-    print(nn[3])
+    print(nn.weights)
+    print(nn.biases)
diff --git a/z_helper.py b/z_helper.py
@@ -1,6 +1,5 @@
 import numpy as np
-from numba import cuda
-from numba import njit, types, typed
+from numba import njit
 
 
 def import_from_csv(path, data_type):
@@ -19,49 +18,32 @@ def kfold(k, data, seed=99):
     fold_size = int(len(data) / k)
     return data[fold_size*2:], data[:fold_size], data[fold_size:fold_size*2]
 
-# @cuda.jit
-# def multiply_stride(a, b, c): 
-#   s1, s2 = cuda.grid(2)
-#   d1, d2 = cuda.gridsize(2)
-#   for i1 in range(s1, a.shape[0], d1): 
-#     for i2 in range(s2, b.shape[1], d2): 
-#       the_sum = 0
-#       for k in range(b.shape[0]): # or a.shape[1] 
-#         the_sum += a[i1][k]*b[k][i2]
-#       c[i1, i2] = the_sum
 
-
-# @njit
-# def multiply(a, b):
-#     d_a = cuda.to_device(a)
-#     d_b = cuda.to_device(b)
-#     c = np.zeros((a.shape[0], b.shape[1]))
-#     d_c = cuda.to_device(c)
-#     multiply_stride[(1,), (2,2)](d_a, d_b, d_c)
-#     print(d_c.copy_to_host())
-
-
-@njit
-def activation(x, ftype, derivative):
+@njit('float64[:, ::1](float64[:, ::1], boolean)')
+def sigmoid(x, derivative):
     if derivative:
         return x * (1.0 - x)
     else:
         return 1.0 / (1.0 + np.exp(-x))
 
 
-@njit
+@njit('float64[:, ::1](float64[:, ::1], boolean)')
 def relu(x, derivative):
     if derivative:
-        return np.where(x <= 0, 0, 1)
+        return np.where(x <= 0.0, 0.0, 1.0)
     else:
-        return np.maximum(0, x)
+        return np.maximum(0.0, x)
 
 
-def softmax(x):
-    return np.exp(x) / np.sum(np.exp(x))
-
+@njit('float64[:, ::1](float64[:, ::1], boolean)')
+def leaky_relu(x, derivative):
+    if derivative:
+        return np.where(x <= 0.0, -0.01*x, 1.0)
+    else:
+        return np.maximum(-0.01*x, x)
 
-# aa = np.arange(1000*1000).reshape(1000,1000)
-# bb = np.arange(1000*1000).reshape(1000,1000)
 
-# multiply(aa, bb)
+@njit('float64[:, ::1](float64[:, ::1], boolean)')
+def softmax(x, derivative):
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum()
