Skip to content

Commit 3385818

Browse files
CapsarCapsar
committed
Merge branch 'numba'
2 parents 5d5bc66 + def0b67 commit 3385818

File tree

9 files changed

+187
-82
lines changed

9 files changed

+187
-82
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
.vscode/
22
__pycache__/
3+
4+
data/mnist_train.csv

‎README.md

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,38 @@ If you have any tips on how to imporve performace, let me know!
77

88
```
99
import numpy as np
10-
import z_helper as h
10+
from numba.experimental import jitclass
11+
from numba import types, typed
1112
```
1213

1314
```
14-
random_seed = random.randint(10, 1010)
15-
np.random.seed(random_seed)
15+
data_input = np.load("data/ci_inputs.npy")
16+
data_output = np.load("data/ci_outputs.npy")
17+
18+
print("Begin compiling!")
19+
begin_time = time.time_ns()
20+
compile_nn = make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=["sigmoid"])
21+
compile_nn.train(data_input[:1], data_output[:1], data_input[1: 2], data_output[1: 2])
22+
end_time = time.time_ns()
23+
print("Compile time:", (end_time-begin_time) / 1e9)
1624
17-
data_input = h.import_from_csv("data/features.txt", float)
18-
data_output = h.import_from_csv("data/targets.txt", int)
19-
data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output])
25+
for i in range(10):
26+
27+
random_seed = np.random.randint(10, 1010)
28+
np.random.seed(random_seed)
2029
2130
train_input, validate_input, test_input = h.kfold(4, data_input, random_seed)
2231
train_output, validate_output, test_output = h.kfold(4, data_output, random_seed)
2332
24-
nn = NeuralNetwork(layer_sizes=[10, 15, 7], layer_activations=["sigmoid", "sigmoid"])
25-
26-
# print("Beginning training")
27-
previous_mse = 1
28-
current_mse = 0
29-
epochs = 0
30-
while(current_mse < previous_mse):
31-
previous_mse = h.calculate_MSE(nn, validate_input, validate_output)
32-
for i in range(len(train_input)):
33-
nn.train(train_input[i], train_output[i])
34-
current_mse = h.calculate_MSE(nn, validate_input, validate_output)
35-
36-
epochs += 1
37-
# if epochs % 10 == 0: print("Epoch: " + str(epochs) + " MSE: " + str(current_mse))
38-
39-
40-
train_mse = h.calculate_MSE(nn, train_input, train_output)
41-
test_mse = h.calculate_MSE(nn, test_input, test_output)
42-
print("Random_Seed: " + str(random_seed) + " Epochs: " + str(epochs) + " Tr: " + str(train_mse) + " V: " + str(current_mse) + " T: " + str(test_mse))
33+
nn = make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=["sigmoid", "sigmoid"])
34+
35+
begin_time = time.time_ns()
36+
epochs, current_mse = nn.train(train_input, train_output, validate_input, validate_output)
37+
end_time = time.time_ns()
38+
39+
train_mse = nn.calculate_MSE(train_input, train_output)
40+
test_mse = nn.calculate_MSE(test_input, test_output)
41+
42+
accuracy_test = nn.evaluate(test_input, test_output)
43+
print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse)
4344
```

‎data/features.txt renamed to ‎data/ci_features.txt

File renamed without changes.

‎data/ci_inputs.npy

614 KB
Binary file not shown.

‎data/ci_outputs.npy

430 KB
Binary file not shown.

‎data/targets.txt renamed to ‎data/ci_targets.txt

File renamed without changes.

‎main.py

Lines changed: 108 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,69 @@
11
import numpy as np
2+
from numba.experimental import jitclass
3+
from numba import types, typed
24
import z_helper as h
35
import time
46

57

8+
def make_neural_network(layer_sizes, layer_activations, learning_rate=0.05, low=-2, high=2):
9+
10+
# Initialize typed layer sizes list.
11+
typed_layer_sizes = typed.List()
12+
for size in layer_sizes:
13+
typed_layer_sizes.append(size)
14+
# print(typeof(typed_layer_sizes))
15+
16+
# Initialie typed layer activation method strings list.
17+
typed_layer_activations = typed.List()
18+
for activation in layer_activations:
19+
typed_layer_activations.append(activation)
20+
# print(typeof(typed_layer_activations))
21+
22+
# Initialize weights between every neuron in all adjacent layers.
23+
typed_weights = typed.List()
24+
for i in range(1, len(layer_sizes)):
25+
typed_weights.append(np.random.uniform(low, high, (layer_sizes[i-1], layer_sizes[i])))
26+
# print(typeof(typed_weights))
27+
28+
# Initialize biases for every neuron in all layers
29+
typed_biases = typed.List()
30+
for i in range(1, len(layer_sizes)):
31+
typed_biases.append(np.random.uniform(low, high, (layer_sizes[i], 1)))
32+
# print(typeof(typed_biases))
33+
34+
# Initialize empty list of output of every neuron in all layers.
35+
typed_layer_outputs = typed.List()
36+
for i in range(len(layer_sizes)):
37+
typed_layer_outputs.append(np.zeros((layer_sizes[i], 1)))
38+
# print(typeof(typed_layer_outputs))
39+
40+
return NeuralNetwork(typed_layer_sizes, typed_layer_activations, typed_weights, typed_biases, typed_layer_outputs, learning_rate, low, high)
41+
42+
43+
spec = [
44+
("layer_sizes", types.ListType(types.int64)),
45+
("layer_activations", types.ListType(types.string)),
46+
("weights", types.ListType(types.float64[:, ::1])),
47+
("biases", types.ListType(types.float64[:, ::1])),
48+
("layer_outputs", types.ListType(types.float64[:, ::1])),
49+
("learning_rate", types.float64),
50+
("low", types.int64),
51+
("high", types.int64)
52+
]
53+
@jitclass(spec)
654
class NeuralNetwork:
7-
def __init__(self, layer_sizes, layer_activations, learning_rate=0.1, low=-2, high=2):
55+
def __init__(self, layer_sizes, layer_activations, weights, biases, layer_outputs, learning_rate, low, high):
856
assert len(layer_sizes) >= 2
957
assert len(layer_sizes)-1 == len(layer_activations)
1058

11-
# Initialize weights between every neuron in all adjacent layers.
12-
self.weights = np.array([np.random.uniform(low, high, (layer_sizes[i-1], layer_sizes[i])) for i in range(1, len(layer_sizes))])
13-
# Initialize biases for every neuron in all layers
14-
15-
self.biases = np.array([np.random.uniform(low, high, (layer_sizes[i], 1)) for i in range(1, len(layer_sizes))])
16-
# Initialize empty list of output of every neuron in all layers.
17-
self.layer_outputs = np.array([np.zeros((layer_sizes[i], 1)) for i in range(len(layer_sizes))])
59+
self.layer_sizes = layer_sizes
1860

61+
# Initialize list with activation functions per layer.
1962
self.layer_activations = layer_activations
20-
self.layer_sizes = layer_sizes
63+
self.weights = weights
64+
self.biases = biases
65+
self.layer_outputs = layer_outputs
66+
2167
self.learning_rate = learning_rate
2268

2369
def calculate_output(self, input_data):
@@ -26,22 +72,24 @@ def calculate_output(self, input_data):
2672
y = input_data
2773
self.layer_outputs[0] = y
2874

29-
for i in range(self.weights.shape[0]):
30-
y = self.layer_activations[i](np.dot(self.weights[i].T, y) + self.biases[i], False)
75+
for i in range(len(self.weights)):
76+
y = h.activation(np.dot(self.weights[i].T, y) + self.biases[i], self.layer_activations[i], False)
3177
self.layer_outputs[i+1] = y
3278
return y
3379

34-
def train(self, input_data, desired_output_data):
35-
assert input_data.shape[0] == self.layer_sizes[0]
36-
assert desired_output_data.shape[0] == self.layer_sizes[-1]
80+
def train_single(self, input_data, desired_output_data):
81+
assert len(input_data) == self.layer_sizes[0]
82+
assert len(desired_output_data) == self.layer_sizes[-1]
3783
self.calculate_output(input_data)
3884

39-
error = (desired_output_data - self.layer_outputs[-1]) * self.layer_activations[-1](self.layer_outputs[-1], True)
85+
error = (desired_output_data - self.layer_outputs[-1]) * h.activation(self.layer_outputs[-1], self.layer_activations[-1], True)
4086
self.weights[-1] += (self.learning_rate * self.layer_outputs[-2] * error.T)
4187
self.biases[-1] += self.learning_rate * error
4288

43-
for i in reversed(range(self.weights.shape[0]-1)):
44-
error = np.dot(self.weights[i+1], error) * self.layer_activations[i](self.layer_outputs[i+1], True)
89+
length_weights = len(self.weights)
90+
for i in range(1, length_weights):
91+
i = length_weights - i - 1
92+
error = np.dot(self.weights[i+1], error) * h.activation(self.layer_outputs[i+1], self.layer_activations[i], True)
4593
self.weights[i] += (self.learning_rate * self.layer_outputs[i] * error.T)
4694
self.biases[i] += self.learning_rate * error
4795

@@ -50,51 +98,71 @@ def calculate_SSE(self, input_data, desired_output_data):
5098
assert desired_output_data.shape[0] == self.layer_sizes[-1]
5199
return np.sum(np.power(desired_output_data - self.calculate_output(input_data), 2))
52100

53-
def calculate_MSE(self, input_data, output_data):
54-
assert input_data.shape[0] == output_data.shape[0]
101+
def calculate_MSE(self, input_data, desired_output_data):
102+
assert input_data.shape[0] == desired_output_data.shape[0]
55103
size = input_data.shape[0]
56104
sum_error = 0
57105
for i in range(size):
58-
sum_error += self.calculate_SSE(input_data[i], output_data[i])
106+
sum_error += self.calculate_SSE(input_data[i], desired_output_data[i])
59107
return sum_error / size
60108

109+
def train(self, test_input_data, test_desired_output_data, validate_input_data, validate_output_data):
110+
previous_mse = 1.0
111+
current_mse = 0.0
112+
epochs = 0
113+
while(current_mse < previous_mse):
114+
epochs += 1
115+
previous_mse = self.calculate_MSE(validate_input_data, validate_output_data)
116+
for i in range(len(test_input_data)):
117+
self.train_single(test_input_data[i], test_desired_output_data[i])
118+
current_mse = self.calculate_MSE(validate_input_data, validate_output_data)
119+
return epochs, current_mse
120+
121+
def evaluate(self, input_data, desired_output_data):
122+
corrects, wrongs = 0, 0
123+
for i in range(len(input_data)):
124+
output = self.calculate_output(input_data[i])
125+
output_max = output.argmax()
126+
desired_output_max = desired_output_data[i].argmax()
127+
if output_max == desired_output_max:
128+
corrects += 1
129+
else:
130+
wrongs += 1
131+
return corrects / (corrects + wrongs)
132+
61133
def print_weights_and_biases(self):
62134
print(self.weights)
63135
print(self.biases)
64136

65137

66138
np.set_printoptions(linewidth=200)
67139

68-
data_input = h.import_from_csv("data/features.txt", float)
69-
data_output = h.import_from_csv("data/targets.txt", int)
70-
data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output])
140+
data_input = np.load("data/ci_inputs.npy")
141+
data_output = np.load("data/ci_outputs.npy")
71142

72-
data_input = data_input.reshape((len(data_input), -1, 1))
73-
data_output = data_output.reshape((len(data_input), -1, 1))
143+
print("Begin compiling!")
144+
begin_time = time.time_ns()
145+
compile_nn = make_neural_network(layer_sizes=[data_input.shape[1], data_output.shape[1]], layer_activations=["sigmoid"])
146+
compile_nn.train(data_input[:1], data_output[:1], data_input[1: 2], data_output[1: 2])
147+
end_time = time.time_ns()
148+
print("Compile time:", (end_time-begin_time) / 1e9)
74149

75-
for i in range(4):
76-
random_seed = 10
150+
for i in range(10):
151+
152+
random_seed = np.random.randint(10, 1010)
77153
np.random.seed(random_seed)
78154

79155
train_input, validate_input, test_input = h.kfold(4, data_input, random_seed)
80156
train_output, validate_output, test_output = h.kfold(4, data_output, random_seed)
81157

82-
nn = NeuralNetwork(layer_sizes=[10, 15, 7], layer_activations=[h.sigmoid, h.sigmoid])
83-
# test_mse = nn.calculate_MSE(test_input, test_output)
84-
# print("TEST MSE:", test_mse)
158+
nn = make_neural_network(layer_sizes=[train_input.shape[1], 20, train_output.shape[1]], layer_activations=["sigmoid", "sigmoid"])
85159

86-
previous_mse = 1
87-
current_mse = 0
88-
epochs = 0
89160
begin_time = time.time_ns()
90-
while(current_mse < previous_mse):
91-
epochs += 1
92-
previous_mse = nn.calculate_MSE(validate_input, validate_output)
93-
for i in range(len(train_input)):
94-
nn.train(train_input[i], train_output[i])
95-
current_mse = nn.calculate_MSE(validate_input, validate_output)
161+
epochs, current_mse = nn.train(train_input, train_output, validate_input, validate_output)
96162
end_time = time.time_ns()
97163

98164
train_mse = nn.calculate_MSE(train_input, train_output)
99165
test_mse = nn.calculate_MSE(test_input, test_output)
100-
print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Tr:", train_mse, "V:", current_mse, "T:", test_mse)
166+
167+
accuracy_test = nn.evaluate(test_input, test_output)
168+
print("Seed:", random_seed, "Epochs:", epochs, "Time:", (end_time-begin_time)/1e9, "Accuracy:", accuracy_test, "Tr:", train_mse, "V:", current_mse, "T:", test_mse)

‎prepare_data.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import numpy as np
2+
import z_helper as h
3+
4+
5+
def prepare_mnist_data():
6+
print(1)
7+
mnist_dataset = h.import_from_csv("data/mnist_train.csv", int)
8+
print(2)
9+
data_input = mnist_dataset[:, 1:].astype(float)
10+
print(3)
11+
data_input = data_input * (0.99 / 255.0) + 0.01
12+
print(4)
13+
data_output = mnist_dataset[:, :1].astype(int)
14+
print(5)
15+
16+
data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output])
17+
print(6)
18+
data_input = data_input.reshape((len(data_input), -1, 1))
19+
print(7)
20+
data_output = data_output.reshape((len(data_output), -1, 1))
21+
print(8)
22+
23+
np.save("data/mnist_inputs", data_input)
24+
np.save("data/mnist_outputs", data_output)
25+
26+
27+
def prepare_ci_data():
28+
data_input = h.import_from_csv("data/ci_features.txt", float)
29+
data_output = h.import_from_csv("data/ci_targets.txt", int)
30+
31+
data_output = np.array([h.class_to_array(np.amax(data_output), x) for x in data_output])
32+
data_input = data_input.reshape((len(data_input), -1, 1))
33+
data_output = data_output.reshape((len(data_output), -1, 1))
34+
np.save("data/ci_inputs", data_input)
35+
np.save("data/ci_outputs", data_output)
36+
37+
prepare_ci_data()

‎z_helper.py

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import numpy as np
2-
2+
from numba import njit, types, typed
33

44
def import_from_csv(path, data_type):
55
return np.genfromtxt(path, dtype=data_type, delimiter=',')
66

77

88
def class_to_array(maximum_class, x):
9-
data = np.zeros(maximum_class)
10-
data[x-1] = 1
9+
data = np.zeros(maximum_class) + 0.01
10+
data[x-1] = 0.99
1111
return data
1212

1313

@@ -18,26 +18,23 @@ def kfold(k, data, seed=99):
1818
return data[fold_size*2:], data[:fold_size], data[fold_size:fold_size*2]
1919

2020

21-
def random_np(low, high, size):
22-
assert low <= high
23-
return np.random.random(size)*(high-low) + low
24-
25-
26-
def sigmoid(x, derivative):
27-
if derivative:
28-
return x * (1.0 - x)
29-
else:
30-
return 1.0 / (1.0 + np.exp(-x))
31-
21+
@njit
22+
def activation(x, ftype, derivative):
23+
if ftype == "sigmoid":
24+
if derivative:
25+
return x * (1.0 - x)
26+
else:
27+
return 1.0 / (1.0 + np.exp(-x))
3228

29+
@njit
3330
def relu(x, derivative):
3431
if derivative:
35-
x[x <= 0] = 0
36-
x[x > 0] = 1
37-
return x
32+
return np.where(x <= 0, 0, 1)
3833
else:
3934
return np.maximum(0, x)
4035

4136

4237
def softmax(x):
4338
return np.exp(x) / np.sum(np.exp(x))
39+
40+

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy