@vzhou842/

A CNN from scratch, Part 2

Python

A Convolutional Neural Network implemented from scratch (using only numpy) in Python. https://github.com/vzhou842/cnn-from-scratch

fork
loading
Files
  • main.py
  • conv.py
  • maxpool.py
  • requirements.txt
  • softmax.py

This Plugin Crashed!

Error: Error: must not create an existing file {"type":"CREATE_FILE","wid":"0.05363100278228483","path":"main.py","file":{"path":"main.py","content":{"asEncoding":{"base64":"aW1wb3J0IG1uaXN0CmltcG9ydCBudW1weSBhcyBucApmcm9tIGNvbnYgaW1wb3J0IENvbnYzeDMKZnJvbSBtYXhwb29sIGltcG9ydCBNYXhQb29sMgpmcm9tIHNvZnRtYXggaW1wb3J0IFNvZnRtYXgKCiMgV2Ugb25seSB1c2UgdGhlIGZpcnN0IDFrIGV4YW1wbGVzIG9mIGVhY2ggc2V0IGluIHRoZSBpbnRlcmVzdCBvZiB0aW1lLgojIEZlZWwgZnJlZSB0byBjaGFuZ2UgdGhpcyBpZiB5b3Ugd2FudC4KdHJhaW5faW1hZ2VzID0gbW5pc3QudHJhaW5faW1hZ2VzKClbOjEwMDBdCnRyYWluX2xhYmVscyA9IG1uaXN0LnRyYWluX2xhYmVscygpWzoxMDAwXQp0ZXN0X2ltYWdlcyA9IG1uaXN0LnRlc3RfaW1hZ2VzKClbOjEwMDBdCnRlc3RfbGFiZWxzID0gbW5pc3QudGVzdF9sYWJlbHMoKVs6MTAwMF0KCmNvbnYgPSBDb252M3gzKDgpICAgICAgICAgICAgICAgICAgIyAyOHgyOHgxIC0+IDI2eDI2eDgKcG9vbCA9IE1heFBvb2wyKCkgICAgICAgICAgICAgICAgICAjIDI2eDI2eDggLT4gMTN4MTN4OApzb2Z0bWF4ID0gU29mdG1heCgxMyAqIDEzICogOCwgMTApICMgMTN4MTN4OCAtPiAxMAoKZGVmIGZvcndhcmQoaW1hZ2UsIGxhYmVsKToKICAnJycKICBDb21wbGV0ZXMgYSBmb3J3YXJkIHBhc3Mgb2YgdGhlIENOTiBhbmQgY2FsY3VsYXRlcyB0aGUgYWNjdXJhY3kgYW5kCiAgY3Jvc3MtZW50cm9weSBsb3NzLgogIC0gaW1hZ2UgaXMgYSAyZCBudW1weSBhcnJheQogIC0gbGFiZWwgaXMgYSBkaWdpdAogICcnJwogICMgV2UgdHJhbnNmb3JtIHRoZSBpbWFnZSBmcm9tIFswLCAyNTVdIHRvIFstMC41LCAwLjVdIHRvIG1ha2UgaXQgZWFzaWVyCiAgIyB0byB3b3JrIHdpdGguIFRoaXMgaXMgc3RhbmRhcmQgcHJhY3RpY2UuCiAgb3V0ID0gY29udi5mb3J3YXJkKChpbWFnZSAvIDI1NSkgLSAwLjUpCiAgb3V0ID0gcG9vbC5mb3J3YXJkKG91dCkKICBvdXQgPSBzb2Z0bWF4LmZvcndhcmQob3V0KQoKICAjIENhbGN1bGF0ZSBjcm9zcy1lbnRyb3B5IGxvc3MgYW5kIGFjY3VyYWN5LiBucC5sb2coKSBpcyB0aGUgbmF0dXJhbCBsb2cuCiAgbG9zcyA9IC1ucC5sb2cob3V0W2xhYmVsXSkKICBhY2MgPSAxIGlmIG5wLmFyZ21heChvdXQpID09IGxhYmVsIGVsc2UgMAoKICByZXR1cm4gb3V0LCBsb3NzLCBhY2MKCmRlZiB0cmFpbihpbSwgbGFiZWwsIGxyPS4wMDUpOgogICcnJwogIENvbXBsZXRlcyBhIGZ1bGwgdHJhaW5pbmcgc3RlcCBvbiB0aGUgZ2l2ZW4gaW1hZ2UgYW5kIGxhYmVsLgogIFJldHVybnMgdGhlIGNyb3NzLWVudHJvcHkgbG9zcyBhbmQgYWNjdXJhY3kuCiAgLSBpbWFnZSBpcyBhIDJkIG51bXB5IGFycmF5CiAgLSBsYWJlbCBpcyBhIGRpZ2l0CiAgLSBsciBpcyB0aGUgbGVhcm5pbmcgcmF0ZQogICcnJwogICMgRm9yd2FyZAogIG91dCwgbG9zcywgYWNjID0gZm9yd2FyZChpbSwgbGFiZWwpCgogICMgQ2FsY3VsYXRlIGluaXRpYWwgZ3JhZGllbnQKICBncmFkaWVudCA9IG5wLnplcm9zKDEwKQogIGdyYWRpZW50W2xhYmVsXSA9IC0xIC8gb3V0W2xhYmVsXQoKICAjIEJhY2twcm9wCiAgZ3JhZGllbnQgPSBzb2Z0bWF4LmJhY2twcm9wKGdyYWRpZW50LCBscikKICBncmFkaWVudCA9IHBvb2wuYmFja3Byb3AoZ3JhZGllbnQpCiAgZ3JhZGllbnQgPSBjb252LmJhY2twcm9wKGdyYWRpZW50LCBscikKCiAgcmV0dXJuIGxvc3MsIGFjYwoKcHJpbnQoJ01OSVNUIENOTiBpbml0aWFsaXplZCEnKQoKIyBUcmFpbiB0aGUgQ05OIGZvciAzIGVwb2Nocwpmb3IgZXBvY2ggaW4gcmFuZ2UoMyk6CiAgcHJpbnQoJy0tLSBFcG9jaCAlZCAtLS0nICUgKGVwb2NoICsgMSkpCgogICMgU2h1ZmZsZSB0aGUgdHJhaW5pbmcgZGF0YQogIHBlcm11dGF0aW9uID0gbnAucmFuZG9tLnBlcm11dGF0aW9uKGxlbih0cmFpbl9pbWFnZXMpKQogIHRyYWluX2ltYWdlcyA9IHRyYWluX2ltYWdlc1twZXJtdXRhdGlvbl0KICB0cmFpbl9sYWJlbHMgPSB0cmFpbl9sYWJlbHNbcGVybXV0YXRpb25dCgogICMgVHJhaW4hCiAgbG9zcyA9IDAKICBudW1fY29ycmVjdCA9IDAKICBmb3IgaSwgKGltLCBsYWJlbCkgaW4gZW51bWVyYXRlKHppcCh0cmFpbl9pbWFnZXMsIHRyYWluX2xhYmVscykpOgogICAgaWYgaSA+IDAgYW5kIGkgJSAxMDAgPT0gOTk6CiAgICAgIHByaW50KAogICAgICAgICdbU3RlcCAlZF0gUGFzdCAxMDAgc3RlcHM6IEF2ZXJhZ2UgTG9zcyAlLjNmIHwgQWNjdXJhY3k6ICVkJSUnICUKICAgICAgICAoaSArIDEsIGxvc3MgLyAxMDAsIG51bV9jb3JyZWN0KQogICAgICApCiAgICAgIGxvc3MgPSAwCiAgICAgIG51bV9jb3JyZWN0ID0gMAoKICAgIGwsIGFjYyA9IHRyYWluKGltLCBsYWJlbCkKICAgIGxvc3MgKz0gbAogICAgbnVtX2NvcnJlY3QgKz0gYWNjCgojIFRlc3QgdGhlIENOTgpwcmludCgnXG4tLS0gVGVzdGluZyB0aGUgQ05OIC0tLScpCmxvc3MgPSAwCm51bV9jb3JyZWN0ID0gMApmb3IgaW0sIGxhYmVsIGluIHppcCh0ZXN0X2ltYWdlcywgdGVzdF9sYWJlbHMpOgogIF8sIGwsIGFjYyA9IGZvcndhcmQoaW0sIGxhYmVsKQogIGxvc3MgKz0gbAogIG51bV9jb3JyZWN0ICs9IGFjYwoKbnVtX3Rlc3RzID0gbGVuKHRlc3RfaW1hZ2VzKQpwcmludCgnVGVzdCBMb3NzOicsIGxvc3MgLyBudW1fdGVzdHMpCnByaW50KCdUZXN0IEFjY3VyYWN5OicsIG51bV9jb3JyZWN0IC8gbnVtX3Rlc3RzKQo="},"asBuffer":null},"loaded":true}}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import mnist
import numpy as np
from conv import Conv3x3
from maxpool import MaxPool2
from softmax import Softmax

# We only use the first 1k examples of each set in the interest of time.
# Feel free to change this if you want.
train_images = mnist.train_images()[:1000]
train_labels = mnist.train_labels()[:1000]
test_images = mnist.test_images()[:1000]
test_labels = mnist.test_labels()[:1000]

conv = Conv3x3(8)                  # 28x28x1 -> 26x26x8
pool = MaxPool2()                  # 26x26x8 -> 13x13x8
softmax = Softmax(13 * 13 * 8, 10) # 13x13x8 -> 10

def forward(image, label):
  '''
  Completes a forward pass of the CNN and calculates the accuracy and
  cross-entropy loss.
  - image is a 2d numpy array
  - label is a digit
  '''
  # We transform the image from [0, 255] to [-0.5, 0.5] to make it easier
  # to work with. This is standard practice.
  out = conv.forward((image / 255) - 0.5)
  out = pool.forward(out)
  out = softmax.forward(out)

  # Calculate cross-entropy loss and accuracy. np.log() is the natural log.
  loss = -np.log(out[label])
  acc = 1 if np.argmax(out) == label else 0

  return out, loss, acc

def train(im, label, lr=.005):
  '''
  Completes a full training step on the given image and label.
  Returns the cross-entropy loss and accuracy.
  - image is a 2d numpy array
  - label is a digit
  - lr is the learning rate
  '''
  # Forward
  out, loss, acc = forward(im, label)

  # Calculate initial gradient
  gradient = np.zeros(10)
  gradient[label] = -1 / out[label]

  # Backprop
  gradient = softmax.backprop(gradient, lr)
  gradient = pool.backprop(gradient)
  gradient = conv.backprop(gradient, lr)

  return loss, acc

print('MNIST CNN initialized!')

# Train the CNN for 3 epochs
for epoch in range(3):
  print('--- Epoch %d ---' % (epoch + 1))

  # Shuffle the training data
  permutation = np.random.permutation(len(train_images))
  train_images = train_images[permutation]
  train_labels = train_labels[permutation]

  # Train!
  loss = 0
  num_correct = 0
  for i, (im, label) in enumerate(zip(train_images, train_labels)):
    if i > 0 and i % 100 == 99:
      print(
        '[Step %d] Past 100 steps: Average Loss %.3f | Accuracy: %d%%' %
        (i + 1, loss / 100, num_correct)
      )
      loss = 0
      num_correct = 0

    l, acc = train(im, label)
    loss += l
    num_correct += acc

# Test the CNN
print('\n--- Testing the CNN ---')
loss = 0
num_correct = 0
for im, label in zip(test_images, test_labels):
  _, l, acc = forward(im, label)
  loss += l
  num_correct += acc

num_tests = len(test_images)
print('Test Loss:', loss / num_tests)
print('Test Accuracy:', num_correct / num_tests)