Repository navigation
Expand file tree
/
Copy pathdA.py
More file actions
executable file
·155 lines (106 loc) · 4.75 KB
/
Copy pathdA.py
File metadata and controls
executable file
·155 lines (106 loc) · 4.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Denoising Autoencoders (dA)
References :
- P. Vincent, H. Larochelle, Y. Bengio, P.A. Manzagol: Extracting and
Composing Robust Features with Denoising Autoencoders, ICML'08, 1096-1103,
2008
- DeepLearningTutorials
https://github.com/lisa-lab/DeepLearningTutorials
- Yusuke Sugomori: Stochastic Gradient Descent for Denoising Autoencoders,
http://yusugomori.com/docs/SGD_DA.pdf
"""
import sys
import numpy
from utils import *
class dA(object):
def __init__(self, input=None, n_visible=2, n_hidden=3, \
W=None, hbias=None, vbias=None, numpy_rng=None):
self.n_visible = n_visible # num of units in visible (input) layer
self.n_hidden = n_hidden # num of units in hidden layer
if numpy_rng is None:
numpy_rng = numpy.random.RandomState(1234)
if W is None:
a = 1. / n_visible
initial_W = numpy.array(numpy_rng.uniform( # initialize W uniformly
low=-a,
high=a,
size=(n_visible, n_hidden)))
W = initial_W
if hbias is None:
hbias = numpy.zeros(n_hidden) # initialize h bias 0
if vbias is None:
vbias = numpy.zeros(n_visible) # initialize v bias 0
self.numpy_rng = numpy_rng
self.x = input
self.W = W
self.W_prime = self.W.T
self.hbias = hbias
self.vbias = vbias
# self.params = [self.W, self.hbias, self.vbias]
def get_corrupted_input(self, input, corruption_level):
assert corruption_level < 1
return self.numpy_rng.binomial(size=input.shape,
n=1,
p=1-corruption_level) * input
# Encode
def get_hidden_values(self, input):
return sigmoid(numpy.dot(input, self.W) + self.hbias)
# Decode
def get_reconstructed_input(self, hidden):
return sigmoid(numpy.dot(hidden, self.W_prime) + self.vbias)
def train(self, lr=0.1, corruption_level=0.3, input=None):
if input is not None:
self.x = input
x = self.x
tilde_x = self.get_corrupted_input(x, corruption_level)
y = self.get_hidden_values(tilde_x)
z = self.get_reconstructed_input(y)
L_h2 = x - z
L_h1 = numpy.dot(L_h2, self.W) * y * (1 - y)
L_vbias = L_h2
L_hbias = L_h1
L_W = numpy.dot(tilde_x.T, L_h1) + numpy.dot(L_h2.T, y)
self.W += lr * L_W
self.hbias += lr * numpy.mean(L_hbias, axis=0)
self.vbias += lr * numpy.mean(L_vbias, axis=0)
def negative_log_likelihood(self, corruption_level=0.3):
tilde_x = self.get_corrupted_input(self.x, corruption_level)
y = self.get_hidden_values(tilde_x)
z = self.get_reconstructed_input(y)
cross_entropy = - numpy.mean(
numpy.sum(self.x * numpy.log(z) +
(1 - self.x) * numpy.log(1 - z),
axis=1))
return cross_entropy
def reconstruct(self, x):
y = self.get_hidden_values(x)
z = self.get_reconstructed_input(y)
return z
def test_dA(learning_rate=0.1, corruption_level=0.3, training_epochs=50):
data = numpy.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0]])
rng = numpy.random.RandomState(123)
# construct dA
da = dA(input=data, n_visible=20, n_hidden=5, numpy_rng=rng)
# train
for epoch in xrange(training_epochs):
da.train(lr=learning_rate, corruption_level=corruption_level)
# cost = da.negative_log_likelihood(corruption_level=corruption_level)
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
# learning_rate *= 0.95
# test
x = numpy.array([[1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1],
[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0]])
print da.reconstruct(x)
if __name__ == "__main__":
test_dA()