saliksyed · November 18, 2015 03:30 · nikwolfe7 · Dec 22, 2015 · prince16 · Jan 18, 2017
diff --git a/autoencoder.py b/autoencoder.py
 """ Deep Auto-Encoder implementation
 	
 	An auto-encoder works as follows:

 	Data of dimension k is reduced to a lower dimension j using a matrix multiplication:
 	softmax(W*x + b)  = x'
 	
 	where W is matrix from R^k --> R^j

 	A reconstruction matrix W' maps back from R^j --> R^k

 	so our reconstruction function is softmax'(W' * x' + b') 

 	Now the point of the auto-encoder is to create a reduction matrix (values for W, b) 
 	that is "good" at reconstructing  the original data. 

 	Thus we want to minimize  ||softmax'(W' * (softmax(W *x+ b)) + b')  - x||

 	A deep auto-encoder is nothing more than stacking successive layers of these reductions.
 """
 import tensorflow as tf
 import numpy as np
 import math
 import random

 def create(x, layer_sizes):

 	# Build the encoding layers
 	next_layer_input = x

 	encoding_matrices = []
 	for dim in layer_sizes:
 		input_dim = int(next_layer_input.get_shape()[1])

 		# Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]
 		W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))

 		# Initialize b to zero
 		b = tf.Variable(tf.zeros([dim]))

 		# We are going to use tied-weights so store the W matrix for later reference.
 		encoding_matrices.append(W)

 		output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)

 		# the input into the next layer is the output of this layer
 		next_layer_input = output

 	# The fully encoded x value is now stored in the next_layer_input
 	encoded_x = next_layer_input

 	# build the reconstruction layers by reversing the reductions
 	layer_sizes.reverse()
 	encoding_matrices.reverse()


 	for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :
 		# we are using tied weights, so just lookup the encoding matrix for this step and transpose it
 		W = tf.transpose(encoding_matrices[i])
 		b = tf.Variable(tf.zeros([dim]))
 		output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
 		next_layer_input = output

 	# the fully encoded and reconstructed value of x is here:
 	reconstructed_x = next_layer_input

 	return {
 		'encoded': encoded_x,
 		'decoded': reconstructed_x,
 		'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))
 	}

 def simple_test():
 	sess = tf.Session()
 	x = tf.placeholder("float", [None, 4])
 	autoencoder = create(x, [2])
 	init = tf.initialize_all_variables()
 	sess.run(init)
 	train_step = tf.train.GradientDescentOptimizer(0.01).minimize(autoencoder['cost'])


 	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
 	c1 = np.array([0,0,0.5,0])
 	c2 = np.array([0.5,0,0,0])

 	# do 1000 training steps
 	for i in range(2000):
 		# make a batch of 100:
 		batch = []
 		for j in range(100):
 			# pick a random centroid
 			if (random.random() > 0.5):
 				vec = c1
 			else:
 				vec = c2
  			batch.append(np.random.normal(vec, 0.1))
 		sess.run(train_step, feed_dict={x: np.array(batch)})
 		if i % 100 == 0:
 			print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})


 def deep_test():
 	sess = tf.Session()
 	start_dim = 5
 	x = tf.placeholder("float", [None, start_dim])
 	autoencoder = create(x, [4, 3, 2])
 	init = tf.initialize_all_variables()
 	sess.run(init)
 	train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost'])


 	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
 	c1 = np.zeros(start_dim)
 	c1[0] = 1

 	print c1

 	c2 = np.zeros(start_dim)
 	c2[1] = 1

 	# do 1000 training steps
 	for i in range(5000):
 		# make a batch of 100:
 		batch = []
 		for j in range(1):
 			# pick a random centroid
 			if (random.random() > 0.5):
 				vec = c1
 			else:
 				vec = c2
  			batch.append(np.random.normal(vec, 0.1))
 		sess.run(train_step, feed_dict={x: np.array(batch)})
 		if i % 100 == 0:
 			print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
 			print i, " original", batch[0]
 			print i, " decoded", sess.run(autoencoder['decoded'], feed_dict={x: batch})
 			
 if __name__ == '__main__':
 	deep_test()
	""" Deep Auto-Encoder implementation

	An auto-encoder works as follows:

	Data of dimension k is reduced to a lower dimension j using a matrix multiplication:
	softmax(W*x + b) = x'

	where W is matrix from R^k --> R^j

	A reconstruction matrix W' maps back from R^j --> R^k

	so our reconstruction function is softmax'(W' * x' + b')

	Now the point of the auto-encoder is to create a reduction matrix (values for W, b)
	that is "good" at reconstructing the original data.

	Thus we want to minimize \|\|softmax'(W' * (softmax(W *x+ b)) + b') - x\|\|

	A deep auto-encoder is nothing more than stacking successive layers of these reductions.
	"""
	import tensorflow as tf
	import numpy as np
	import math
	import random

	def create(x, layer_sizes):

	# Build the encoding layers
	next_layer_input = x

	encoding_matrices = []
	for dim in layer_sizes:
	input_dim = int(next_layer_input.get_shape()[1])

	# Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]
	W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))

	# Initialize b to zero
	b = tf.Variable(tf.zeros([dim]))

	# We are going to use tied-weights so store the W matrix for later reference.
	encoding_matrices.append(W)

	output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)

	# the input into the next layer is the output of this layer
	next_layer_input = output

	# The fully encoded x value is now stored in the next_layer_input
	encoded_x = next_layer_input

	# build the reconstruction layers by reversing the reductions
	layer_sizes.reverse()
	encoding_matrices.reverse()


	for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :
	# we are using tied weights, so just lookup the encoding matrix for this step and transpose it
	W = tf.transpose(encoding_matrices[i])
	b = tf.Variable(tf.zeros([dim]))
	output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
	next_layer_input = output

	# the fully encoded and reconstructed value of x is here:
	reconstructed_x = next_layer_input

	return {
	'encoded': encoded_x,
	'decoded': reconstructed_x,
	'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))
	}

	def simple_test():
	sess = tf.Session()
	x = tf.placeholder("float", [None, 4])
	autoencoder = create(x, [2])
	init = tf.initialize_all_variables()
	sess.run(init)
	train_step = tf.train.GradientDescentOptimizer(0.01).minimize(autoencoder['cost'])


	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
	c1 = np.array([0,0,0.5,0])
	c2 = np.array([0.5,0,0,0])

	# do 1000 training steps
	for i in range(2000):
	# make a batch of 100:
	batch = []
	for j in range(100):
	# pick a random centroid
	if (random.random() > 0.5):
	vec = c1
	else:
	vec = c2
	batch.append(np.random.normal(vec, 0.1))
	sess.run(train_step, feed_dict={x: np.array(batch)})
	if i % 100 == 0:
	print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})


	def deep_test():
	sess = tf.Session()
	start_dim = 5
	x = tf.placeholder("float", [None, start_dim])
	autoencoder = create(x, [4, 3, 2])
	init = tf.initialize_all_variables()
	sess.run(init)
	train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost'])


	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
	c1 = np.zeros(start_dim)
	c1[0] = 1

	print c1

	c2 = np.zeros(start_dim)
	c2[1] = 1

	# do 1000 training steps
	for i in range(5000):
	# make a batch of 100:
	batch = []
	for j in range(1):
	# pick a random centroid
	if (random.random() > 0.5):
	vec = c1
	else:
	vec = c2
	batch.append(np.random.normal(vec, 0.1))
	sess.run(train_step, feed_dict={x: np.array(batch)})
	if i % 100 == 0:
	print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
	print i, " original", batch[0]
	print i, " decoded", sess.run(autoencoder['decoded'], feed_dict={x: batch})

	if __name__ == '__main__':
	deep_test()
No results found