Commit 3ac05696 by Sebastian Höfer

implemented test suite for PatternTrainer -- focusing on testing of…

implemented test suite for PatternTrainer -- focusing on testing of side_validation. restructured and renamed side validation functions. added special case handling for squared_error loss with vector as target var (shape problem)
parent a7876ebe
# Compiled python modules.
*.pyc
*.tar
*.npz
# thumbfiles
**/.DS_Store
# Setuptools distribution folder.
build/
/dist/
# Python egg metadata, regenerated from source files by setuptools.
/*.egg-info
/*.egg
*/.ipynb_checkpoints
......@@ -7,6 +7,8 @@ from lasagne.layers import get_all_layers
from lasagne.layers import InputLayer
from lasagne.layers import Layer
import theano.tensor as T
import numpy as np
import itertools
......@@ -16,6 +18,7 @@ import inspect
import copy
import os
import tarfile
from warnings import warn
class Pattern(object):
"""
......@@ -25,7 +28,7 @@ class Pattern(object):
It is similar to :class:`lasagne.layers.Layer` and mimics some of
its functionality, but does not inherit from it.
=== How to implement your own pattern? ===
### How to implement your own pattern?
The minimal example should implement the following functions:
- get_side_objective
......@@ -42,9 +45,9 @@ class Pattern(object):
If you beta has multiple inputs, you will need to implement:
- get_beta_output_for
Optionally, you might want to implement in order to use :method:`Pattern.score_side`:
- validation_side_input_vars(self):
- validation_side_target_var(self)
Optionally, if your side variable is a supervised learning target, then you
should return the theano variable representing this target in the method
- side_target_var(self)
Parameters
----------
......@@ -70,12 +73,11 @@ class Pattern(object):
Shape of the intermediate representation to be learned
(for some patterns that may coincide with the side_shape)
target_loss: theano tensor variable, optional
Theano expression or lasagne objective for the optimizing the
target.
Function (e.g. lasagne objective) for the optimizing the target.
All patterns have standard objectives applicable here
side_loss: theano tensor variable, optional
Theano expression or lasagne objective for the side loss.
All patterns have standard objectives applicable here
Most patterns have standard objectives applicable here.
name : string, optional
An optional name to attach to this layer.
"""
......@@ -116,9 +118,20 @@ class Pattern(object):
if isfunction(self.target_loss):
self.target_loss_fn = self.target_loss
self.target_loss = None
else:
warn("target_loss: passing something different than a python function object "
"to the constructor of a Pattern is deprecated. "
"Recommended way is to use a function from lasagne.objectives "
"or equivalent." )
if isfunction(self.side_loss):
self.side_loss_fn = self.side_loss
self.side_loss = None
else:
warn("side_loss: passing something different than a python function object "
"to the constructor of a Pattern is deprecated. "
"Recommended way is to use a function from lasagne.objectives "
"or equivalent." )
# convert phi, psi and beta to real lasagne layers if they
# are passed as a list/dictionary
......@@ -195,40 +208,37 @@ class Pattern(object):
return (self.input_var, self.target_var, self.side_var)
@property
def validation_side_input_vars(self):
def side_input_vars(self):
"""Return the theano input variables for validating the side loss.
Order matters!
This strongly depends on the pattern and is therefore implemented
individually.
It is not obligatory, but you will not be able to compute the side validation
loss.
Per default we assume that it is all training variables except for the
target variable (see :method:`Pattern.training_input_vars`) and the
optional side target variable (see :method:`Pattern.side_target_var`).
Also see :method:`Pattern.validation_target_var`
You can override this method in your pattern.
Order matters!
Returns
-------
tuple of theano tensor variables
"""
raise NotImplementedError()
excluded_vars = [self.target_var]
if self.side_target_var is not None:
excluded_vars.append(self.side_target_var)
return tuple([i for i in self.training_input_vars if i not in excluded_vars])
@property
def validation_target_var(self):
def side_target_var(self):
"""Return the theano target variable required for validating
the side information (optional).
This returns None per default. Override it the side loss of the pattern is a not a
supervised loss.
This returns None per default.
Override it the side loss of the pattern is a supervised loss, and one of the
side variables is the supervised (side) target - and then return this variable.
This strongly depends on the pattern and is therefore implemented
individually.
It is not obligatory, but you will not be able to compute the side validation
loss during training.
Also see :method:`Pattern.validation_input_vars`
Also see :method:`Pattern.side_input_vars`
Returns
-------
......@@ -600,6 +610,12 @@ class Pattern(object):
else:
#print ("Target loss is function object: %s" % str(self.target_loss_fn))
fn = self.target_loss_fn
# special case: if we use the squared_error loss, but target_var is a vector
# (1 dim target) we flatten the prediction -- otherwise we get a theano error
if fn == lasagne.objectives.squared_error and \
target.type == T.dvector or target.type == T.dvector:
output = output.flatten()
# define target loss
self.target_loss = fn(output, target).mean()
......
......@@ -11,6 +11,7 @@ import lasagne
import concarne.lasagne
from concarne.lasagne import score_categorical_crossentropy,\
score_multivariate_categorical_crossentropy
from concarne.utils import isfunction
import theano
import theano.tensor as T
......@@ -153,8 +154,6 @@ class PatternTrainer(object):
XYpsi_num_epochs=None,
target_weight=None,
side_weight=None,
# test_objective=lasagne.objectives.categorical_crossentropy,
# side_test_objective=None,
verbose=None,
save_params=False,
**kwargs):
......@@ -204,8 +203,21 @@ class PatternTrainer(object):
if side_weight is not None:
self.loss_weights['side_weight'] = side_weight
self.test_objective = pattern.target_loss_fn
self.side_test_objective = pattern.side_loss_fn
self.test_objective_fn = pattern.target_loss_fn
if self.test_objective_fn is None:
warn("You initialized your pattern with a custom target_loss, "
+"i.e. not a python function object, but probably a theano "
+"expression. PatternTrainer will not be able to compute "
+"the validation/test accuracy")
self.test_objective_fn = pattern.target_loss
self.side_test_objective_fn = pattern.side_loss_fn
if self.side_test_objective_fn is None:
warn("You initialized your pattern with a custom side_loss, "
+"i.e. not a python function object, but probably a theano "
+"expression. PatternTrainer will not be able to compute "
+"the validation/test accuracy")
self.side_test_objective_fn = pattern.side_loss
self.val_fn = None
self.val_batch_iterator = None
......@@ -280,14 +292,6 @@ class PatternTrainer(object):
except:
pass
# TODO can we give better feedback for classification case?
# target_var = self.pattern.target_var
# if self.test_objective == lasagne.objectives.categorical_crossentropy:
# outputs['acc'] = score_categorical_crossentropy(test_prediction, target_var)
# else:
# test_acc = None
# -----
# get trainable params and build update
params = lasagne.layers.get_all_params(self.pattern, trainable=True, **tags)
......@@ -311,27 +315,27 @@ class PatternTrainer(object):
train_fn = theano.function(train_fn_inputs, outputs, updates=updates)
# FIXME
self._train_fn_no_update = theano.function(train_fn_inputs, outputs)
#self._train_fn_no_update = theano.function(train_fn_inputs, outputs)
return train_fn
def _compile_val_fn(self):
return self.__compile_val_fn([self.pattern.input_var], self.pattern.target_var,
lasagne.layers.get_output(self.pattern, deterministic=True),
self.test_objective)
self.test_objective_fn)
def _compile_side_val_fn(self):
input_vars = self.pattern.validation_side_input_vars
target_var = self.pattern.validation_side_target_var
side_input_vars = self.pattern.side_input_vars
side_target_var = self.pattern.side_target_var
if target_var is not None:
return self.__compile_val_fn(input_vars, target_var,
if side_target_var is not None:
return self.__compile_val_fn(side_input_vars, side_target_var,
self.pattern.get_beta_output_for(deterministic=True),
self.side_test_objective)
self.side_test_objective_fn)
else:
_, _, side_loss = self.pattern.training_loss(all_losses=True)
return theano.function(input_vars, [side_loss.mean()])
return theano.function(side_input_vars, [side_loss.mean()])
def __compile_val_fn(self, input_vars, target_var,
......@@ -343,14 +347,23 @@ class PatternTrainer(object):
if type(input_vars) == tuple:
input_vars = list(input_vars)
test_loss = test_objective(test_prediction, target_var).mean()
# Create an expression for the classification accuracy
if test_objective == lasagne.objectives.categorical_crossentropy:
test_acc = score_categorical_crossentropy(test_prediction, target_var)
elif test_objective == concarne.lasagne.multivariate_categorical_crossentropy:
test_acc = score_multivariate_categorical_crossentropy(test_prediction, target_var)
else:
test_acc = None
# special case: if we use the squared_error loss, but target_var is a vector
# (1 dim target) we flatten the prediction -- otherwise we get a theano error
if test_objective == lasagne.objectives.squared_error and \
target_var.type == T.dvector or target_var.type == T.dvector:
test_prediction = test_prediction.flatten()
test_acc = None
if isfunction(test_objective):
test_loss = test_objective(test_prediction, target_var).mean()
# Create an expression for the classification accuracy
if test_objective == lasagne.objectives.categorical_crossentropy:
test_acc = score_categorical_crossentropy(test_prediction, target_var)
elif test_objective == concarne.lasagne.multivariate_categorical_crossentropy:
test_acc = score_multivariate_categorical_crossentropy(test_prediction, target_var)
else:
test_loss = test_objective
outputs = [test_loss]
if test_acc is not None:
......@@ -460,7 +473,8 @@ class PatternTrainer(object):
Verbosity level (default 0/False)
"""
if not isiterable(Zs):
#if not isiterable(Zs):
if type(Zs) != list and type(Zs) != tuple:
raise Exception("Make sure that you provide a list of side "
+ " information Zs, even if the pattern only expects one side variable")
......@@ -606,7 +620,8 @@ class PatternTrainer(object):
print ("Optimize phi & psi & beta using a weighted sum of target and side objective")
print (" "+self.__update_info(self.update, update_params))
if data_alignment == "XYZ":
print (" -> standard mode with single training function")
if verbose:
print (" -> standard mode with single training function")
train_fn = self._compile_train_fn(self.pattern.training_input_vars,
loss_weights=self.loss_weights,
tags= {},
......@@ -614,7 +629,8 @@ class PatternTrainer(object):
**update_params)
train_fn = [train_fn]*2
else:
print (" -> alternating mode with two training functions")
if verbose:
print (" -> alternating mode with two training functions")
lw1 = copy.copy(self.loss_weights)
lw1['target_weight'] = 0.
train_fn1 = self._compile_train_fn(train_vars_phase1,
......@@ -757,11 +773,24 @@ class PatternTrainer(object):
for batch in self.val_batch_iterator(X, y):
inputs, targets = batch
res = self.val_fn(inputs, targets)
if len(res) == 2:
err, acc = res
else:
err = res
assert (len(res) == 1)
if isiterable(res[0]):
assert (len(res) == 1)
err = res[0]
else:
err = res
acc = np.nan
# if len(res) == 2:
# err, acc = res
# else:
# err = res
# acc = np.nan
test_err += err
test_acc += acc
test_batches += 1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment