[pymvpa] Stopping Criteria
Susanne Greiner
susanne.greiner at gmail.com
Tue Feb 17 08:48:52 UTC 2009
Hi all,
I was experimenting a bit with the stopping criteria for RFE and
everything worked fine till I arrived at the NBackHistoryStopCrit.
This code is working for
stopping_criterion=NStepsStopCrit(3)
but produces a strange error for
stopping_criterion=NBackHistoryStopCrit(3)
(relevant code and error messages in the following)
So what am I actually doing wrong? I got the same errors for two
different combinations of versions (thinking at first that maybe it
depended on my new installation).
It was not working for me trying with:
numpy 1.0.4; matplotlib 0.91.2; pymvpa 0.4.0
numpy 1.2.1; matplotlib 0.98.5.2; pymvpa 0.4.1
All kind of comments are welcome,
thx,
Susanne
---------------------------------------------------
Relevant Code:
from mvpa.suite import *
if __debug__:
debug.active += ["CROSSC"]
#
# load PyMVPA example dataset
#
attr = SampleAttributes('data/attributes.txt')
dataset = NiftiDataset(samples='data/bold.nii.gz',
labels=attr.labels,
chunks=attr.chunks,
mask='data/mask.nii.gz')
#
# preprocessing
#
# do chunkswise linear detrending on dataset
detrend(dataset, perchunk=True, model='linear')
dataset = dataset.selectSamples(
N.array([ l in [0,1,2] for l in dataset.labels],
dtype='bool'))
# zscore dataset relative to baseline ('rest') mean
zscore(dataset, perchunk=True, baselinelabels=[0], targetdtype='float32')
# remove baseline samples from dataset for final analysis
dataset = dataset.selectSamples(N.array([l != 0 for l in dataset.labels],
dtype='bool'))
print dataset
# Specify the base classifier to be used
# To parametrize the classifier to be used
# Clf = lambda *args:LinearCSVMC(C=-10, *args)
# Just to assign a particular classifier class
rfesvm_split = SplitClassifier(LinearCSVMC())
results = []
labels = []
clf = \
FeatureSelectionClassifier(
clf = LinearCSVMC(),
# on features selected via RFE
feature_selection = RFE(
# based on sensitivity of a clf which does splitting internally
sensitivity_analyzer=rfesvm_split.getSensitivityAnalyzer(),
transfer_error=ConfusionBasedError(
rfesvm_split,
confusion_state="confusion"),
#### This stopping criterion works fine
# stopping_criterion=NStepsStopCrit(3),
### This stopping criterion produces a strange error
stopping_criterion=NBackHistoryStopCrit(3),
# and whose internal error we use
feature_selector=FractionTailSelector(
0.05, mode='discard', tail='lower'),
# remove ptd% of features at each step
update_sensitivity=True),
# update sensitivity at each step
descr='LinSVM+RFE(splits_avg)' )
desc=3
# run and visualize in barplot
cv = CrossValidatedTransferError(
TransferError(clf),
NFoldSplitter(),
enable_states=['results'])
cv(dataset)
results.append(cv.results)
labels.append(desc)
plotBars(results, labels=labels,
title='Linear C-SVM classification ( faces vs. houses )',
ylabel='Mean classification error (N-1 cross-validation, 12-fold)',
distance=0.25,ylim=(0,0.5))
if cfg.getboolean('examples', 'interactive', True):
P.show()
---------------------------------
Error Messages:
In [8]: ## working on region in file /tmp/python-1339wAZ.py...
Dataset / float32 216 x 530 uniq: 12 chunks 2 labels
.*
optimization finished, #iter = 250
nu = 0.246132
obj = -0.036467, rho = -0.537746
nSV = 75, nBSV = 21
Total nSV = 75
.*
optimization finished, #iter = 255
nu = 0.261928
obj = -0.037437, rho = -0.570851
nSV = 74, nBSV = 19
Total nSV = 74
.*
optimization finished, #iter = 222
nu = 0.255514
obj = -0.037128, rho = -0.512139
nSV = 75, nBSV = 22
Total nSV = 75
.*
optimization finished, #iter = 224
nu = 0.266124
obj = -0.039276, rho = -0.572545
nSV = 77, nBSV = 23
Total nSV = 77
.*
optimization finished, #iter = 230
nu = 0.257976
obj = -0.037712, rho = -0.624339
nSV = 76, nBSV = 21
Total nSV = 76
.*
optimization finished, #iter = 263
nu = 0.246272
obj = -0.036374, rho = -0.563382
nSV = 74, nBSV = 20
Total nSV = 74
.*
optimization finished, #iter = 241
nu = 0.240206
obj = -0.033656, rho = -0.481273
nSV = 69, nBSV = 15
Total nSV = 69
.*
optimization finished, #iter = 227
nu = 0.259685
obj = -0.037864, rho = -0.545578
nSV = 73, nBSV = 21
Total nSV = 73
.*
optimization finished, #iter = 232
nu = 0.250284
obj = -0.036744, rho = -0.595110
nSV = 76, nBSV = 19
Total nSV = 76
*
optimization finished, #iter = 174
nu = 0.265358
obj = -0.038435, rho = -0.613108
nSV = 71, nBSV = 23
Total nSV = 71
.*
optimization finished, #iter = 210
nu = 0.254034
obj = -0.036023, rho = -0.706735
nSV = 71, nBSV = 19
Total nSV = 71
.*
optimization finished, #iter = 250
nu = 0.246132
obj = -0.036467, rho = -0.537746
nSV = 75, nBSV = 21
Total nSV = 75
.*
optimization finished, #iter = 255
nu = 0.261928
obj = -0.037437, rho = -0.570851
nSV = 74, nBSV = 19
Total nSV = 74
.*
optimization finished, #iter = 222
nu = 0.255514
obj = -0.037128, rho = -0.512139
nSV = 75, nBSV = 22
Total nSV = 75
.*
optimization finished, #iter = 224
nu = 0.266124
obj = -0.039276, rho = -0.572545
nSV = 77, nBSV = 23
Total nSV = 77
.*
optimization finished, #iter = 230
nu = 0.257976
obj = -0.037712, rho = -0.624339
nSV = 76, nBSV = 21
Total nSV = 76
.*
optimization finished, #iter = 263
nu = 0.246272
obj = -0.036374, rho = -0.563382
nSV = 74, nBSV = 20
Total nSV = 74
.*
optimization finished, #iter = 241
nu = 0.240206
obj = -0.033656, rho = -0.481273
nSV = 69, nBSV = 15
Total nSV = 69
.*
optimization finished, #iter = 227
nu = 0.259685
obj = -0.037864, rho = -0.545578
nSV = 73, nBSV = 21
Total nSV = 73
.*
optimization finished, #iter = 232
nu = 0.250284
obj = -0.036744, rho = -0.595110
nSV = 76, nBSV = 19
Total nSV = 76
*
optimization finished, #iter = 174
nu = 0.265358
obj = -0.038435, rho = -0.613108
nSV = 71, nBSV = 23
Total nSV = 71
.*
optimization finished, #iter = 210
nu = 0.254034
obj = -0.036023, rho = -0.706735
nSV = 71, nBSV = 19
Total nSV = 71
---------------------------------------------------------------------------
<type 'exceptions.TypeError'> Traceback (most recent call last)
/home/cattt84/PhD/projects/pyMVPA_game/examples/<ipython console> in
<module>()
/tmp/python-1339wAZ.py in <module>()
76 NFoldSplitter(),
77 enable_states=['results'])
---> 78 cv(dataset)
79
80 results.append(cv.results)
/home/cattt84/local/lib/python2.5/site-packages/mvpa/measures/base.py in
__call__(self, dataset)
103 container applying transformer if such is defined
104 """
--> 105 result = self._call(dataset)
106 result = self._postcall(dataset, result)
107 return result
/home/cattt84/local/lib/python2.5/site-packages/mvpa/algorithms/cvtranserror.py
in _call(self, dataset)
171
172 # run the beast
--> 173 result = transerror(split[1], split[0])
174
175 # unbind the testdataset from the classifier
/home/cattt84/local/lib/python2.5/site-packages/mvpa/clfs/transerror.py
in __call__(self, testdataset, trainingdataset)
1275 Returns a scalar value of the transfer error.
1276 """
-> 1277 self._precall(testdataset, trainingdataset)
1278 error = self._call(testdataset, trainingdataset)
1279 self._postcall(testdataset, trainingdataset, error)
/home/cattt84/local/lib/python2.5/site-packages/mvpa/clfs/transerror.py
in _precall(self, testdataset, trainingdataset)
1231 self.__clf.states._changeTemporarily(
1232 enable_states=['training_confusion'])
-> 1233 self.__clf.train(trainingdataset)
1234 if self.states.isEnabled('training_confusion'):
1235 self.training_confusion =
self.__clf.training_confusion
/home/cattt84/local/lib/python2.5/site-packages/mvpa/clfs/base.py in
train(self, dataset)
352
353 if dataset.nfeatures > 0:
--> 354 result = self._train(dataset)
355 else:
356 warning("Trying to train on dataset with no features
present")
/home/cattt84/local/lib/python2.5/site-packages/mvpa/clfs/meta.py in
_train(self, dataset)
1055
1056 (wdataset, tdataset) = self.__feature_selection(dataset,
-> 1057
self.__testdataset)
1058 if __debug__:
1059 add_ = ""
/home/cattt84/local/lib/python2.5/site-packages/mvpa/featsel/rfe.py in
__call__(self, dataset, testdataset)
234 # Check if it is time to stop and if we got
235 # the best result
--> 236 stop = self.__stopping_criterion(errors)
237 isthebest = self.__bestdetector(errors)
238
/home/cattt84/local/lib/python2.5/site-packages/mvpa/featsel/helpers.py
in __call__(self, errors)
219
220 # charge best detector
--> 221 self.__bestdetector(errors)
222
223 # if number of elements after the min >= len -- stop
<type 'exceptions.TypeError'>: 'int' object is not callable
More information about the Pkg-ExpPsy-PyMVPA
mailing list