Projects/laydi
Archived
7
0
Fork 0
Code Issues Pull Requests Activity

Tralala ...

Browse Source
This commit is contained in:
Arnar Flatberg
2007-11-07 12:34:13 +00:00
parent ca51a0b382
commit aef2e1daf0
11 changed files with 316 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fluents/lib/blmfuncs.py
View File

@@ -350,15 +350,7 @@ class PLS(Model):
self.validation()
self.make_model(self.model['E0'], self.model['F0'],
**options.make_model_options())
# variance captured
var_x, exp_var_x = variances(self.model['E0'], self.model['T'], self.model['P'])
self.model['evx'] = var_x
self.model['exp_var_x'] = exp_var_x
var_y, exp_var_y = variances(self.model['F0'], self.model['T'], self.model['Q'])
self.model['evy'] = var_y
self.model['exp_var_y'] = exp_var_y
if options['calc_conf']:
self.confidence(**options.confidence_options())
@@ -435,7 +427,7 @@ class LPLS(Model):
engine = opt['engine']
dat = engine(self._data['X'], self._data['Y'], self._data['Z'],
opt['amax'], opt['xz_alpha'], opt['center_mth'],
opt['mode'], opt['scale'], False)
opt['scale'], False)
self.model.update(dat)
def as_dataset(self, name, dtype='Dataset'):
@@ -464,17 +456,21 @@ class LPLS(Model):
match_ids = {'E' : [ids_0, ids_1],
'P' : [ids_1, pc_ids],
'T' : [ids_0, pc_ids],
'U' : [ids_0, pc_ids],
'W' : [ids_1, pc_ids],
'L' : [ids_4, pc_ids],
'Q' : [ids_3, pc_ids],
'F' : [ids_0, ids_3],
'B' : [ids_1, ids_3],
'K' : [ids_1, pc_ids],
'tsqx' : [ids_1, zero_dim],
'tsqz' : [ids_4, zero_dim],
'K' : [ids_1, pc_ids],
'rmsep' : [ids_3, pc_ids],
'Rx' : [ids_1, pc_ids],
'Rz' : [ids_4, pc_ids]
'Rz' : [ids_4, pc_ids],
'evx' : [ids_1, zero_dim],
'evy' : [ids_3, zero_dim],
'evz' : [ids_4, zero_dim]
}
try:

38
fluents/lib/cx_stats.py
View File

@@ -1,14 +1,20 @@
import time
import cPickle
from scipy import zeros,zeros_like,sqrt,dot,trace,sign,round_,argmax,\
sort,ravel,newaxis,asarray,diag,sum,outer,argsort,arange,ones_like,\
all,apply_along_axis,eye,atleast_2d
all,apply_along_axis,eye,atleast_2d,empty
from scipy.linalg import svd,inv,norm,det,sqrtm
from scipy.stats import mean,median
#import plots_lpls
from cx_utils import mat_center
from validation import pls_jkW, lpls_jk
from select_generators import shuffle_1d
from engines import pca, pls, bridge
from engines import nipals_lpls as lpls
import time
def hotelling(Pcv, P, p_center='med', cov_center='med',
@@ -292,7 +298,7 @@ def variances(a, t, p):
tot_var = sum(a**2)
var = 100*(sum(p**2, 0)*sum(t**2, 0))/tot_var
var_exp = cumsum(var)
var_exp = var.cumsum()
return var, var_exp
def residual_diagnostics(Y, Yhat, aopt=1):
@@ -365,7 +371,7 @@ def mahalanobis(a, loc=None, acov=None, invcov=None):
def lpls_qvals(a, b, c, aopt=None, alpha=.3, zx_alpha=.5, n_iter=20,
sim_method='shuffle',p_center='med', cov_center='med',crot=True,
strict=False, mean_ctr=[2,0,2]):
strict=False, mean_ctr=[2,0,2], nsets=None):
"""Returns qvals for l-pls model.
@@ -392,26 +398,25 @@ def lpls_qvals(a, b, c, aopt=None, alpha=.3, zx_alpha=.5, n_iter=20,
# Full model
#print "Full model start"
dat = lpls(a, b, c, aopt, scale='loads', mean_ctr=mean_ctr)
Wc,Lc = lpls_jk(a, b, c ,aopt)
Wc, Lc = lpls_jk(a, b, c , aopt, nsets=nsets)
#print "Full hot"
cal_tsq_x = hotelling(Wc, dat['W'], alpha=alpha)
cal_tsq_z = hotelling(Lc, dat['L'], alpha=alpha)
cal_tsq_x = hotelling(Wc, dat['W'], alpha = alpha)
cal_tsq_z = hotelling(Lc, dat['L'], alpha = 0)
# Perturbations
Vs = shuffle_1d(b, n_iter, axis=0)
for i, b_shuff in enumerate(Vs):
#print i
time.sleep(.01)
print i
dat = lpls(a, b_shuff,c, aopt, scale='loads', mean_ctr=mean_ctr)
Wi, Li = lpls_jk(a, b_shuff, c, aopt)
Wi, Li = lpls_jk(a, b_shuff, c, aopt, nsets=nsets)
pert_tsq_x[:,i] = hotelling(Wi, dat['W'], alpha=alpha)
pert_tsq_z[:,i] = hotelling(Li, dat['L'], alpha=alpha)
return fdr(cal_tsq_z, pert_tsq_z, median), fdr(cal_tsq_x, pert_tsq_x, median)
return cal_tsq_z, pert_tsq_z, cal_tsq_x, pert_tsq_x
def fdr(tsq, tsqp, loc_method=median):
def fdr(tsq, tsqp, loc_method='mean'):
n, = tsq.shape
k, m = tsqp.shape
assert(n==k)
@@ -421,8 +426,13 @@ def fdr(tsq, tsqp, loc_method=median):
for i in xrange(m):
for j in xrange(n):
n_false[j,i] = (tsqp[:,i]>tsq[j]).sum()
fp = loc_method(n_false,1)
#cPickle.dump(n_false, open("/tmp/nfalse.dat_"+str(n), "w"))
if loc_method=='mean':
fp = mean(n_false,1)
elif loc_method == 'median':
fp = median(n_false.T)
else:
raise ValueError
n_signif = (arange(n) + 1.0)[r_index]
fd_rate = fp/n_signif
return fd_rate

6
fluents/lib/engines.py
View File

@@ -367,7 +367,6 @@ def w_simpls(aat, b, aopt):
T = empty((m, aopt))
H = empty((m, aopt)) # R
PROJ = empty((m, aopt)) # P?
for i in range(aopt):
q, s, vh = svd(dot(dot(b.T, aat), b), full_matrices=0)
u = dot(b, q[:,:1]) #y-factor scores
@@ -722,6 +721,11 @@ def esvd(data, amax=None):
:notes:
Numpy supports this by setting full_matrices=0
"""
has_arpack = True
try:
import arpack
except:
has_arpack = False
m, n = data.shape
if m>=n:
kernel = dot(data.T, data)

4
fluents/lib/nx_utils.py
View File

@@ -517,7 +517,7 @@ def K_diffusion2(W, normalised=True, alpha=1.0, beta=0.5, ncomp=None):
return expm(-beta*L)
def K_modularity(W,alpha=1.0):
def K_modularity(W, alpha=1.0):
""" Returns the matrix square root of Newmans modularity."""
W = asarray(W)
t = W.dtype.char
@@ -559,7 +559,7 @@ def modularity_matrix(G, nodelist=None):
A = NX.adj_matrix(G, nodelist=nodelist)
d = atleast_2d(G.degree(nbunch=nodelist))
m = 1.*G.number_of_edges()
B = A - A/m
B = A - dot(d.T, d)/m
return B

4
fluents/lib/select_generators.py
View File

@@ -42,8 +42,8 @@ def pls_gen(a, b, n_blocks=None, center=False, index_out=False,axis=0):
"""Random block crossvalidation
Leave-one-out is a subset, with n_blocks equals a.shape[-1]
"""
#index = randperm(a.shape[axis])
index = arange(a.shape[axis])
index = randperm(a.shape[axis])
#index = arange(a.shape[axis])
if n_blocks==None:
n_blocks = a.shape[axis]
n_in_set = ceil(float(a.shape[axis])/n_blocks)

6
fluents/lib/validation.py
View File

@@ -9,6 +9,7 @@ from select_generators import w_pls_gen,w_pls_gen_jk,pls_gen,pca_gen,diag_pert
from engines import w_simpls,pls,bridge,pca,nipals_lpls
from cx_utils import m_shape
def w_pls_cv_val(X, Y, amax, n_blocks=None):
"""Returns rmsep and aopt for pls tailored for wide X.
@@ -117,6 +118,7 @@ def lpls_val(X, Y, Z, a_max=2, nsets=None,alpha=.5, mean_ctr=[2,0,2]):
Yhatc = empty((a_max, k, l), 'd')
sep2 = empty((a_max, k, l), 'd')
for i, (xcal,xi,ycal,yi,ind) in enumerate(cv_iter):
print ind
dat = nipals_lpls(xcal,ycal,Z,
a_max=a_max,
alpha=alpha,
@@ -140,7 +142,7 @@ def lpls_val(X, Y, Z, a_max=2, nsets=None,alpha=.5, mean_ctr=[2,0,2]):
# todo: need a better support for class validation
y_is_class = Y.dtype.char.lower() in ['i','p', 'b', 'h','?']
print Y.dtype.char
#print Y.dtype.char
if y_is_class:
Yhat_class = zeros_like(Yhat)
for a in range(a_max):
@@ -280,7 +282,7 @@ def lpls_jk(X, Y, Z, a_max, nsets=None, xz_alpha=.5, mean_ctr=[2,0,2]):
WWx = empty((nsets, n, a_max), 'd')
WWz = empty((nsets, o, a_max), 'd')
#WWy = empty((nsets, l, a_max), 'd')
for i, (xcal,xi,ycal,yi) in enumerate(cv_iter):
for i, (xcal, xi, ycal, yi) in enumerate(cv_iter):
dat = nipals_lpls(xcal,ycal,Z,a_max=a_max,alpha=xz_alpha,
mean_ctr=mean_ctr,scale='loads',verbose=False)
WWx[i,:,:] = dat['W']

1
fluents/plots.py
View File

@@ -783,6 +783,7 @@ class NetworkPlot(Plot):
index = scipy.nonzero((xdata>x1) & (xdata<x2) & (ydata>y1) & (ydata<y2))[0]
ids = self.dataset.get_identifiers(self.current_dim, index)
print "ids in rsc: %s" %str(ids)
ids = self.update_selection(ids, key)
self.selection_listener(self.current_dim, ids)