Tralala ...
This commit is contained in:
parent
ca51a0b382
commit
aef2e1daf0
|
|
@ -350,15 +350,7 @@ class PLS(Model):
|
|||
self.validation()
|
||||
self.make_model(self.model['E0'], self.model['F0'],
|
||||
**options.make_model_options())
|
||||
# variance captured
|
||||
var_x, exp_var_x = variances(self.model['E0'], self.model['T'], self.model['P'])
|
||||
self.model['evx'] = var_x
|
||||
self.model['exp_var_x'] = exp_var_x
|
||||
|
||||
var_y, exp_var_y = variances(self.model['F0'], self.model['T'], self.model['Q'])
|
||||
self.model['evy'] = var_y
|
||||
self.model['exp_var_y'] = exp_var_y
|
||||
|
||||
|
||||
if options['calc_conf']:
|
||||
self.confidence(**options.confidence_options())
|
||||
|
||||
|
|
@ -435,7 +427,7 @@ class LPLS(Model):
|
|||
engine = opt['engine']
|
||||
dat = engine(self._data['X'], self._data['Y'], self._data['Z'],
|
||||
opt['amax'], opt['xz_alpha'], opt['center_mth'],
|
||||
opt['mode'], opt['scale'], False)
|
||||
opt['scale'], False)
|
||||
self.model.update(dat)
|
||||
|
||||
def as_dataset(self, name, dtype='Dataset'):
|
||||
|
|
@ -464,17 +456,21 @@ class LPLS(Model):
|
|||
match_ids = {'E' : [ids_0, ids_1],
|
||||
'P' : [ids_1, pc_ids],
|
||||
'T' : [ids_0, pc_ids],
|
||||
'U' : [ids_0, pc_ids],
|
||||
'W' : [ids_1, pc_ids],
|
||||
'L' : [ids_4, pc_ids],
|
||||
'Q' : [ids_3, pc_ids],
|
||||
'F' : [ids_0, ids_3],
|
||||
'B' : [ids_1, ids_3],
|
||||
'K' : [ids_1, pc_ids],
|
||||
'tsqx' : [ids_1, zero_dim],
|
||||
'tsqz' : [ids_4, zero_dim],
|
||||
'K' : [ids_1, pc_ids],
|
||||
'rmsep' : [ids_3, pc_ids],
|
||||
'Rx' : [ids_1, pc_ids],
|
||||
'Rz' : [ids_4, pc_ids]
|
||||
'Rz' : [ids_4, pc_ids],
|
||||
'evx' : [ids_1, zero_dim],
|
||||
'evy' : [ids_3, zero_dim],
|
||||
'evz' : [ids_4, zero_dim]
|
||||
}
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,14 +1,20 @@
|
|||
import time
|
||||
import cPickle
|
||||
|
||||
from scipy import zeros,zeros_like,sqrt,dot,trace,sign,round_,argmax,\
|
||||
sort,ravel,newaxis,asarray,diag,sum,outer,argsort,arange,ones_like,\
|
||||
all,apply_along_axis,eye,atleast_2d
|
||||
all,apply_along_axis,eye,atleast_2d,empty
|
||||
from scipy.linalg import svd,inv,norm,det,sqrtm
|
||||
from scipy.stats import mean,median
|
||||
|
||||
#import plots_lpls
|
||||
|
||||
from cx_utils import mat_center
|
||||
from validation import pls_jkW, lpls_jk
|
||||
from select_generators import shuffle_1d
|
||||
from engines import pca, pls, bridge
|
||||
from engines import nipals_lpls as lpls
|
||||
import time
|
||||
|
||||
|
||||
|
||||
def hotelling(Pcv, P, p_center='med', cov_center='med',
|
||||
|
|
@ -292,7 +298,7 @@ def variances(a, t, p):
|
|||
|
||||
tot_var = sum(a**2)
|
||||
var = 100*(sum(p**2, 0)*sum(t**2, 0))/tot_var
|
||||
var_exp = cumsum(var)
|
||||
var_exp = var.cumsum()
|
||||
return var, var_exp
|
||||
|
||||
def residual_diagnostics(Y, Yhat, aopt=1):
|
||||
|
|
@ -365,7 +371,7 @@ def mahalanobis(a, loc=None, acov=None, invcov=None):
|
|||
|
||||
def lpls_qvals(a, b, c, aopt=None, alpha=.3, zx_alpha=.5, n_iter=20,
|
||||
sim_method='shuffle',p_center='med', cov_center='med',crot=True,
|
||||
strict=False, mean_ctr=[2,0,2]):
|
||||
strict=False, mean_ctr=[2,0,2], nsets=None):
|
||||
|
||||
"""Returns qvals for l-pls model.
|
||||
|
||||
|
|
@ -392,26 +398,25 @@ def lpls_qvals(a, b, c, aopt=None, alpha=.3, zx_alpha=.5, n_iter=20,
|
|||
# Full model
|
||||
#print "Full model start"
|
||||
dat = lpls(a, b, c, aopt, scale='loads', mean_ctr=mean_ctr)
|
||||
Wc,Lc = lpls_jk(a, b, c ,aopt)
|
||||
Wc, Lc = lpls_jk(a, b, c , aopt, nsets=nsets)
|
||||
#print "Full hot"
|
||||
cal_tsq_x = hotelling(Wc, dat['W'], alpha=alpha)
|
||||
cal_tsq_z = hotelling(Lc, dat['L'], alpha=alpha)
|
||||
cal_tsq_x = hotelling(Wc, dat['W'], alpha = alpha)
|
||||
cal_tsq_z = hotelling(Lc, dat['L'], alpha = 0)
|
||||
|
||||
# Perturbations
|
||||
Vs = shuffle_1d(b, n_iter, axis=0)
|
||||
for i, b_shuff in enumerate(Vs):
|
||||
#print i
|
||||
time.sleep(.01)
|
||||
print i
|
||||
dat = lpls(a, b_shuff,c, aopt, scale='loads', mean_ctr=mean_ctr)
|
||||
Wi, Li = lpls_jk(a, b_shuff, c, aopt)
|
||||
Wi, Li = lpls_jk(a, b_shuff, c, aopt, nsets=nsets)
|
||||
pert_tsq_x[:,i] = hotelling(Wi, dat['W'], alpha=alpha)
|
||||
pert_tsq_z[:,i] = hotelling(Li, dat['L'], alpha=alpha)
|
||||
|
||||
return fdr(cal_tsq_z, pert_tsq_z, median), fdr(cal_tsq_x, pert_tsq_x, median)
|
||||
return cal_tsq_z, pert_tsq_z, cal_tsq_x, pert_tsq_x
|
||||
|
||||
|
||||
|
||||
def fdr(tsq, tsqp, loc_method=median):
|
||||
def fdr(tsq, tsqp, loc_method='mean'):
|
||||
n, = tsq.shape
|
||||
k, m = tsqp.shape
|
||||
assert(n==k)
|
||||
|
|
@ -421,8 +426,13 @@ def fdr(tsq, tsqp, loc_method=median):
|
|||
for i in xrange(m):
|
||||
for j in xrange(n):
|
||||
n_false[j,i] = (tsqp[:,i]>tsq[j]).sum()
|
||||
|
||||
fp = loc_method(n_false,1)
|
||||
#cPickle.dump(n_false, open("/tmp/nfalse.dat_"+str(n), "w"))
|
||||
if loc_method=='mean':
|
||||
fp = mean(n_false,1)
|
||||
elif loc_method == 'median':
|
||||
fp = median(n_false.T)
|
||||
else:
|
||||
raise ValueError
|
||||
n_signif = (arange(n) + 1.0)[r_index]
|
||||
fd_rate = fp/n_signif
|
||||
return fd_rate
|
||||
|
|
|
|||
|
|
@ -367,7 +367,6 @@ def w_simpls(aat, b, aopt):
|
|||
T = empty((m, aopt))
|
||||
H = empty((m, aopt)) # R
|
||||
PROJ = empty((m, aopt)) # P?
|
||||
|
||||
for i in range(aopt):
|
||||
q, s, vh = svd(dot(dot(b.T, aat), b), full_matrices=0)
|
||||
u = dot(b, q[:,:1]) #y-factor scores
|
||||
|
|
@ -722,6 +721,11 @@ def esvd(data, amax=None):
|
|||
:notes:
|
||||
Numpy supports this by setting full_matrices=0
|
||||
"""
|
||||
has_arpack = True
|
||||
try:
|
||||
import arpack
|
||||
except:
|
||||
has_arpack = False
|
||||
m, n = data.shape
|
||||
if m>=n:
|
||||
kernel = dot(data.T, data)
|
||||
|
|
|
|||
|
|
@ -517,7 +517,7 @@ def K_diffusion2(W, normalised=True, alpha=1.0, beta=0.5, ncomp=None):
|
|||
return expm(-beta*L)
|
||||
|
||||
|
||||
def K_modularity(W,alpha=1.0):
|
||||
def K_modularity(W, alpha=1.0):
|
||||
""" Returns the matrix square root of Newmans modularity."""
|
||||
W = asarray(W)
|
||||
t = W.dtype.char
|
||||
|
|
@ -559,7 +559,7 @@ def modularity_matrix(G, nodelist=None):
|
|||
A = NX.adj_matrix(G, nodelist=nodelist)
|
||||
d = atleast_2d(G.degree(nbunch=nodelist))
|
||||
m = 1.*G.number_of_edges()
|
||||
B = A - A/m
|
||||
B = A - dot(d.T, d)/m
|
||||
return B
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ def pls_gen(a, b, n_blocks=None, center=False, index_out=False,axis=0):
|
|||
"""Random block crossvalidation
|
||||
Leave-one-out is a subset, with n_blocks equals a.shape[-1]
|
||||
"""
|
||||
#index = randperm(a.shape[axis])
|
||||
index = arange(a.shape[axis])
|
||||
index = randperm(a.shape[axis])
|
||||
#index = arange(a.shape[axis])
|
||||
if n_blocks==None:
|
||||
n_blocks = a.shape[axis]
|
||||
n_in_set = ceil(float(a.shape[axis])/n_blocks)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from select_generators import w_pls_gen,w_pls_gen_jk,pls_gen,pca_gen,diag_pert
|
|||
from engines import w_simpls,pls,bridge,pca,nipals_lpls
|
||||
from cx_utils import m_shape
|
||||
|
||||
|
||||
def w_pls_cv_val(X, Y, amax, n_blocks=None):
|
||||
"""Returns rmsep and aopt for pls tailored for wide X.
|
||||
|
||||
|
|
@ -117,6 +118,7 @@ def lpls_val(X, Y, Z, a_max=2, nsets=None,alpha=.5, mean_ctr=[2,0,2]):
|
|||
Yhatc = empty((a_max, k, l), 'd')
|
||||
sep2 = empty((a_max, k, l), 'd')
|
||||
for i, (xcal,xi,ycal,yi,ind) in enumerate(cv_iter):
|
||||
print ind
|
||||
dat = nipals_lpls(xcal,ycal,Z,
|
||||
a_max=a_max,
|
||||
alpha=alpha,
|
||||
|
|
@ -140,7 +142,7 @@ def lpls_val(X, Y, Z, a_max=2, nsets=None,alpha=.5, mean_ctr=[2,0,2]):
|
|||
|
||||
# todo: need a better support for class validation
|
||||
y_is_class = Y.dtype.char.lower() in ['i','p', 'b', 'h','?']
|
||||
print Y.dtype.char
|
||||
#print Y.dtype.char
|
||||
if y_is_class:
|
||||
Yhat_class = zeros_like(Yhat)
|
||||
for a in range(a_max):
|
||||
|
|
@ -280,7 +282,7 @@ def lpls_jk(X, Y, Z, a_max, nsets=None, xz_alpha=.5, mean_ctr=[2,0,2]):
|
|||
WWx = empty((nsets, n, a_max), 'd')
|
||||
WWz = empty((nsets, o, a_max), 'd')
|
||||
#WWy = empty((nsets, l, a_max), 'd')
|
||||
for i, (xcal,xi,ycal,yi) in enumerate(cv_iter):
|
||||
for i, (xcal, xi, ycal, yi) in enumerate(cv_iter):
|
||||
dat = nipals_lpls(xcal,ycal,Z,a_max=a_max,alpha=xz_alpha,
|
||||
mean_ctr=mean_ctr,scale='loads',verbose=False)
|
||||
WWx[i,:,:] = dat['W']
|
||||
|
|
|
|||
|
|
@ -783,6 +783,7 @@ class NetworkPlot(Plot):
|
|||
|
||||
index = scipy.nonzero((xdata>x1) & (xdata<x2) & (ydata>y1) & (ydata<y2))[0]
|
||||
ids = self.dataset.get_identifiers(self.current_dim, index)
|
||||
print "ids in rsc: %s" %str(ids)
|
||||
ids = self.update_selection(ids, key)
|
||||
self.selection_listener(self.current_dim, ids)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ def dag(terms, ontology):
|
|||
__parents = {'bp' : rpy.r.GOBPPARENTS,
|
||||
'mf' : rpy.r.GOMFPARENTS,
|
||||
'cc' : rpy.r.GOCCPARENTS}
|
||||
gograph = rpy.r.GOGraph(terms, __parents.get(ontology))
|
||||
gograph = rpy.r.GOGraph(terms, __parents.get(ontology.lower()))
|
||||
dag = rpy.r.edges(gograph)
|
||||
#setattr(dag, "_ontology", ontology)
|
||||
return dag
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import rpy
|
|||
silent_eval = rpy.with_mode(rpy.NO_CONVERSION, rpy.r)
|
||||
import collections
|
||||
|
||||
def goterms_from_gene(genelist, ontology='BP', garbage=None):
|
||||
def goterms_from_gene(genelist, ontology='BP', garbage=None, ic_cutoff=2.0):
|
||||
""" Returns the go-terms from a specified genelist (Entrez id).
|
||||
|
||||
Recalculates the information content if needed based on selected evidence codes.
|
||||
|
|
@ -37,38 +37,30 @@ def goterms_from_gene(genelist, ontology='BP', garbage=None):
|
|||
ic = rpy.r('get("IC", envir=GOSimEnv)')
|
||||
print "loading GO definitions environment"
|
||||
|
||||
gene2terms = {}
|
||||
gene2terms = collections.defaultdict(list)
|
||||
cc = 0
|
||||
dd = 0
|
||||
ii = 0
|
||||
for gene in genelist:
|
||||
info = rpy.r('GOENTREZID2GO[["' + str(gene) + '"]]')
|
||||
#print info
|
||||
if info:
|
||||
skip=False
|
||||
for term, desc in info.items():
|
||||
if ic.get(term)==scipy.isinf:
|
||||
print "\nIC is Inf on this GO term %s for this gene: %s" %(term,gene)
|
||||
skip=True
|
||||
if ic.get(term)==None:
|
||||
#print "\nHave no IC on this GO term %s for this gene: %s" %(term,gene)
|
||||
skip=True
|
||||
ii += 1
|
||||
if desc['Ontology']!=ontology:
|
||||
#print "\nThis GO term %s belongs to: %s:" %(term,desc['Ontology'])
|
||||
skip = True
|
||||
dd += 1
|
||||
if not skip:
|
||||
if gene2terms.has_key(gene):
|
||||
gene2terms[gene].append(term)
|
||||
else:
|
||||
gene2terms[gene] = [term]
|
||||
jj = 0
|
||||
all = rpy.r.mget(gene_ids, rpy.r.GOENTREZID2GO,ifnotfound="NA")
|
||||
for gene, terms in all.items():
|
||||
if terms!="NA":
|
||||
for term,desc in terms.items():
|
||||
if desc['Ontology'].lower() == ontology and term in ic:
|
||||
if ic[term]>.88:
|
||||
jj+=1
|
||||
continue
|
||||
cc+=1
|
||||
gene2terms[gene].append(term)
|
||||
else:
|
||||
dd+=1
|
||||
else:
|
||||
cc += 1
|
||||
|
||||
print "\nNumber of genes without annotation: %d" %cc
|
||||
ii+=1
|
||||
|
||||
|
||||
print "\nNumber of genes without annotation: %d" %ii
|
||||
print "\nNumber of genes not in %s : %d " %(ontology, dd)
|
||||
print "\nNumber of genes with infs : %d " %ii
|
||||
print "\nNumber of genes with too high IC : %d " %jj
|
||||
|
||||
return gene2terms
|
||||
|
||||
|
|
@ -220,3 +212,15 @@ def data_aff2loc_hgu133a(X, aff_ids, verbose=False):
|
|||
print "Ids with unique probeset: %d" %s
|
||||
X = scipy.asarray(new_data).T
|
||||
return X, new_ids
|
||||
|
||||
def R_PLS(x,y,ncomp=3, validation='"LOO"'):
|
||||
rpy.r.library("pls")
|
||||
rpy.r.assign("X", x)
|
||||
rpy.r.assign("Y", y)
|
||||
callstr = "plsr(Y~X, ncomp=" + str(ncomp) + ", validation=" + validation + ")"
|
||||
print callstr
|
||||
result = rpy.r(callstr)
|
||||
return result
|
||||
|
||||
|
||||
def gogene()
|
||||
|
|
|
|||
|
|
@ -1,17 +1,56 @@
|
|||
import sys
|
||||
import rpy
|
||||
from pylab import gca, figure, subplot
|
||||
from pylab import gca, figure, subplot,plot
|
||||
from scipy import *
|
||||
from lpls import *
|
||||
from scipy.linalg import norm
|
||||
from lpls import correlation_loadings
|
||||
|
||||
import rpy_go
|
||||
sys.path.append("../../fluents") # home of dataset
|
||||
sys.path.append("../../fluents/lib") # home of cx_stats
|
||||
sys.path.append("/home/flatberg/fluents/scripts/lpls")
|
||||
import dataset
|
||||
import cx_stats
|
||||
from plots_lpls import plot_corrloads
|
||||
from engines import nipals_lpls
|
||||
from validation import lpls_val, lpls_jk
|
||||
from plots_lpls import plot_corrloads, plot_dag
|
||||
import plots_lpls
|
||||
|
||||
# Possible outliers
|
||||
# http://www.pubmedcentral.nih.gov/articlerender.fcgi?tool=pubmed&pubmedid=16817967
|
||||
sample_outliers = ['OV:NCI_ADR_RES', 'CNS:SF_295', 'CNS:SF_539', 'RE:SN12C', 'LC:NCI_H226', 'LC:NCI_H522', 'PR:PC_3', 'PR:DU_145']
|
||||
|
||||
|
||||
####### OPTIONS ###########
|
||||
# data
|
||||
chip = "hgu133a"
|
||||
use_data = 'uma'
|
||||
subset = 'plsr'
|
||||
small_test = False
|
||||
use_sbg_subset = True # the sandberg nci-Ygroups subset
|
||||
std_y = True
|
||||
std_z = False
|
||||
# go
|
||||
ontology = "bp"
|
||||
min_genes = 5
|
||||
similarities = ("JiangConrath","Resnik","Lin","CoutoEnriched","CoutoJiangConrath","CoutoResnik","CoutoLin")
|
||||
meth = similarities[2]
|
||||
go_term_sim = "OA"
|
||||
# lpls
|
||||
a_max = 5
|
||||
aopt = 2
|
||||
xz_alpha = .4
|
||||
w_alpha = .3
|
||||
mean_ctr = [2, 0, 2]
|
||||
nsets = None
|
||||
qval_cutoff = 0.01
|
||||
n_iter = 200
|
||||
|
||||
alpha_check = False
|
||||
calc_rmsep = False
|
||||
|
||||
|
||||
######## DATA ##########
|
||||
use_data='uma'
|
||||
if use_data=='smoker':
|
||||
# full smoker data
|
||||
DX = dataset.read_ftsv(open("../../data/smokers-full/Smokers.ftsv"))
|
||||
|
|
@ -21,7 +60,7 @@ if use_data=='smoker':
|
|||
elif use_data=='scherf':
|
||||
DX = dataset.read_ftsv(open("../../data/scherf/scherfX.ftsv"))
|
||||
DY = dataset.read_ftsv(open("../../data/scherf/scherfY.ftsv"))
|
||||
Y = DY.asarray().astype('d')
|
||||
Yg = DY.asarray().astype('d')
|
||||
gene_ids = DX.get_identifiers('gene_ids', sorted=True)
|
||||
elif use_data=='staunton':
|
||||
pass
|
||||
|
|
@ -30,10 +69,16 @@ elif use_data=='uma':
|
|||
DYg = dataset.read_ftsv(open("../../data/uma/Yg133.ftsv"))
|
||||
DY = dataset.read_ftsv(open("../../data/uma/Yd.ftsv"))
|
||||
Y = DY.asarray().astype('d')
|
||||
Yg = DYg.asarray().astype('d')
|
||||
gene_ids = DX.get_identifiers('gene_ids', sorted=True)
|
||||
|
||||
# use subset with defined GO-terms
|
||||
gene2goterms = rpy_go.goterms_from_gene(gene_ids)
|
||||
ic_all = 2026006.0 # sum of all ic in BP
|
||||
max_ic = -log(1/ic_all)
|
||||
ic_cutoff = -log(min_genes/ic_all)/max_ic
|
||||
print "Information cutoff for min %d genes: %.2f" %(min_genes, ic_cutoff)
|
||||
|
||||
gene2goterms = rpy_go.goterms_from_gene(gene_ids, ic_cutoff=ic_cutoff)
|
||||
all_terms = set()
|
||||
for t in gene2goterms.values():
|
||||
all_terms.update(t)
|
||||
|
|
@ -48,20 +93,18 @@ X = DX.asarray()
|
|||
index = DX.get_indices('gene_ids', gene_ids)
|
||||
X = X[:,index]
|
||||
|
||||
1/0
|
||||
|
||||
# Use only subset defined on GO
|
||||
ontology = 'BP'
|
||||
print "\n\nFiltering genes by Go terms "
|
||||
|
||||
# use subset based on SAM or IQR
|
||||
subset = 'not'
|
||||
# use subset based on SAM,PLSR or (IQR)
|
||||
|
||||
if subset=='sam':
|
||||
# select subset genes by SAM
|
||||
rpy.r.library("siggenes")
|
||||
rpy.r.library("qvalue")
|
||||
rpy.r.assign("data", X.T)
|
||||
cl = dot(DY.asarray(), diag(arange(Y.shape[1])+1)).sum(1)
|
||||
cl = dot(DYg.asarray(), diag(arange(Yg.shape[1])+1)).sum(1)
|
||||
rpy.r.assign("cl", cl)
|
||||
rpy.r.assign("B", 20)
|
||||
# Perform a SAM analysis.
|
||||
|
|
@ -72,21 +115,30 @@ if subset=='sam':
|
|||
qq = rpy.r('qobj<-qvalue(sam.out@p.value)')
|
||||
qvals = asarray(qq['qvalues'])
|
||||
# cut off
|
||||
cutoff = 0.01
|
||||
cutoff = 0.001
|
||||
index = where(qvals<cutoff)[0]
|
||||
if small_test:
|
||||
index = index[:20]
|
||||
|
||||
# Subset data
|
||||
X = X[:,index]
|
||||
|
||||
gene_ids = [gid for i, gid in enumerate(gene_ids) if i in index]
|
||||
print "\nNumber of genes: %s" %len(gene_ids)
|
||||
print "\nWorking on subset with %s genes " %len(gene_ids)
|
||||
|
||||
# update valid go-terms
|
||||
gene2goterms = rpy_go.goterms_from_gene(gene_ids)
|
||||
gene2goterms = rpy_go.goterms_from_gene(gene_ids, ic_cutoff=ic_cutoff)
|
||||
all_terms = set()
|
||||
for t in gene2goterms.values():
|
||||
all_terms.update(t)
|
||||
terms = list(all_terms)
|
||||
print "\nNumber of go-terms: %s" %len(terms)
|
||||
# update genelist
|
||||
gene_ids = gene2goterms.keys()
|
||||
print "\nNumber of genes: %s" %len(gene_ids)
|
||||
elif subset=='plsr':
|
||||
cx_stats.pls_qvals(X, Y)
|
||||
else:
|
||||
# noimp (smoker data is prefiltered)
|
||||
pass
|
||||
|
|
@ -94,90 +146,151 @@ else:
|
|||
|
||||
rpy.r.library("GOSim")
|
||||
# Go-term similarity matrix
|
||||
methods = ("JiangConrath","Resnik","Lin","CoutoEnriched","CoutoJiangConrath","CoutoResnik","CoutoLin")
|
||||
meth = methods[2]
|
||||
print "Term-term similarity matrix (method = %s)" %meth
|
||||
|
||||
print "Term-term similarity matrix (method = %s)" %meth
|
||||
print "\nCalculating term-term similarity matrix"
|
||||
|
||||
if meth=="CoutoEnriched":
|
||||
aa = 0
|
||||
ba = 0
|
||||
rpy.r.setEnrichmentFactors(alpha = aa, beta =ba)
|
||||
rpytmat = rpy.with_mode(rpy.NO_CONVERSION, rpy.r.getTermSim)(terms, method=meth,verbose=False)
|
||||
tmat = rpy.r.assign("haha", rpytmat)
|
||||
print "\n Calculating Z matrix"
|
||||
Z = rpy_go.genego_sim(gene2goterms,gene_ids,terms,rpytmat,go_term_sim="OA",term_sim=meth)
|
||||
Z = rpy_go.genego_sim(gene2goterms,gene_ids,terms,rpytmat,go_term_sim=go_term_sim,term_sim=meth)
|
||||
|
||||
# update data (X) matrix
|
||||
newind = DX.get_indices('gene_ids', gene_ids)
|
||||
Xr = DX.asarray()[:,newind]
|
||||
|
||||
|
||||
######## LPLSR ########
|
||||
######## LPLSR ########
|
||||
print "LPLSR ..."
|
||||
a_max = 10
|
||||
aopt = 3
|
||||
xz_alpha = .6
|
||||
w_alpha = .1
|
||||
mean_ctr = [2, 0, 2]
|
||||
Y = Yg
|
||||
|
||||
if use_sbg_subset:
|
||||
Y_old = Y.copy()
|
||||
Xr_old = Xr.copy()
|
||||
keep_samples = ['CN', 'ME', 'LE', 'CO', 'RE']
|
||||
sample_ids = DY.get_identifiers('cline', sorted=True)
|
||||
keep_ind = [i for i,name in enumerate(sample_ids) if name[:2] in keep_samples]
|
||||
Xr = Xr[keep_ind,:]
|
||||
Y = Y[keep_ind,:]
|
||||
Y = Y[:, where(Y.sum(0)>1)[0]]
|
||||
|
||||
|
||||
# standardize Z?
|
||||
sdtz = False
|
||||
if sdtz:
|
||||
Z = Z/Z.std(0)
|
||||
|
||||
T, W, P, Q, U, L, K, B, b0, evx, evy, evz,mnx,mny,mnz = nipals_lpls(Xr,Y,Z, a_max,
|
||||
alpha=xz_alpha,
|
||||
mean_ctr=mean_ctr)
|
||||
sdty = True
|
||||
if sdty:
|
||||
Y = Y/Y.std(0)
|
||||
lpls_result = nipals_lpls(Xr,Y,Z, a_max,alpha=xz_alpha,mean_ctr=mean_ctr)
|
||||
globals().update(lpls_result)
|
||||
|
||||
# Correlation loadings
|
||||
dx,Rx,rssx = correlation_loadings(Xr, T, P)
|
||||
dx,Ry,rssy = correlation_loadings(Y, T, Q)
|
||||
cadz,Rz,rssz = correlation_loadings(Z.T, W, L)
|
||||
|
||||
# Prediction error
|
||||
rmsep , yhat, class_error = cv_lpls(Xr, Y, Z, a_max, alpha=xz_alpha,mean_ctr=mean_ctr)
|
||||
alpha_check=True
|
||||
if alpha_check:
|
||||
|
||||
if calc_rmsep:
|
||||
rmsep , yhat, class_error = lpls_val(Xr, Y, Z, a_max, alpha=xz_alpha,mean_ctr=mean_ctr)
|
||||
|
||||
|
||||
if alpha_check:
|
||||
Alpha = arange(0.01, 1, .1)
|
||||
Rmsep,Yhat, CE = [],[],[]
|
||||
for a in Alpha:
|
||||
print "alpha %f" %a
|
||||
rmsep , yhat, ce = cv_lpls(Xr, Y, Z, a_max, alpha=xz_alpha,mean_ctr=mean_ctr)
|
||||
Rmsep.append(rmsep)
|
||||
Yhat.append(yhat)
|
||||
CE.append(ce)
|
||||
rmsep , yhat, ce = lpls_val(Xr, Y, Z, a_max, alpha=a,mean_ctr=mean_ctr,nsets=nsets)
|
||||
Rmsep.append(rmsep.copy())
|
||||
#Yhat.append(yhat.copy())
|
||||
#CE.append(ce.copy())
|
||||
Rmsep = asarray(Rmsep)
|
||||
Yhat = asarray(Yhat)
|
||||
CE = asarray(CE)
|
||||
#Yhat = asarray(Yhat)
|
||||
#CE = asarray(CE)
|
||||
|
||||
|
||||
# Significance Hotellings T
|
||||
Wx, Wz, Wy, = jk_lpls(Xr, Y, Z, aopt, mean_ctr=mean_ctr,alpha=xz_alpha)
|
||||
Ws = W*apply_along_axis(norm, 0, T)
|
||||
tsqx = cx_stats.hotelling(Wx, Ws[:,:aopt], alpha=w_alpha)
|
||||
tsqz = cx_stats.hotelling(Wz, L[:,:aopt], alpha=0)
|
||||
#Wx, Wz = lpls_jk(Xr, Y, Z, aopt, mean_ctr=mean_ctr, xz_alpha=xz_alpha, nsets=nsets)
|
||||
#Ws = W*apply_along_axis(norm, 0, T)
|
||||
#tsqx = cx_stats.hotelling(Wx, Ws[:,:aopt], alpha=w_alpha)
|
||||
#tsqz = cx_stats.hotelling(Wz, L[:,:aopt], alpha=0)
|
||||
|
||||
# qvals
|
||||
cal_tsq_z, pert_tsq_z, cal_tsq_x, pert_tsq_x = cx_stats.lpls_qvals(Xr, Y, Z, aopt=aopt, zx_alpha=xz_alpha, n_iter=n_iter)
|
||||
|
||||
qvalz = cx_stats.fdr(cal_tsq_z, pert_tsq_z, 'median')
|
||||
qvalx = cx_stats.fdr(cal_tsq_x, pert_tsq_x, 'median')
|
||||
|
||||
|
||||
## plots ##
|
||||
figure(1) #rmsep
|
||||
bar_w = .2
|
||||
bar_col = 'rgb'*5
|
||||
m = Y.shape[1]
|
||||
for a in range(m):
|
||||
bar(arange(a_max)+a*bar_w+.1, rmsep[:,a], width=bar_w, color=bar_col[a])
|
||||
ylim([rmsep.min()-.05, rmsep.max()+.05])
|
||||
title('RMSEP')
|
||||
# p-values, set-enrichment analysis
|
||||
active_genes_ids = where(qvalx < qval_cutoff)[0]
|
||||
active_genes = [name for i,name in enumerate(gene_ids) if i in active_genes_ids]
|
||||
active_universe = gene_ids
|
||||
gsea_result, gsea_params= rpy_go.gene_GO_hypergeo_test(genelist=active_genes,universe=active_universe,chip=chip,pval_cutoff=1.0,cond=False,test_direction="over")
|
||||
active_goterms_ids = where(qvalz < qval_cutoff)[0]
|
||||
active_goterms = [name for i,name in enumerate(terms) if i in active_goterms_ids]
|
||||
|
||||
figure(2)
|
||||
for a in range(m):
|
||||
bar(arange(a_max)+a*bar_w+.1, class_error[:,a], width=bar_w, color=bar_col[a])
|
||||
ylim([class_error.min()-.05, class_error.max()+.05])
|
||||
title('Classification accuracy')
|
||||
gsea_t2p = dict(zip(gsea_result['GOBPID'], gsea_result['Pvalue']))
|
||||
|
||||
figure(3) # Hypoid correlations
|
||||
|
||||
|
||||
|
||||
#### PLOTS ####
|
||||
|
||||
from pylab import *
|
||||
from scipy import where
|
||||
dg = plots_lpls.dag(terms, "bp")
|
||||
pos = None
|
||||
|
||||
figure(300)
|
||||
subplot(2,1,1)
|
||||
pos = plots_lpls.plot_dag(dg, node_color=cal_tsq_z, pos=pos, nodelist=terms)
|
||||
subplot(2,1,2)
|
||||
pos = plot_dag(dg, node_color=qvalz, pos=pos, nodelist=terms)
|
||||
|
||||
|
||||
if calc_rmsep:
|
||||
figure(1) #rmsep
|
||||
bar_w = .2
|
||||
bar_col = 'rgb'*5
|
||||
m = Y.shape[1]
|
||||
for a in range(m):
|
||||
bar(arange(a_max)+a*bar_w+.1, rmsep[:,a], width=bar_w, color=bar_col[a])
|
||||
ylim([rmsep.min()-.05, rmsep.max()+.05])
|
||||
title('RMSEP: Y(%s)' %Y.get_name())
|
||||
|
||||
#figure(2)
|
||||
#for a in range(m):
|
||||
# bar(arange(a_max)+a*bar_w+.1, class_error[:,a], width=bar_w, color=bar_col[a])
|
||||
#ylim([class_error.min()-.05, class_error.max()+.05])
|
||||
#title('Classification accuracy')
|
||||
|
||||
figure(3) # Hyploid correlations
|
||||
tsqz = cal_tsq_z
|
||||
tsqx = cal_tsq_x
|
||||
tsqz_s = 250*tsqz/tsqz.max()
|
||||
plot_corrloads(Rz, pc1=0, pc2=1, s=tsqz_s, c=tsqz, zorder=5, expvar=evz, ax=None,alpha=.5)
|
||||
td = rpy_go.goterm2desc(terms)
|
||||
tlabels = [td[i] for i in terms]
|
||||
keep = where(qvalz<0.01)[0]
|
||||
k_Rz = Rz[keep,:]
|
||||
k_tsqz_s = tsqz_s[keep]
|
||||
k_tsq = tsqz[keep]
|
||||
k_tlabels = [name for i,name in enumerate(tlabels) if i in keep]
|
||||
plot_corrloads(Rz, pc1=0, pc2=1, s=tsqz_s, c=tsqz, zorder=5, expvar=evz, ax=None,alpha=.5,labels=None)
|
||||
#plot_corrloads(k_Rz, pc1=0, pc2=1, s=k_tsqz_s, c=k_tsqz, zorder=5, expvar=evz, ax=None,alpha=.5,labels=None)
|
||||
ax = gca()
|
||||
yglabels = DYg.get_identifiers(DYg.get_dim_name()[1], sorted=True)
|
||||
ylabels = DY.get_identifiers(DY.get_dim_name()[1], sorted=True)
|
||||
plot_corrloads(Ry, pc1=0, pc2=1, s=150, c='g', zorder=5, expvar=evy, ax=ax,labels=ylabels,alpha=.5)
|
||||
blabels = yglabels[:]
|
||||
blabels.append(ylabels[0])
|
||||
plot_corrloads(Ry, pc1=0, pc2=1, s=150, c='g', marker='s', zorder=5, expvar=evy, ax=ax,labels=None,alpha=.9)
|
||||
plot_corrloads(Rx, pc1=0, pc2=1, s=5, c='k', zorder=1, expvar=evx, ax=ax)
|
||||
|
||||
|
||||
figure(4)
|
||||
subplot(221)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import sys,os
|
||||
import webbrowser
|
||||
import cPickle
|
||||
|
||||
from fluents import logger, plots,workflow,dataset,main
|
||||
from fluents.lib import blmfuncs,nx_utils,validation,engines,cx_stats,cx_utils
|
||||
|
|
@ -57,6 +58,8 @@ class SmallTestWorkflow(workflow.Workflow):
|
|||
go.add_function(gobrowser.PlotDagFunction())
|
||||
go.add_function(GoEnrichment())
|
||||
go.add_function(GoEnrichmentCond())
|
||||
go.add_function(MapGO2Gene())
|
||||
go.add_function(MapGene2GO())
|
||||
self.add_stage(go)
|
||||
|
||||
# EXTRA PLOTS
|
||||
|
|
@ -354,7 +357,7 @@ class GoEnrichment(workflow.Function):
|
|||
# Get universe
|
||||
# Here, we are using a defined dataset to represent the universe
|
||||
if not 'gene_ids' in data:
|
||||
logger.log('notice', 'No dimension called [gene_ids] in dataset: %s', data.get_name())
|
||||
logger.log('notice', 'No dimension called [gene_ids] in dataset: %s' %data.get_name())
|
||||
return
|
||||
universe = list(data.get_identifiers('gene_ids'))
|
||||
logger.log('notice', 'Universe consists of %s gene ids from %s' %(len(universe), data.get_name()))
|
||||
|
|
@ -417,7 +420,7 @@ class GoEnrichmentCond(workflow.Function):
|
|||
return
|
||||
|
||||
# Hypergeometric parameter object
|
||||
pval_cutoff = 0.9999
|
||||
pval_cutoff = 1
|
||||
cond = True
|
||||
test_direction = 'over'
|
||||
params = rpy.r.new("GOHyperGParams",
|
||||
|
|
@ -443,3 +446,65 @@ class GoEnrichmentCond(workflow.Function):
|
|||
name='P values (enrichment)')
|
||||
return [xout]
|
||||
|
||||
|
||||
class MapGene2GO(workflow.Function):
|
||||
def __init__(self, ont='bp', gene_id_name='gene_ids'):
|
||||
self._ont = ont
|
||||
self._gene_id_name = gene_id_name
|
||||
workflow.Function.__init__(self, 'gene2go', 'gene->GO')
|
||||
# load data at init
|
||||
try:
|
||||
fname = "/home/flatberg/fluents/data/gene2go.pcl"
|
||||
self._gene2go = cPickle.load(open(fname))
|
||||
except:
|
||||
logger.log("notice", "could not load mapping")
|
||||
|
||||
def run(self):
|
||||
selection = main.project.get_selection()
|
||||
if not selection.has_key(self._gene_id_name):
|
||||
logger.log("notice", "Expected gene ids: %s, but got. %s" %(self._gene_id_name, selection.keys()))
|
||||
return None
|
||||
if len(selection[self._gene_id_name])==0:
|
||||
logger.log("notice", "No selected genes to query")
|
||||
return None
|
||||
|
||||
gene_ids = selection[self._gene_id_name]
|
||||
go_ids = set()
|
||||
for gene in gene_ids:
|
||||
go_ids_new = self._gene2go.get(gene, [])
|
||||
if not go_ids_new:
|
||||
logger.log("notice", "Could not find any goterms for %s" %gene)
|
||||
go_ids.update(self._gene2go.get(gene, []))
|
||||
main.project.set_selection('go-terms', go_ids)
|
||||
logger.log("notice", "GO terms updated")
|
||||
|
||||
|
||||
class MapGO2Gene(workflow.Function):
|
||||
def __init__(self, ont='bp', gene_id_name='go-terms'):
|
||||
self._ont = ont
|
||||
self._gene_id_name = gene_id_name
|
||||
workflow.Function.__init__(self, 'go2gene', 'GO->gene')
|
||||
# load data at init
|
||||
try:
|
||||
fname = "/home/flatberg/fluents/data/go2gene.pcl"
|
||||
self._go2gene = cPickle.load(open(fname))
|
||||
except:
|
||||
logger.log("notice", "could not load mapping")
|
||||
|
||||
def run(self):
|
||||
selection = main.project.get_selection()
|
||||
if not selection.has_key(self._gene_id_name):
|
||||
logger.log("notice", "Expected gene ids: %s, but got. %s" %(self._gene_id_name, selection.keys()))
|
||||
return None
|
||||
if len(selection[self._gene_id_name])==0:
|
||||
logger.log("notice", "No selected genes to query")
|
||||
return None
|
||||
|
||||
go_ids = selection[self._gene_id_name]
|
||||
gene_ids = set()
|
||||
for go in go_ids:
|
||||
if not self._go2gene.get(go,[]):
|
||||
logger.log("notice", "Could not find any gene ids for %s" %go)
|
||||
gene_ids.update(self._go2gene.get(go,[]))
|
||||
main.project.set_selection('gene_ids', gene_ids)
|
||||
logger.log("notice", "GO terms updated")
|
||||
|
|
|
|||
Reference in New Issue