Path: blob/master/sandbox/paper_analysis/insample/estimators.R
1433 views
# Function for data cleaning1#-------------------------------------------23cleaning = function(mData, alpha , trim=1e-3)4{5T=dim(mData)[1];6N=dim(mData)[2];7MCD = covMcd(mData,alpha=1-alpha)8mu = as.matrix(MCD$raw.center) #no reweighting9sigma = MCD$raw.cov10invSigma = solve(sigma);11vd2t = c();12cleaneddata = mData13outlierdate = c()1415# 1. Sort the data in function of their extremeness16# Extremeness is proxied by the robustly estimated squared Mahalanbobis distance1718for(t in c(1:T) )19{20d2t = as.matrix(mData[t,]-mu)%*%invSigma%*%t(as.matrix(mData[t,]-mu));21vd2t = c(vd2t,d2t);22}23sortt = sort(vd2t,index.return=TRUE)$ix2425# 2. Outlier detection26# 2.1. Only the alpha most extreme observations can be qualified as outliers2728T.alpha = floor(T * (1-alpha))+129cleanedt=sortt[c(T.alpha:T)]3031# 2.2. multivariate winsorization (Khan et al, 2007) :32# bound the MD of the most exteme observations to a quantile of the chi squared distribution, N df3334for(t in cleanedt ){35if(vd2t[t]>qchisq(1-trim,N)){36# print(c("Observation",as.character(date[t]),"is detected as outlier and cleaned") );37cleaneddata[t,] = sqrt(qchisq(1-trim,N)/vd2t[t])*mData[t,];38outlierdate = c(outlierdate,date[t]) } }39return(list(cleaneddata,outlierdate))40}4142# Definition of statistics needed to compute Gaussian and modified VaR and ES for univariate time series43#---------------------------------------------------------------------------------------------------------4445centeredmoment = function(series,power)46{47location = mean(series);48out = sum( (series-location)^power )/length(series);49return(out);50}5152skew = function(series)53{54m2 = centeredmoment(series,2)55m3 = centeredmoment(series,3)56out = m3 / m2^(3/2)57return(out)58}5960exkur = function(series)61{62m2 = centeredmoment(series,2)63m4 = centeredmoment(series,4)64out = m4 / m2^(2) - 365return(out)66}6768pvalJB = function(series)69{70m2 = centeredmoment(series,2)71m3 = centeredmoment(series,3)72m4 = centeredmoment(series,4)73skew = m3 / m2^(3/2);74exkur = m4 / m2^(2) - 3;75JB = ( length(series)/6 )*( skew^2 + (1/4)*(exkur^2) )76out = 1-pchisq(JB,df=2)77}7879gausVaR = function(series,alpha)80{81location = mean(series);82m2 = centeredmoment(series,2)83out = - location - qnorm(alpha)*sqrt(m2)84return(out)85}8687gausES = function(series,alpha)88{89location = mean(series);90m2 = centeredmoment(series,2)91out = - location + dnorm(qnorm(alpha))*sqrt(m2)/alpha92return(out)93}9495MVaR = function(series,alpha,r)96{97z = qnorm(alpha)98location = mean(series);99m2 = centeredmoment(series,2)100m3 = centeredmoment(series,3)101m4 = centeredmoment(series,4)102skew = m3 / m2^(3/2);103exkurt = m4 / m2^(2) - 3;104105h = z + (1/6)*(z^2 -1)*skew106if(r==2){ h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2; }107108out = - location - h*sqrt(m2)109return(out)110}111112Ipower = function(power,h)113{114115fullprod = 1;116117if( (power%%2)==0 ) #even number: number mod is zero118{119pstar = power/2;120for(j in c(1:pstar))121{122fullprod = fullprod*(2*j)123}124I = fullprod*dnorm(h);125126for(i in c(1:pstar) )127{128prod = 1;129for(j in c(1:i) )130{131prod = prod*(2*j)132}133I = I + (fullprod/prod)*(h^(2*i))*dnorm(h)134}135}136else{137pstar = (power-1)/2138for(j in c(0:pstar) )139{140fullprod = fullprod*( (2*j)+1 )141}142I = -fullprod*pnorm(h);143144for(i in c(0:pstar) )145{146prod = 1;147for(j in c(0:i) )148{149prod = prod*( (2*j) + 1 )150}151I = I + (fullprod/prod)*(h^( (2*i) + 1))*dnorm(h)152}153}154return(I)155}156157MES = function(series,alpha,r=2)158{159z = qnorm(alpha)160location = mean(series);161m2 = centeredmoment(series,2)162m3 = centeredmoment(series,3)163m4 = centeredmoment(series,4)164skew = m3 / m2^(3/2);165exkurt = m4 / m2^(2) - 3;166167h = z + (1/6)*(z^2 -1)*skew168if(r==2){ h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2};169170MES = dnorm(h)171MES = MES + (1/24)*( Ipower(4,h) - 6*Ipower(2,h) + 3*dnorm(h) )*exkurt172MES = MES + (1/6)*( Ipower(3,h) - 3*Ipower(1,h) )*skew;173MES = MES + (1/72)*( Ipower(6,h) -15*Ipower(4,h)+ 45*Ipower(2,h) - 15*dnorm(h) )*(skew^2)174MES = - location + (sqrt(m2)/alpha)*MES175return(MES)176}177178operMES = function(series,alpha,r)179{180z = qnorm(alpha)181location = mean(series);182m2 = centeredmoment(series,2)183m3 = centeredmoment(series,3)184m4 = centeredmoment(series,4)185skew = m3 / m2^(3/2);186exkurt = m4 / m2^(2) - 3;187188h = z + (1/6)*(z^2 -1)*skew189if(r==2){ h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2};190191MES = dnorm(h)192MES = MES + (1/24)*( Ipower(4,h) - 6*Ipower(2,h) + 3*dnorm(h) )*exkurt193MES = MES + (1/6)*( Ipower(3,h) - 3*Ipower(1,h) )*skew;194MES = MES + (1/72)*( Ipower(6,h) -15*Ipower(4,h)+ 45*Ipower(2,h) - 15*dnorm(h) )*(skew^2)195MES = - location - (sqrt(m2))*min( -MES/alpha , h )196return(MES)197}198199# Definition of statistics needed to compute Gaussian and modified VaR and ES for the return series of portfolios200# and to compute the contributions to portfolio downside risk, made by the different positions in the portfolio.201#----------------------------------------------------------------------------------------------------------------202203204205m2 = function(w,sigma)206{207return(t(w)%*%sigma%*%w)208}209derm2 = function(w,sigma)210{211return(2*sigma%*%w)212}213m3 = function(w,M3)214{215return(t(w)%*%M3%*%(w%x%w))216}217derm3 = function(w,M3)218{219return(3*M3%*%(w%x%w))220}221m4 = function(w,M4)222{223return(t(w)%*%M4%*%(w%x%w%x%w))224}225derm4 = function(w,M4)226{227return(4*M4%*%(w%x%w%x%w))228}229230precision = 4;231232Portmean = function(w,mu,precision=4)233{234return( list( round( t(w)%*%mu , precision) , round ( as.vector(w)*as.vector(mu) , precision ) , round( as.vector(w)*as.vector(mu)/t(w)%*%mu) , precision) )235}236237Portsd = function(w,sigma,precision=4)238{239pm2 = m2(w,sigma)240dpm2 = derm2(w,sigma)241dersd = (0.5*as.vector(dpm2))/sqrt(pm2);242contrib = dersd*as.vector(w)243return(list( round( sqrt(pm2) , precision ) , round( contrib , precision ) , round ( contrib/sqrt(pm2) , precision) ))244}245246PortgausVaR = function(alpha,w,mu,sigma,precision=4)247{248location = t(w)%*%mu249pm2 = m2(w,sigma)250dpm2 = derm2(w,sigma)251VaR = - location - qnorm(alpha)*sqrt(pm2)252derVaR = - as.vector(mu)- qnorm(alpha)*(0.5*as.vector(dpm2))/sqrt(pm2);253contrib = derVaR*as.vector(w)254return(list( round( VaR , precision ) , round ( contrib , precision ) , round( contrib/VaR , precision) ))255}256257PortgausES = function(alpha,w,mu,sigma,precision=4)258{259location = t(w)%*%mu260pm2 = m2(w,sigma)261dpm2 = derm2(w,sigma)262ES = - location + dnorm(qnorm(alpha))*sqrt(pm2)/alpha263derES = - as.vector(mu) + (1/alpha)*dnorm(qnorm(alpha))*(0.5*as.vector(dpm2))/sqrt(pm2);264contrib = as.vector(w)*derES;265return(list( round( ES , precision ) , round( contrib , precision) , round( contrib/ES , precision) ))266}267268PortSkew = function(w,sigma,M3)269{270pm2 = m2(w,sigma)271pm3 = m3(w,M3)272skew = pm3 / pm2^(3/2);273return( skew )274}275276PortKurt = function(w,sigma,M4)277{278pm2 = m2(w,sigma)279pm4 = m4(w,M4)280kurt = pm4 / pm2^(2) ;281return( kurt )282}283284PortMVaR = function(alpha,w,mu,sigma,M3,M4,precision=4)285{286z = qnorm(alpha)287location = t(w)%*%mu288pm2 = m2(w,sigma)289dpm2 = as.vector( derm2(w,sigma) )290pm3 = m3(w,M3)291dpm3 = as.vector( derm3(w,M3) )292pm4 = m4(w,M4)293dpm4 = as.vector( derm4(w,M4) )294295skew = pm3 / pm2^(3/2);296exkurt = pm4 / pm2^(2) - 3;297298derskew = ( 2*(pm2^(3/2))*dpm3 - 3*pm3*sqrt(pm2)*dpm2 )/(2*pm2^3)299derexkurt = ( (pm2)*dpm4 - 2*pm4*dpm2 )/(pm2^3)300301h = z + (1/6)*(z^2 -1)*skew302h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2;303304MVaR = - location - h*sqrt(pm2)305306derGausVaR = - as.vector(mu)- qnorm(alpha)*(0.5*as.vector(dpm2))/sqrt(pm2);307derMVaR = derGausVaR + (0.5*dpm2/sqrt(pm2))*( -(1/6)*(z^2 -1)*skew - (1/24)*(z^3 - 3*z)*exkurt + (1/36)*(2*z^3 - 5*z)*skew^2 )308derMVaR = derMVaR + sqrt(pm2)*( -(1/6)*(z^2 -1)*derskew - (1/24)*(z^3 - 3*z)*derexkurt + (1/36)*(2*z^3 - 5*z)*2*skew*derskew )309contrib = as.vector(w)*as.vector(derMVaR)310return(list( round( MVaR , precision) , round( contrib , precision ), round (contrib/MVaR , precision ) ) )311}312313resPortMVaR = function(alpha,w,mu,sigma,resSigma,M3,M4,precision=4)314{315z = qnorm(alpha)316location = t(w)%*%mu317pm2 = m2(w,sigma)318respm2 = m2(w,resSigma)319dpm2 = as.vector( derm2(w,sigma) )320drespm2 = as.vector( derm2(w,resSigma) )321pm3 = m3(w,M3)322dpm3 = as.vector( derm3(w,M3) )323pm4 = m4(w,M4)324dpm4 = as.vector( derm4(w,M4) )325326skew = pm3 / respm2^(3/2);327exkurt = pm4 / respm2^(2) - 3;328329derskew = ( 2*(respm2^(3/2))*dpm3 - 3*pm3*sqrt(respm2)*drespm2 )/(2*respm2^3)330derexkurt = ( (respm2)*dpm4 - 2*pm4*drespm2 )/(respm2^3)331332h = z + (1/6)*(z^2 -1)*skew333h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2;334335MVaR = - location - h*sqrt(pm2)336337derGausVaR = - as.vector(mu)- qnorm(alpha)*(0.5*as.vector(dpm2))/sqrt(pm2);338derMVaR = derGausVaR + (0.5*dpm2/sqrt(pm2))*( -(1/6)*(z^2 -1)*skew - (1/24)*(z^3 - 3*z)*exkurt + (1/36)*(2*z^3 - 5*z)*skew^2 )339derMVaR = derMVaR + sqrt(pm2)*( -(1/6)*(z^2 -1)*derskew - (1/24)*(z^3 - 3*z)*derexkurt + (1/36)*(2*z^3 - 5*z)*2*skew*derskew )340contrib = as.vector(w)*as.vector(derMVaR)341return(list( round( MVaR , precision) , round( contrib , precision ), round (contrib/MVaR , precision ) ) )342}343344derIpower = function(power,h)345{346347fullprod = 1;348349if( (power%%2)==0 ) #even number: number mod is zero350{351pstar = power/2;352for(j in c(1:pstar))353{354fullprod = fullprod*(2*j)355}356I = -fullprod*h*dnorm(h);357358for(i in c(1:pstar) )359{360prod = 1;361for(j in c(1:i) )362{363prod = prod*(2*j)364}365I = I + (fullprod/prod)*(h^(2*i-1))*(2*i-h^2)*dnorm(h)366}367}else{368pstar = (power-1)/2369for(j in c(0:pstar) )370{371fullprod = fullprod*( (2*j)+1 )372}373I = -fullprod*dnorm(h);374375for(i in c(0:pstar) )376{377prod = 1;378for(j in c(0:i) )379{380prod = prod*( (2*j) + 1 )381}382I = I + (fullprod/prod)*(h^(2*i)*(2*i+1-h^2) )*dnorm(h)383}384}385return(I)386}387388389PortMES = function(alpha,w,mu,sigma,M3,M4,precision=4)390{391z = qnorm(alpha)392location = t(w)%*%mu393pm2 = m2(w,sigma)394dpm2 = as.vector( derm2(w,sigma) )395pm3 = m3(w,M3)396dpm3 = as.vector( derm3(w,M3) )397pm4 = m4(w,M4)398dpm4 = as.vector( derm4(w,M4) )399400skew = pm3 / pm2^(3/2);401exkurt = pm4 / pm2^(2) - 3;402403derskew = ( 2*(pm2^(3/2))*dpm3 - 3*pm3*sqrt(pm2)*dpm2 )/(2*pm2^3)404derexkurt = ( (pm2)*dpm4 - 2*pm4*dpm2 )/(pm2^3)405406h = z + (1/6)*(z^2 -1)*skew407h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2;408409derh = (1/6)*(z^2 -1)*derskew + (1/24)*(z^3 - 3*z)*derexkurt - (1/18)*(2*z^3 - 5*z)*skew*derskew410411E = dnorm(h)412E = E + (1/24)*( Ipower(4,h) - 6*Ipower(2,h) + 3*dnorm(h) )*exkurt413E = E + (1/6)*( Ipower(3,h) - 3*Ipower(1,h) )*skew;414E = E + (1/72)*( Ipower(6,h) -15*Ipower(4,h)+ 45*Ipower(2,h) - 15*dnorm(h) )*(skew^2)415E = E/alpha416MES = - location + sqrt(pm2)*E417418derMES = -mu + 0.5*(dpm2/sqrt(pm2))*E419derE = (1/24)*( Ipower(4,h) - 6*Ipower(2,h) + 3*dnorm(h) )*derexkurt420derE = derE + (1/6)*( Ipower(3,h) - 3*Ipower(1,h) )*derskew421derE = derE + (1/36)*( Ipower(6,h) -15*Ipower(4,h)+ 45*Ipower(2,h) - 15*dnorm(h) )*skew*derskew422X = -h*dnorm(h) + (1/24)*( derIpower(4,h) - 6*derIpower(2,h) -3*h*dnorm(h) )*exkurt423X = X + (1/6)*( derIpower(3,h) - 3*derIpower(1,h) )*skew424X = X + (1/72)*( derIpower(6,h) - 15*derIpower(4,h) + 45*derIpower(2,h) + 15*h*dnorm(h) )*skew^2425derE = derE+derh*X # X is a scalar426derE = derE/alpha427derMES = derMES + sqrt(pm2)*derE428contrib = as.vector(w)*as.vector(derMES)429return(list( round( MES , precision ) , round( contrib , precision ), round( contrib/MES, precision )) )430}431432433operPortMES = function(alpha,w,mu,sigma,M3,M4,precision=4)434{435z = qnorm(alpha)436location = t(w)%*%mu437pm2 = m2(w,sigma)438dpm2 = as.vector( derm2(w,sigma) )439pm3 = m3(w,M3)440dpm3 = as.vector( derm3(w,M3) )441pm4 = m4(w,M4)442dpm4 = as.vector( derm4(w,M4) )443444skew = pm3 / pm2^(3/2);445exkurt = pm4 / pm2^(2) - 3;446447derskew = ( 2*(pm2^(3/2))*dpm3 - 3*pm3*sqrt(pm2)*dpm2 )/(2*pm2^3)448derexkurt = ( (pm2)*dpm4 - 2*pm4*dpm2 )/(pm2^3)449450h = z + (1/6)*(z^2 -1)*skew451h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2;452I1 = Ipower(1,h); I2 = Ipower(2,h); I3 = Ipower(3,h); I4 = Ipower(4,h); I6 = Ipower(6,h);453454derh = (1/6)*(z^2 -1)*derskew + (1/24)*(z^3 - 3*z)*derexkurt - (1/18)*(2*z^3 - 5*z)*skew*derskew455456E = dnorm(h)457E = E + (1/24)*( I4 - 6*I2 + 3*dnorm(h) )*exkurt458E = E + (1/6)*( I3 - 3*I1 )*skew;459E = E + (1/72)*( I6 -15*I4+ 45*I2 - 15*dnorm(h) )*(skew^2)460E = E/alpha461462MES = - location - sqrt(pm2)*min(-E,h)463464if(-E<=h){465derMES = -mu + 0.5*(dpm2/sqrt(pm2))*E466derE = (1/24)*( I4 - 6*I2 + 3*dnorm(h) )*derexkurt467derE = derE + (1/6)*( I3 - 3*I1 )*derskew468derE = derE + (1/36)*( I6 -15*I4 + 45*I2 - 15*dnorm(h) )*skew*derskew469X = -h*dnorm(h) + (1/24)*( derIpower(4,h) - 6*derIpower(2,h) -3*h*dnorm(h) )*exkurt470X = X + (1/6)*( derIpower(3,h) - 3*derIpower(1,h) )*skew471X = X + (1/72)*( derIpower(6,h) - 15*derIpower(4,h) + 45*derIpower(2,h) + 15*h*dnorm(h) )*skew^2472derE = derE+derh*X # X is a scalar473derE = derE/alpha474derMES = derMES + sqrt(pm2)*derE }else{475derMES = -mu - 0.5*(dpm2/sqrt(pm2))*h - sqrt(pm2)*derh ; }476contrib = as.vector(w)*as.vector(derMES)477return(list( round( MES, precision) , round( contrib , precision ) , round(contrib/MES,precision) ) )478}479480481resoperPortMES = function(alpha,w,mu,sigma,resSigma,M3,M4,precision=4)482{483z = qnorm(alpha)484location = t(w)%*%mu485pm2 = m2(w,sigma)486respm2 = m2(w,resSigma)487dpm2 = as.vector( derm2(w,sigma) )488drespm2 = as.vector( derm2(w,resSigma) )489pm3 = m3(w,M3)490dpm3 = as.vector( derm3(w,M3) )491pm4 = m4(w,M4)492dpm4 = as.vector( derm4(w,M4) )493494skew = pm3 / respm2^(3/2);495exkurt = pm4 / respm2^(2) - 3;496497derskew = ( 2*(respm2^(3/2))*dpm3 - 3*pm3*sqrt(respm2)*drespm2 )/(2*respm2^3)498derexkurt = ( (respm2)*dpm4 - 2*pm4*drespm2 )/(respm2^3)499500h = z + (1/6)*(z^2 -1)*skew501h = h + (1/24)*(z^3 - 3*z)*exkurt - (1/36)*(2*z^3 - 5*z)*skew^2;502I1 = Ipower(1,h); I2 = Ipower(2,h); I3 = Ipower(3,h); I4 = Ipower(4,h); I6 = Ipower(6,h);503504derh = (1/6)*(z^2 -1)*derskew + (1/24)*(z^3 - 3*z)*derexkurt - (1/18)*(2*z^3 - 5*z)*skew*derskew505506E = dnorm(h)507E = E + (1/24)*( I4 - 6*I2 + 3*dnorm(h) )*exkurt508E = E + (1/6)*( I3 - 3*I1 )*skew;509E = E + (1/72)*( I6 -15*I4+ 45*I2 - 15*dnorm(h) )*(skew^2)510E = E/alpha511512MES = - location - sqrt(pm2)*min(-E,h)513514if(-E<=h){515derMES = -mu + 0.5*(dpm2/sqrt(pm2))*E516derE = (1/24)*( I4 - 6*I2 + 3*dnorm(h) )*derexkurt517derE = derE + (1/6)*( I3 - 3*I1 )*derskew518derE = derE + (1/36)*( I6 -15*I4 + 45*I2 - 15*dnorm(h) )*skew*derskew519X = -h*dnorm(h) + (1/24)*( derIpower(4,h) - 6*derIpower(2,h) -3*h*dnorm(h) )*exkurt520X = X + (1/6)*( derIpower(3,h) - 3*derIpower(1,h) )*skew521X = X + (1/72)*( derIpower(6,h) - 15*derIpower(4,h) + 45*derIpower(2,h) + 15*h*dnorm(h) )*skew^2522derE = derE+derh*X # X is a scalar523derE = derE/alpha524derMES = derMES + sqrt(pm2)*derE }else{525derMES = -mu - 0.5*(dpm2/sqrt(pm2))*h - sqrt(pm2)*derh ; }526contrib = as.vector(w)*as.vector(derMES)527return(list( round( MES, precision) , round( contrib , precision ) , round(contrib/MES,precision) ) )528}529530realportES = function(data,w,VaR,precision=4)531{532T = dim(data)[1]533N = dim(data)[2]534c_exceed = 0;535r_exceed = 0;536realizedcontrib = rep(0,N);537for(t in c(1:T) )538{539rt = as.vector(data[t,])540rp = sum(w*rt)541if(rp<=-VaR){542c_exceed = c_exceed + 1;543r_exceed = r_exceed + rp;544for( i in c(1:N) ){545realizedcontrib[i] =realizedcontrib[i] + w[i]*rt[i] }546}547}548realizedcontrib=as.numeric(realizedcontrib)/r_exceed ;549return( list(round(-r_exceed/c_exceed,precision),c_exceed,round(realizedcontrib,precision)) )550}551552realportVaR = function(data,w,alpha,precision=4)553{554portret = c();555T = dim(data)[1]556N = dim(data)[2]557for( t in c(1:T) ){558portret = c(portret,sum(w*as.numeric(data[t,])))559}560VaR = sort(portret)[floor(alpha*T)]561return(-VaR)562}563564