20200507_hopevfear_research.ipynb

Kernel: Python 3

---

S&P500 Hope vs. Fear

As the world attempts to deal with the effects of COVID-19, the market has been filled with news of every emotion. We measure two of these emotions “hope” and “fear” in news specifically about the S&P500 companies in early 2020.

This research notebook goes through the steps to plot the instances of fearful and hopeful words in the news related to S&P500 companies over the last several months. We used QuantConnect's alternative Data source TiingoNews which crawls the web for thousands of news articles a day. The data is linked to ticker symbols, allowing users to use the news about specific assets across time to determine trading strategies.

Powerfully, we can see news of hope battling it out with fear over the last months but ultimately coming out ahead. The percentage scale used for the y-axis represents the percentage of fearful words out of all mentions of fearful and hopeful words and the percentage of hopeful words out of all mentions of fearful and hopeful words.

from QuantConnect.Data.Custom.Tiingo import *
import pandas as pd
import numpy as np
import scipy

from datetime import datetime, timedelta
qb = QuantBook()

# Assets to analyze for sentiment
sp500 = ["MMM","ABT","ABBV","ABMD","ACN","ATVI","ADBE","AMD","AAP","AES","AFL","A","APD","AKAM","ALK","ALB","ARE","ALXN","ALGN","ALLE","AGN","ADS","LNT","ALL","GOOGL","GOOG","MO","AMZN","AMCR","AEE","AAL","AEP","AXP","AIG","AMT","AWK","AMP","ABC","AME","AMGN","APH","ADI","ANSS","ANTM","AON","AOS","APA","AIV","AAPL","AMAT","APTV","ADM","ANET","AJG","AIZ","T","ATO","ADSK","ADP","AZO","AVB","AVY","BKR","BLL","BAC","BK","BAX","BDX","BRK.B","BBY","BIIB","BLK","BA","BKNG","BWA","BXP","BSX","BMY","AVGO","BR","BF.B","CHRW","COG","CDNS","CPB","COF","CPRI","CAH","KMX","CCL","CARR","CAT","CBOE","CBRE","CDW","CE","CNC","CNP","CTL","CERN","CF","SCHW","CHTR","CVX","CMG","CB","CHD","CI","CINF","CTAS","CSCO","C","CFG","CTXS","CLX","CME","CMS","KO","CTSH","CL","CMCSA","CMA","CAG","CXO","COP","ED","STZ","COO","CPRT","GLW","CTVA","COST","COTY","CCI","CSX","CMI","CVS","DHI","DHR","DRI","DVA","DE","DAL","XRAY","DVN","FANG","DLR","DFS","DISCA","DISCK","DISH","DG","DLTR","D","DOV","DOW","DTE","DUK","DRE","DD","DXC","ETFC","EMN","ETN","EBAY","ECL","EIX","EW","EA","EMR","ETR","EOG","EFX","EQIX","EQR","ESS","EL","EVRG","ES","RE","EXC","EXPE","EXPD","EXR","XOM","FFIV","FB","FAST","FRT","FDX","FIS","FITB","FE","FRC","FISV","FLT","FLIR","FLS","FMC","F","FTNT","FTV","FBHS","FOXA","FOX","BEN","FCX","GPS","GRMN","IT","GD","GE","GIS","GM","GPC","GILD","GL","GPN","GS","GWW","HRB","HAL","HBI","HOG","HIG","HAS","HCA","PEAK","HP","HSIC","HSY","HES","HPE","HLT","HFC","HOLX","HD","HON","HRL","HST","HWM","HPQ","HUM","HBAN","HII","IEX","IDXX","INFO","ITW","ILMN","INCY","IR","INTC","ICE","IBM","IP","IPG","IFF","INTU","ISRG","IVZ","IPGP","IQV","IRM","JKHY","J","JBHT","SJM","JNJ","JCI","JPM","JNPR","KSU","K","KEY","KEYS","KMB","KIM","KMI","KLAC","KSS","KHC","KR","LB","LHX","LH","LRCX","LW","LVS","LEG","LDOS","LEN","LLY","LNC","LIN","LYV","LKQ","LMT","L","LOW","LYB","MTB","MRO","MPC","MKTX","MAR","MMC","MLM","MAS","MA","MKC","MXIM","MCD","MCK","MDT","MRK","MET","MTD","MGM","MCHP","MU","MSFT","MAA","MHK","TAP","MDLZ","MNST","MCO","MS","MOS","MSI","MSCI","MYL","NDAQ","NOV","NTAP","NFLX","NWL","NEM","NWSA","NWS","NEE","NLSN","NKE","NI","NBL","JWN","NSC","NTRS","NOC","NLOK","NCLH","NRG","NUE","NVDA","NVR","ORLY","OXY","ODFL","OMC","OKE","ORCL","OTIS","PCAR","PKG","PH","PAYX","PAYC","PYPL","PNR","PBCT","PEP","PKI","PRGO","PFE","PM","PSX","PNW","PXD","PNC","PPG","PPL","PFG","PG","PGR","PLD","PRU","PEG","PSA","PHM","PVH","QRVO","PWR","QCOM","DGX","RL","RJF","RTX","O","REG","REGN","RF","RSG","RMD","RHI","ROK","ROL","ROP","ROST","RCL","SPGI","CRM","SBAC","SLB","STX","SEE","SRE","NOW","SHW","SPG","SWKS","SLG","SNA","SO","LUV","SWK","SBUX","STT","STE","SYK","SIVB","SYF","SNPS","SYY","TMUS","TROW","TTWO","TPR","TGT","TEL","FTI","TFX","TXN","TXT","TMO","TIF","TJX","TSCO","TT","TDG","TRV","TFC","TWTR","TSN","UDR","ULTA","USB","UAA","UA","UNP","UAL","UNH","UPS","URI","UHS","UNM","VFC","VLO","VAR","VTR","VRSN","VRSK","VZ","VRTX","VIAC","V","VNO","VMC","WRB","WAB","WMT","WBA","DIS","WM","WAT","WEC","WFC","WELL","WDC","WU","WRK","WY","WHR","WMB","WLTW","WYNN","XEL","XRX","XLNX","XYL","YUM","ZBRA","ZBH","ZION","ZTS"]

# Words for sentiment scoring  
fearful_words = ["hopeless", "death", "fear", "fearful", "bearish", "panic", "loss", "scared", "crisis", "emergency", "pessimistic"]
hopeful_words = ["hope", "hopeful", "uplifting", "optimism", "hoping", "hopes", "bullish", "bright", "improve", "optimistic", "growth"]

def count_instances(master, article):
    count = 0
    for w in article:
        if w in master:
            count = count + 1
    return count

# Extract history and get word count of fearful and hopeful words, per day, per ticker
hopeful_word_sums = []
fearful_word_sums = []
tickers_not_in_data = []
# Note: This takes 45 minutes to run
for ticker in sp500:
    symbol = qb.AddEquity(ticker).Symbol
    news = qb.AddData(TiingoNews, symbol).Symbol
    history = qb.History(TiingoNews, news, timedelta(days=127), Resolution.Daily)
    try:
        description = history.reset_index(level=0)['description']
        hopeful_word_count = description.apply(lambda x: count_instances(hopeful_words, x.split(' ')))
        hopeful_word_count_daily = hopeful_word_count.resample('D').sum()
        hopeful_word_sums.append(hopeful_word_count_daily)
        fearful_word_count = description.apply(lambda x: count_instances(fearful_words, x.split(' ')))
        fearful_word_count_daily = fearful_word_count.resample('D').sum()
        fearful_word_sums.append(fearful_word_count_daily)    
    except:
        tickers_not_in_data.append(ticker)
        continue

import pickle 
# Speed up iterative analysis by pickling the history call
hopeful_word_sums_pickle = pickle.dumps(hopeful_word_sums)
fearful_word_sums_pickle = pickle.dumps(fearful_word_sums)
tickers_not_in_data_pickle = pickle.dumps(tickers_not_in_data)

# Load pickled history to avoid calling data again
# hopeful_word_sums = pickle.loads(hopeful_word_sums_pickle) 
# fearful_word_sums = pickle.Loads(fearful_word_sums_pickle)
# tickers_not_in_data = pickle.Loads(tickers_not_in_data_pickle)

# Get the sum of hopeful words per day for all tickers
hopeful_data = []
for item in hopeful_word_sums:
    hopeful_data.append(item)
hope_data = pd.concat(hopeful_data)
hope = hope_data.groupby(level=0, axis=0).sum()

fearful_data = []
for item in fearful_word_sums:
    fearful_data.append(item)
fear_data = pd.concat(fearful_data)
fear = fear_data.groupby(level=0, axis=0).sum()

# Rename the columns to "hope" & "fear" because they're both currently called "description" 
hope = hope.rename("hope")
fear = fear.rename("fear")
df = pd.concat([hope, fear], axis=1)
time = df.index.to_series()
df = pd.concat([time, hope, fear], axis=1)

# Transform into percentage from absolute numbers
df['total'] = df['hope'] + df['fear']
df['pct_hope'] = (df['hope']/ df['total']).round(2)
df['pct_fear'] = (df['fear']/ df['total']).round(2)
# Drop incomplete data
df = df[:-2]

import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot

trace1 = go.Scatter(x=df.time,
                    y=df.pct_hope,
                    name = "Hope in the News",
                    line = {'color':'rgb(138, 185, 211)'})

trace2 = go.Scatter(x=df.time,
                    y=df.pct_fear,
                    name = "Fear in the News",
                    line = {'color': 'rgb(237, 193, 218)'})

layout = {'title':'S&P500 Hope vs. Fear', 'plot_bgcolor' : 'rgb(248, 247, 247)' , 'yaxis': { 'tickformat': ',.0%', 'range': [0,1]}}

fig = dict(data=[trace1, trace2], layout=layout)     

iplot(fig)