CoCalc -- Arctic.ipynb

Arctic import error

²²⁰ views

Kernel: Python 3 (Ubuntu Linux)

In [4]:

# !pip3 install git+https://github.com/manahl/arctic.git
# Start mongo in terminal: mongod --dbpath /home/user/<mongo_data>/
    
from arctic import Arctic
import quandl
import pandas as pd

In [5]:

# Connect to Local MONGODB
store = Arctic('localhost')
store

Out[5]:

<Arctic at 0x7fee5729de10, connected to MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True, sockettimeoutms=600000, connecttimeoutms=2000, serverselectiontimeoutms=30000, maxpoolsize=4)>

In [6]:


# Create the library - defaults to VersionStore
# store.initialize_library('NASDAQ')
store.initialize_library('test')

# Access the library
# library = store['NASDAQ']
testlib = store['test']

# Load some data - maybe from Quandl
# aapl = quandl.get("WIKI/AAPL", authtoken="your token here")
testdata = pd.DataFrame({
        'a': [1,2,3,4,5],
        'b': ['a', 'a', 'b', 'a', 'c']
    })

quandlAPIkey = 'rkeazqH5yf2RspNwACei'

# Store the data in the library
# library.write('AAPL', aapl, metadata={'source': 'Quandl'})
testlib.write('data1', testdata, metadata={'source': 'completely made up'})


# Load some data - maybe from Quandl
aapl = quandl.get("WIKI/AAPL", authtoken=quandlAPIkey)

# Store the data in the library
testlib.write("AAPL", aapl, metadata={'source': 'Quandl'})

# Reading the data
item1 = testlib.read('AAPL')
aapl = item1.data
metadata = item1.metadata



# Reading the data
#item = library.read('AAPL')
item2 = testlib.read('data1')
data1 = item2.data
metadata = item2.metadata

Out[6]:

Library created, but couldn't enable sharding: no such command: 'enablesharding', bad cmd: '{ enablesharding: "arctic", lsid: { id: UUID("062b9ce4-f5be-4366-857a-d6376a07b058") }, $readPreference: { mode: "secondaryPreferred" }, $db: "admin" }'. This is OK if you're not 'admin'

In [7]:

# What symbols (keys) are stored in the library
testlib.list_symbols()

Out[7]:

['AAPL', 'TESTDATA', 'data1', 'futuresfills']

In [8]:

# Additional stuff
# Find available versions of a symbol
list(testlib.list_versions('data1'))

Out[8]:

[{'date': datetime.datetime(2018, 2, 5, 3, 13, 34, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')),
  'deleted': False,
  'snapshots': [],
  'symbol': 'data1',
  'version': 15},
 {'date': datetime.datetime(2018, 2, 5, 3, 10, 39, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')),
  'deleted': False,
  'snapshots': [],
  'symbol': 'data1',
  'version': 14},
 {'date': datetime.datetime(2018, 2, 5, 0, 25, 35, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')),
  'deleted': False,
  'snapshots': [],
  'symbol': 'data1',
  'version': 13},
 {'date': datetime.datetime(2018, 1, 27, 8, 18, 48, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')),
  'deleted': False,
  'snapshots': ['snapshot_name'],
  'symbol': 'data1',
  'version': 7}]

In [0]:

In [9]:

# What symbols (keys) are stored in the library
testlib.list_symbols()

Out[9]:

['AAPL', 'TESTDATA', 'data1', 'futuresfills']

In [10]:

testlib.delete('data2')

In [11]:

# Version
testlib.read('TESTDATA').version

Out[11]:

1

In [12]:

aapl.iloc[:2]

Out[12]:

In [13]:

testlib.read('AAPL').data[1:4]

Out[13]:

In [14]:

testlib.list_symbols()

Out[14]:

['AAPL', 'TESTDATA', 'data1', 'futuresfills']

In [15]:

testlib.write('AAPL', aapl.iloc[:2])

Out[15]:

VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=50,metadata=None

In [16]:

testlib.read('AAPL').data

Out[16]:

In [17]:

testlib.write('AAPL', aapl.iloc[1:5])

Out[17]:

VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=51,metadata=None

In [18]:

testlib.read('AAPL').data

Out[18]:

In [19]:

# Write the same data + one day
testlib.write('AAPL', aapl.iloc[1:6])

Out[19]:

VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=52,metadata=None

In [20]:

# Check the data
testlib.read('AAPL').data

Out[20]:

In [21]:

aapl_melted  = pd.melt(aapl.reset_index(), id_vars=['Date'], value_vars=['Open', 'High', 'Low','Close'])
aapl_melted.head()

Out[21]:

In [22]:

# Add some one one (miss out a bunch of columns)
testlib.write('AAPL', aapl.iloc[5:7,:4])

Out[22]:

VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=53,metadata=None

In [23]:

testlib.read('AAPL').data

Out[23]:

In [24]:

allFills_futures = pd.read_csv('/home/user/allFills_futures.csv')

In [25]:

allFills_futures.head()

Out[25]:

In [26]:

# Write the same data + one day
testlib.write('futuresfills', allFills_futures)

Out[26]:

VersionedItem(symbol=futuresfills,library=arctic.test,data=<class 'NoneType'>,version=3,metadata=None

In [27]:

z = testlib.read('futuresfills')
z.data.size()


head()

Out[27]:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-27-e0abc38ad668> in <module>()
      1 z = testlib.read('futuresfills')
----> 2 z.data.size()

TypeError: 'numpy.int64' object is not callable

In [0]:

1+1

In [30]:

whos

Out[30]:

Variable           Type             Data/Info
---------------------------------------------
Arctic             type             <class 'arctic.arctic.Arctic'>
aapl               DataFrame                       Open      <...>n[9364 rows x 12 columns]
aapl_melted        DataFrame                    Date variable<...>n[37456 rows x 3 columns]
allFills_futures   DataFrame                OrderNumber bbgTi<...>165760 rows x 21 columns]
data1              DataFrame               a  b\nindex      \<...>n3      4  a\n4      5  c
item1              VersionedItem    VersionedItem(symbol=AAPL<...>data={'source': 'Quandl'}
item2              VersionedItem    VersionedItem(symbol=data<...>e': 'completely made up'}
metadata           dict             n=1
pd                 module           <module 'pandas' from '/u<...>ages/pandas/__init__.py'>
quandl             module           <module 'quandl' from '/u<...>ages/quandl/__init__.py'>
quandlAPIkey       str              rkeazqH5yf2RspNwACei
store              Arctic           <Arctic at 0x7fee5729de10<...>ms=30000, maxpoolsize=4)>
testdata           DataFrame           a  b\n0  1  a\n1  2  a<...>2  3  b\n3  4  a\n4  5  c
testlib            VersionStore     <VersionStore at 0x7fee55<...>ms=30000, maxpoolsize=4)>
z                  VersionedItem    VersionedItem(symbol=futu<...>>,version=3,metadata=None

In [0]:

Product

Resources

Company