Sharedpmarket / Arctic.ipynbOpen in CoCalc
Arctic import error
# !pip3 install git+https://github.com/manahl/arctic.git
# Start mongo in terminal: mongod --dbpath /home/user/<mongo_data>/
    
from arctic import Arctic
import quandl
import pandas as pd
# Connect to Local MONGODB
store = Arctic('localhost')
store
<Arctic at 0x7fee5729de10, connected to MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True, sockettimeoutms=600000, connecttimeoutms=2000, serverselectiontimeoutms=30000, maxpoolsize=4)>

# Create the library - defaults to VersionStore
# store.initialize_library('NASDAQ')
store.initialize_library('test')

# Access the library
# library = store['NASDAQ']
testlib = store['test']

# Load some data - maybe from Quandl
# aapl = quandl.get("WIKI/AAPL", authtoken="your token here")
testdata = pd.DataFrame({
        'a': [1,2,3,4,5],
        'b': ['a', 'a', 'b', 'a', 'c']
    })

quandlAPIkey = 'rkeazqH5yf2RspNwACei'

# Store the data in the library
# library.write('AAPL', aapl, metadata={'source': 'Quandl'})
testlib.write('data1', testdata, metadata={'source': 'completely made up'})


# Load some data - maybe from Quandl
aapl = quandl.get("WIKI/AAPL", authtoken=quandlAPIkey)

# Store the data in the library
testlib.write("AAPL", aapl, metadata={'source': 'Quandl'})

# Reading the data
item1 = testlib.read('AAPL')
aapl = item1.data
metadata = item1.metadata



# Reading the data
#item = library.read('AAPL')
item2 = testlib.read('data1')
data1 = item2.data
metadata = item2.metadata



Library created, but couldn't enable sharding: no such command: 'enablesharding', bad cmd: '{ enablesharding: "arctic", lsid: { id: UUID("062b9ce4-f5be-4366-857a-d6376a07b058") }, $readPreference: { mode: "secondaryPreferred" }, $db: "admin" }'. This is OK if you're not 'admin'
# What symbols (keys) are stored in the library
testlib.list_symbols()
['AAPL', 'TESTDATA', 'data1', 'futuresfills']
# Additional stuff
# Find available versions of a symbol
list(testlib.list_versions('data1'))
[{'date': datetime.datetime(2018, 2, 5, 3, 13, 34, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')), 'deleted': False, 'snapshots': [], 'symbol': 'data1', 'version': 15}, {'date': datetime.datetime(2018, 2, 5, 3, 10, 39, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')), 'deleted': False, 'snapshots': [], 'symbol': 'data1', 'version': 14}, {'date': datetime.datetime(2018, 2, 5, 0, 25, 35, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')), 'deleted': False, 'snapshots': [], 'symbol': 'data1', 'version': 13}, {'date': datetime.datetime(2018, 1, 27, 8, 18, 48, tzinfo=tzfile('/usr/share/zoneinfo/Etc/UTC')), 'deleted': False, 'snapshots': ['snapshot_name'], 'symbol': 'data1', 'version': 7}]

# What symbols (keys) are stored in the library
testlib.list_symbols()
['AAPL', 'TESTDATA', 'data1', 'futuresfills']
testlib.delete('data2')
# Version
testlib.read('TESTDATA').version
1
aapl.iloc[:2]
Open High Low Close Volume Ex-Dividend Split Ratio Adj. Open Adj. High Adj. Low Adj. Close Adj. Volume
Date
1980-12-12 28.75 28.87 28.75 28.75 2093900.0 0.0 1.0 0.422706 0.424470 0.422706 0.422706 117258400.0
1980-12-15 27.38 27.38 27.25 27.25 785200.0 0.0 1.0 0.402563 0.402563 0.400652 0.400652 43971200.0
testlib.read('AAPL').data[1:4]
Open High Low Close Volume Ex-Dividend Split Ratio Adj. Open Adj. High Adj. Low Adj. Close Adj. Volume
Date
1980-12-15 27.38 27.38 27.25 27.25 785200.0 0.0 1.0 0.402563 0.402563 0.400652 0.400652 43971200.0
1980-12-16 25.37 25.37 25.25 25.25 472000.0 0.0 1.0 0.373010 0.373010 0.371246 0.371246 26432000.0
1980-12-17 25.87 26.00 25.87 25.87 385900.0 0.0 1.0 0.380362 0.382273 0.380362 0.380362 21610400.0
testlib.list_symbols()
['AAPL', 'TESTDATA', 'data1', 'futuresfills']
testlib.write('AAPL', aapl.iloc[:2])
VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=50,metadata=None
testlib.read('AAPL').data
Open High Low Close Volume Ex-Dividend Split Ratio Adj. Open Adj. High Adj. Low Adj. Close Adj. Volume
Date
1980-12-12 28.75 28.87 28.75 28.75 2093900.0 0.0 1.0 0.422706 0.424470 0.422706 0.422706 117258400.0
1980-12-15 27.38 27.38 27.25 27.25 785200.0 0.0 1.0 0.402563 0.402563 0.400652 0.400652 43971200.0
testlib.write('AAPL', aapl.iloc[1:5])
VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=51,metadata=None
testlib.read('AAPL').data
Open High Low Close Volume Ex-Dividend Split Ratio Adj. Open Adj. High Adj. Low Adj. Close Adj. Volume
Date
1980-12-15 27.38 27.38 27.25 27.25 785200.0 0.0 1.0 0.402563 0.402563 0.400652 0.400652 43971200.0
1980-12-16 25.37 25.37 25.25 25.25 472000.0 0.0 1.0 0.373010 0.373010 0.371246 0.371246 26432000.0
1980-12-17 25.87 26.00 25.87 25.87 385900.0 0.0 1.0 0.380362 0.382273 0.380362 0.380362 21610400.0
1980-12-18 26.63 26.75 26.63 26.63 327900.0 0.0 1.0 0.391536 0.393300 0.391536 0.391536 18362400.0
# Write the same data + one day
testlib.write('AAPL', aapl.iloc[1:6])
VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=52,metadata=None
# Check the data
testlib.read('AAPL').data
Open High Low Close Volume Ex-Dividend Split Ratio Adj. Open Adj. High Adj. Low Adj. Close Adj. Volume
Date
1980-12-15 27.38 27.38 27.25 27.25 785200.0 0.0 1.0 0.402563 0.402563 0.400652 0.400652 43971200.0
1980-12-16 25.37 25.37 25.25 25.25 472000.0 0.0 1.0 0.373010 0.373010 0.371246 0.371246 26432000.0
1980-12-17 25.87 26.00 25.87 25.87 385900.0 0.0 1.0 0.380362 0.382273 0.380362 0.380362 21610400.0
1980-12-18 26.63 26.75 26.63 26.63 327900.0 0.0 1.0 0.391536 0.393300 0.391536 0.391536 18362400.0
1980-12-19 28.25 28.38 28.25 28.25 217100.0 0.0 1.0 0.415355 0.417266 0.415355 0.415355 12157600.0
aapl_melted  = pd.melt(aapl.reset_index(), id_vars=['Date'], value_vars=['Open', 'High', 'Low','Close'])
aapl_melted.head()

Date variable value
0 1980-12-12 Open 28.75
1 1980-12-15 Open 27.38
2 1980-12-16 Open 25.37
3 1980-12-17 Open 25.87
4 1980-12-18 Open 26.63
# Add some one one (miss out a bunch of columns)
testlib.write('AAPL', aapl.iloc[5:7,:4])
VersionedItem(symbol=AAPL,library=arctic.test,data=<class 'NoneType'>,version=53,metadata=None
testlib.read('AAPL').data
Open High Low Close
Date
1980-12-19 28.25 28.38 28.25 28.25
1980-12-22 29.63 29.75 29.63 29.63
allFills_futures = pd.read_csv('/home/user/allFills_futures.csv')


allFills_futures.head()
OrderNumber bbgTicker Exchange buySell TraderName ExecInstruction Broker ExecFilledAmount dateTime ExecTime ... ExecAsOfTime volume price ExecAvgPrice bbgMktID ExecType ExecSeqNumber ExecPrevSeqNumber bbgCode timeZone
0 3435484.0 CLU5 NYM 1 ATOWERS2 MACD 67.0 736161.196748 0.196748 ... 0.196528 67.0 52.449552 52.449552 Comdty FILL 3.0 0.0 CLU5 Comdty Australia/Brisbane
1 3435485.0 W H6 CBT -1 ATOWERS2 MACD 116.0 736161.188646 0.188646 ... 0.187500 116.0 577.960938 577.960938 Comdty FILL 3.0 0.0 W H6 Comdty Australia/Brisbane
2 3435486.0 CLM6 NYM 1 ATOWERS2 MACD 40.0 736161.196956 0.196956 ... 0.196528 40.0 56.004750 56.004750 Comdty FILL 3.0 0.0 CLM6 Comdty Australia/Brisbane
3 3435487.0 CLZ5 NYM -1 ATOWERS2 MACD 177.0 736161.197187 0.197187 ... 0.196528 177.0 53.831130 53.831130 Comdty FILL 3.0 0.0 CLZ5 Comdty Australia/Brisbane
4 3435488.0 LCG6 CME -1 ATOWERS2 MACD 51.0 736161.165370 0.165370 ... 0.164583 51.0 152.856863 152.856863 Comdty FILL 3.0 0.0 LCG6 Comdty Australia/Brisbane

5 rows × 21 columns

# Write the same data + one day
testlib.write('futuresfills', allFills_futures)
VersionedItem(symbol=futuresfills,library=arctic.test,data=<class 'NoneType'>,version=3,metadata=None
z = testlib.read('futuresfills')
z.data.size()


head()
--------------------------------------------------------------------------- TypeError Traceback (most recent call last) <ipython-input-27-e0abc38ad668> in <module>() 1 z = testlib.read('futuresfills') ----> 2 z.data.size() TypeError: 'numpy.int64' object is not callable
1+1
whos
Variable Type Data/Info --------------------------------------------- Arctic type <class 'arctic.arctic.Arctic'> aapl DataFrame Open <...>n[9364 rows x 12 columns] aapl_melted DataFrame Date variable<...>n[37456 rows x 3 columns] allFills_futures DataFrame OrderNumber bbgTi<...>165760 rows x 21 columns] data1 DataFrame a b\nindex \<...>n3 4 a\n4 5 c item1 VersionedItem VersionedItem(symbol=AAPL<...>data={'source': 'Quandl'} item2 VersionedItem VersionedItem(symbol=data<...>e': 'completely made up'} metadata dict n=1 pd module <module 'pandas' from '/u<...>ages/pandas/__init__.py'> quandl module <module 'quandl' from '/u<...>ages/quandl/__init__.py'> quandlAPIkey str rkeazqH5yf2RspNwACei store Arctic <Arctic at 0x7fee5729de10<...>ms=30000, maxpoolsize=4)> testdata DataFrame a b\n0 1 a\n1 2 a<...>2 3 b\n3 4 a\n4 5 c testlib VersionStore <VersionStore at 0x7fee55<...>ms=30000, maxpoolsize=4)> z VersionedItem VersionedItem(symbol=futu<...>>,version=3,metadata=None