import requests

base_url = "http://api.us.socrata.com/api/catalog/v1?only=datasets&domains=data.cityofnewyork.us"
filename = "row_counts-data.cityofnewyork.us.csv"
r = requests.get(base_url+"&limit=0")
total_count = r.json()['resultSetSize']

print total_count

collection = []
while len(collection) < total_count:
    r = requests.get(base_url+"&limit=100&offset={0}".format(len(collection)))
    collection.extend(r.json()['results'])
    print "{0} datasets collected".format(len(collection))

def get_row_count(domain,fxf):
    r = requests.get("https://{0}/resource/{1}.json?$select=count(*)".format(domain,fxf))
    response = r.json()
    try:
        return response[0]['count']        
    except:
        return 0;
    
csv_file = open(filename, 'w')
csv_file.truncate()
csv_file.write("domain,fx4,row_count\n")
csv_file.close()

# probably hacky but opening and writing each time so that the CSV is saved after every write

for dataset in collection:
    csv_file = open(filename, 'a')
    domain = dataset['metadata']['domain']
    fxf = dataset['resource']['id']
    row_count = get_row_count(domain,fxf)
    line = "{0},{1},{2}".format(domain,fxf,row_count)
    print line
    csv_file.write(line+"\n")
    
csv_file.close()

csv_file.close()