
!pip install h5py

Collecting h5py
  Downloading h5py-3.4.0.tar.gz (371 kB)
     |████████████████████████████████| 371 kB 572 kB/s eta 0:00:01
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
  Installing backend dependencies ... done
    Preparing wheel metadata ... done
Requirement already satisfied: numpy>=1.17.5 in /Users/nam/miniforge3/lib/python3.8/site-packages (from h5py) (1.20.2)
Building wheels for collected packages: h5py
  Building wheel for h5py (PEP 517) ... error
  ERROR: Command errored out with exit status 1:
   command: /Users/nam/miniforge3/bin/python3.8 /Users/nam/miniforge3/lib/python3.8/site-packages/pip/_vendor/pep517/in_process/_in_process.py build_wheel /var/folders/xk/r91vl3h16k5c5ct0q19s9mxw0000gn/T/tmpy2g5qnvg
       cwd: /private/var/folders/xk/r91vl3h16k5c5ct0q19s9mxw0000gn/T/pip-install-h9n4u22o/h5py_390d17ac624646ce90757d3ebd6cd735
  Complete output (71 lines):
  running bdist_wheel
  running build
  running build_py
  creating build
  creating build/lib.macosx-11.0-arm64-3.8
  creating build/lib.macosx-11.0-arm64-3.8/h5py
  copying h5py/h5py_warnings.py -> build/lib.macosx-11.0-arm64-3.8/h5py
  copying h5py/version.py -> build/lib.macosx-11.0-arm64-3.8/h5py
  copying h5py/__init__.py -> build/lib.macosx-11.0-arm64-3.8/h5py
  copying h5py/ipy_completer.py -> build/lib.macosx-11.0-arm64-3.8/h5py
  creating build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/files.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/compat.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/__init__.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/selections.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/dataset.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/vds.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/selections2.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/group.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/datatype.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/attrs.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/dims.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/base.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  copying h5py/_hl/filters.py -> build/lib.macosx-11.0-arm64-3.8/h5py/_hl
  creating build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dimension_scales.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_attribute_create.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_file_image.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/conftest.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5d_direct_chunk.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5f.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dataset_getitem.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_group.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_errors.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dataset_swmr.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_slicing.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5pl.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_attrs.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/__init__.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_attrs_data.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5t.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_big_endian_file.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5p.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dims_dimensionproxy.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5o.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_datatype.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/common.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dataset.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_file.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_selections.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_dtype.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_h5.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_file2.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_completions.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_filters.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_base.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  copying h5py/tests/test_objects.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests
  creating build/lib.macosx-11.0-arm64-3.8/h5py/tests/data_files
  copying h5py/tests/data_files/__init__.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/data_files
  creating build/lib.macosx-11.0-arm64-3.8/h5py/tests/test_vds
  copying h5py/tests/test_vds/test_highlevel_vds.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/test_vds
  copying h5py/tests/test_vds/test_virtual_source.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/test_vds
  copying h5py/tests/test_vds/__init__.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/test_vds
  copying h5py/tests/test_vds/test_lowlevel_vds.py -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/test_vds
  copying h5py/tests/data_files/vlen_string_s390x.h5 -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/data_files
  copying h5py/tests/data_files/vlen_string_dset_utc.h5 -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/data_files
  copying h5py/tests/data_files/vlen_string_dset.h5 -> build/lib.macosx-11.0-arm64-3.8/h5py/tests/data_files
  running build_ext
  Loading library to get build settings and version: libhdf5.dylib
  error: Unable to load dependency HDF5, make sure HDF5 is installed properly
  error: dlopen(libhdf5.dylib, 6): image not found
  ----------------------------------------
  ERROR: Failed building wheel for h5py
Failed to build h5py
ERROR: Could not build wheels for h5py which use PEP 517 and cannot be installed directly


import numpy as np
import pandas as pd

#this will be our HDF5 file 
filename = 'data/test.h5'


df = pd.DataFrame(np.arange(10).reshape((5,2)), 
                      columns=['A', 'B'])
print(df)


# Save to HDF5
df.to_hdf(filename, 'data', mode='w', format='table')

del df    # allow df to be garbage collected


# Append more data
df2 = pd.DataFrame(np.arange(10).reshape((5,2)) * 10, 
                   columns=['A', 'B'])


# This is the important bit! Remove `append` to see the difference 
df2.to_hdf(filename, 'data', append=True)

del df2


print(pd.read_hdf(filename, 'data'))

Issues of Scale¶

HDF5 - The Why and How¶