Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
codebasics
GitHub Repository: codebasics/deep-learning-keras-tf-tutorial
 Path: blob/master/44_tf_data_pipeline/tf_data_pipeline.ipynb
1141 views
Kernel: Python 3
import tensorflow as tf

Create tf dataset from a list

daily_sales_numbers = [21, 22, -108, 31, -1, 32, 34,31]

tf_dataset = tf.data.Dataset.from_tensor_slices(daily_sales_numbers)
tf_dataset
<TensorSliceDataset shapes: (), types: tf.int32>

Iterate through tf dataset

for sales in tf_dataset:
    print(sales.numpy())
21 22 -108 31 -1 32 34 31

Iterate through elements as numpy elements

for sales in tf_dataset.as_numpy_iterator():
    print(sales)
21 22 -108 31 -1 32 34 31

Iterate through first n elements in tf dataset

for sales in tf_dataset.take(3):
    print(sales.numpy())
21 22 -108

Filter sales numbers that are < 0

tf_dataset = tf_dataset.filter(lambda x: x>0)
for sales in tf_dataset.as_numpy_iterator():
    print(sales)
21 22 31 32 34 31

Convert sales numbers from USA dollars ($) to Indian Rupees (INR) Assuming 1->72 conversation rate

tf_dataset = tf_dataset.map(lambda x: x*72)
for sales in tf_dataset.as_numpy_iterator():
    print(sales)
1512 1584 2232 2304 2448 2232

Shuffe

tf_dataset = tf_dataset.shuffle(2)
for sales in tf_dataset.as_numpy_iterator():
    print(sales)
1512 2232 2304 2448 2232 1584

Batching

for sales_batch in tf_dataset.batch(2):
    print(sales_batch.numpy())
[1584 2232] [2304 2448] [2232 1512]

Perform all of the above operations in one shot

tf_dataset = tf.data.Dataset.from_tensor_slices(daily_sales_numbers)

tf_dataset = tf_dataset.filter(lambda x: x>0).map(lambda y: y*72).shuffle(2).batch(2)
for sales in tf_dataset.as_numpy_iterator():
    print(sales)
[1512 2232] [1584 2448] [2304 2232]

Images

images_ds = tf.data.Dataset.list_files('images/*/*', shuffle=False)

image_count = len(images_ds)
image_count
130
type(images_ds)
tensorflow.python.data.ops.dataset_ops.TensorSliceDataset
for file in images_ds.take(3):
    print(file.numpy())
b'images\\cat\\20 Reasons Why Cats Make the Best Pets....jpg' b'images\\cat\\7 Foods Your Cat Can_t Eat.jpg' b'images\\cat\\A cat appears to have caught the....jpg' 
images_ds = images_ds.shuffle(200)
for file in images_ds.take(3):
    print(file.numpy())
b'images\\dog\\The US Army is testing augmented....jpg' b'images\\dog\\Subaru Shows Love for Dogs Through the....jpg' b'images\\cat\\Reality check_ Can cat poop cause....jpg' 
class_names = ["cat","dog"]

train_size = int(image_count*0.8)
train_ds = images_ds.take(train_size)
test_ds = images_ds.skip(train_size)

len(train_ds)
104
len(test_ds)
26
def get_label(file_path):
    import os
    parts = tf.strings.split(file_path, os.path.sep)
    return parts[-2]

get_label("images\\dog\\20 Reasons Why Cats Make the Best Pets....jpg")
<tf.Tensor: shape=(), dtype=string, numpy=b'dog'>
def process_image(file_path):
    label = get_label(file_path)
    img = tf.io.read_file(file_path) # load the raw data from the file as a string
    img = tf.image.decode_jpeg(img)
    img = tf.image.resize(img, [128, 128])
    return img, label

img, label = process_image("images\\cat\\20 Reasons Why Cats Make the Best Pets....jpg")
img.numpy()[:2]
array([[[155. , 186. , 215. ], [156. , 187. , 216. ], [158. , 189. , 218. ], [160.0039 , 189.0039 , 219.0039 ], [161.0039 , 190.0039 , 220.0039 ], [162. , 191. , 221. ], [166. , 193. , 222. ], [167. , 194. , 223. ], [168. , 195. , 224. ], [169. , 196. , 225. ], [170. , 197. , 224. ], [170.0039 , 197.0039 , 224.0039 ], [172. , 199. , 226. ], [173.0039 , 199.0039 , 224.0039 ], [174.0039 , 200.0039 , 225.0039 ], [175.0039 , 201.0039 , 226.0039 ], [176. , 202. , 227. ], [177.0039 , 203.0039 , 228.0039 ], [177.0039 , 203.0039 , 228.0039 ], [179.0039 , 203.0039 , 227.0039 ], [180.0039 , 204.0039 , 228.0039 ], [180.0039 , 204.0039 , 228.0039 ], [181.0039 , 206.0039 , 228.0039 ], [182.0039 , 207.0039 , 229.0039 ], [184. , 209. , 231. ], [184. , 209. , 231. ], [184. , 209. , 231. ], [184. , 209. , 231. ], [182.9961 , 207.9961 , 229.9961 ], [174.9961 , 201.9961 , 222.9961 ], [168.9961 , 195.9961 , 216.9961 ], [158.99219 , 185.99219 , 206.99219 ], [143.9961 , 168.9961 , 190.9961 ], [131.9961 , 148.98828 , 168.98438 ], [115.98828 , 126.97266 , 132.97266 ], [106.98047 , 102.984375, 99.97656 ], [ 98.99219 , 95.984375, 90.98047 ], [ 97.99219 , 95.99219 , 83.98828 ], [ 97.99219 , 91.99219 , 77.99219 ], [ 98.99609 , 91.99609 , 75.99609 ], [ 99. , 92. , 74. ], [102. , 95. , 77. ], [112.99609 , 103.99609 , 86.99609 ], [125.99609 , 113.99609 , 99.99609 ], [133.99219 , 119.99219 , 108.99219 ], [139.99219 , 127.99219 , 113.99219 ], [143.9961 , 131.9961 , 117.99609 ], [143.9961 , 131.9961 , 117.99609 ], [142. , 128. , 119. ], [135.0039 , 122.00391 , 113.00391 ], [122.00391 , 114.00391 , 103.00391 ], [112.00391 , 103.00391 , 94.00391 ], [102.00391 , 95.00391 , 87.00391 ], [ 91. , 86. , 80. ], [ 83. , 78. , 72. ], [ 75.00391 , 70.00391 , 64.00391 ], [ 69. , 64. , 58. ], [ 61. , 58. , 51. ], [ 59. , 54. , 51. ], [ 61.996094, 52.996094, 55.996094], [ 62.996094, 49.996094, 58.996094], [ 61. , 49. , 53. ], [ 60. , 48. , 52. ], [ 59. , 47. , 51. ], [ 57. , 45. , 49. ], [ 55. , 45. , 44. ], [ 50.996094, 43.996094, 37.996094], [ 45.996094, 42.996094, 35.996094], [ 45.996094, 42.996094, 33.996094], [ 45. , 43. , 31. ], [ 46. , 44. , 31. ], [ 48. , 46. , 33. ], [ 48. , 46. , 33. ], [ 49.003906, 47.003906, 34.003906], [ 57.003906, 55.003906, 42.003906], [ 63.007812, 61.007812, 48.007812], [ 71.01172 , 69.01172 , 56.01172 ], [ 75.01172 , 73.01172 , 61.01172 ], [ 77.00781 , 75.00781 , 63.007812], [ 78.00391 , 76.00391 , 64.00391 ], [ 77.00391 , 75.00391 , 63.003906], [ 74.00781 , 72.00781 , 59.007812], [ 73.00391 , 71.00391 , 56.003906], [ 77.00391 , 71.00391 , 55.003906], [ 82. , 76. , 60. ], [ 85.00391 , 79.00391 , 63.003906], [ 88. , 82. , 70. ], [ 88. , 81. , 71. ], [ 85. , 82. , 75. ], [ 83. , 80. , 73. ], [ 77.99609 , 76.99609 , 71.99609 ], [ 74.99609 , 73.99609 , 69.99609 ], [ 72. , 71. , 69. ], [ 70. , 69. , 65. ], [ 69. , 68. , 64. ], [ 69. , 68. , 64. ], [ 70. , 69. , 65. ], [ 71. , 70. , 66. ], [ 70. , 69. , 65. ], [ 69.99609 , 68.99609 , 66.99609 ], [ 68. , 67. , 65. ], [ 66. , 65. , 63. ], [ 64. , 63. , 59. ], [ 63. , 62. , 58. ], [ 63. , 62. , 58. ], [ 65. , 64. , 60. ], [ 69.00391 , 70.00391 , 65.00391 ], [ 78.00391 , 78.00391 , 69.99609 ], [ 89.00781 , 87.00781 , 75.00781 ], [103.00391 , 97.00391 , 83.00391 ], [113.01172 , 106.01172 , 90.01172 ], [123.00781 , 114.00781 , 97.00781 ], [135.01172 , 124.01172 , 102.01172 ], [144.00781 , 133.00781 , 111.00781 ], [149.0039 , 138.0039 , 116.00391 ], [149.00781 , 137.00781 , 111.00781 ], [147.0039 , 135.0039 , 111.00391 ], [145.0039 , 133.0039 , 111.00391 ], [140.0039 , 127.00391 , 108.00391 ], [130.0039 , 119.00391 , 101.00391 ], [118.00391 , 108.00391 , 96.00391 ], [108. , 100. , 89. ], [ 99. , 93. , 79. ], [ 96.99609 , 90.99609 , 76.99609 ], [ 94. , 88. , 74. ], [ 90.99609 , 84.99609 , 72.99609 ], [ 92. , 86. , 74. ], [ 93. , 87. , 75. ]], [[156.01172 , 187.01172 , 216.01172 ], [157.02344 , 188.02344 , 217.02344 ], [159.01172 , 190.01172 , 219.01172 ], [162. , 191. , 221. ], [163. , 192. , 222. ], [164.01172 , 193.01172 , 223.01172 ], [167.01172 , 194.01172 , 223.01172 ], [168.01172 , 195.01172 , 224.01172 ], [169. , 196. , 225. ], [170.01172 , 197.01172 , 226.01172 ], [171.01172 , 198.01172 , 225.01172 ], [172. , 199. , 226. ], [173. , 200. , 227. ], [175. , 201. , 226. ], [176. , 202. , 227. ], [177. , 203. , 228. ], [177. , 203. , 228. ], [178.98828 , 204.98828 , 229.98828 ], [179. , 205. , 230. ], [181. , 205. , 229. ], [182. , 206. , 230. ], [182. , 206. , 230. ], [183. , 208. , 230. ], [184. , 209. , 231. ], [184.98828 , 209.98828 , 231.98828 ], [184.97656 , 209.97656 , 231.97656 ], [186. , 207.98828 , 230.96484 ], [185. , 206.98828 , 229.96484 ], [180.01172 , 202. , 224.97656 ], [173.01172 , 194.98828 , 218.98828 ], [167.01172 , 186.98828 , 210.96484 ], [154.02344 , 175. , 193.97656 ], [138. , 158.97656 , 177.92969 ], [125.98828 , 136.95312 , 142.91797 ], [109. , 105.98828 , 98.96484 ], [101.01172 , 93. , 79.97656 ], [ 99.03516 , 91.02344 , 78. ], [ 99. , 91.97656 , 73.98828 ], [ 98.97656 , 90.97656 , 77.97656 ], [ 99. , 92. , 76. ], [ 99. , 92. , 74. ], [101.96484 , 94.96484 , 76.96484 ], [109.98828 , 100.98828 , 85.98828 ], [120.01172 , 108.01172 , 96.01172 ], [131.01172 , 117.01172 , 108.01172 ], [138.02344 , 126.01172 , 114.046875], [143. , 130.98828 , 119.02344 ], [146.01172 , 134. , 122.03516 ], [144.02344 , 130.02344 , 121.02344 ], [138.98828 , 125.98828 , 116.98828 ], [128.01172 , 120.01172 , 109.01172 ], [115.02344 , 106.02344 , 97.02344 ], [105.01172 , 98.01172 , 90.01172 ], [ 92.98828 , 87.98828 , 81.98828 ], [ 82.98828 , 79.98828 , 72.98828 ], [ 76. , 73. , 66. ], [ 68.01172 , 65.01172 , 58.01172 ], [ 61. , 58. , 51. ], [ 59. , 54. , 51. ], [ 60.023438, 51.023438, 54.023438], [ 61.01172 , 48.01172 , 57.01172 ], [ 60. , 48. , 52. ], [ 60.01172 , 48.01172 , 52.01172 ], [ 59. , 47. , 51. ], [ 56. , 46. , 47. ], [ 53.98828 , 45.98828 , 42.98828 ], [ 45.98828 , 41.98828 , 32.98828 ], [ 42.98828 , 40. , 30.964844], [ 42.98828 , 40.98828 , 28.976562], [ 43.98828 , 41.98828 , 28.964844], [ 46. , 44. , 31. ], [ 48.035156, 46.035156, 33.035156], [ 51.046875, 49.046875, 36.046875], [ 58.046875, 56.046875, 43.046875], [ 70.01172 , 66.01172 , 54.01172 ], [ 76.03516 , 72.03516 , 60.035156], [ 84.05859 , 80.05859 , 68.05859 ], [ 87.03516 , 85.03516 , 73.03516 ], [ 85.05859 , 83.05859 , 71.05859 ], [ 86.02344 , 84.02344 , 72.02344 ], [ 83.02344 , 80.02344 , 71.046875], [ 81.01172 , 79. , 67.03516 ], [ 79.02344 , 77.02344 , 64.02344 ], [ 83.01172 , 77.01172 , 63.035156], [ 84.02344 , 78.02344 , 64.046875], [ 88. , 82. , 68.02344 ], [ 89.98828 , 82.98828 , 72.98828 ], [ 90.01172 , 83.01172 , 73.01172 ], [ 85. , 82. , 75. ], [ 83. , 80. , 73. ], [ 76. , 75. , 70. ], [ 73.01172 , 72.01172 , 68.01172 ], [ 71. , 70. , 68. ], [ 70. , 69. , 65. ], [ 69. , 68. , 64. ], [ 69. , 68. , 64. ], [ 69. , 68. , 64. ], [ 69.98828 , 68.98828 , 64.98828 ], [ 69. , 68. , 64. ], [ 67.98828 , 66.98828 , 62.98828 ], [ 66.98828 , 65.98828 , 61.98828 ], [ 64.98828 , 63.98828 , 59.98828 ], [ 63. , 62. , 58. ], [ 62. , 61. , 57. ], [ 63. , 62. , 58. ], [ 65.02344 , 64.02344 , 60.023438], [ 72.03516 , 71.01172 , 65.98828 ], [ 83.02344 , 80.01172 , 70.98828 ], [ 96.03516 , 92.01172 , 79.98828 ], [112.03516 , 106.02344 , 90.02344 ], [127.02344 , 118.03516 , 101.01172 ], [135.0586 , 127.02344 , 108.03516 ], [145.98828 , 134.98828 , 112.98828 ], [150.98828 , 139.98828 , 117.98828 ], [152.01172 , 141.01172 , 119.01172 ], [150.98828 , 139.01172 , 115. ], [144.98828 , 133. , 111. ], [143.98828 , 131.01172 , 112. ], [139.98828 , 128.98828 , 110.98828 ], [130.97656 , 119.01172 , 103.02344 ], [118.98828 , 110.98828 , 99.98828 ], [107.96484 , 100.96484 , 90.96484 ], [ 96.97656 , 92.97656 , 82. ], [ 92.01172 , 88.01172 , 77.03516 ], [ 89.98828 , 85.98828 , 75.01172 ], [ 86.98828 , 82.97656 , 74.01172 ], [ 89.97656 , 85.96484 , 77. ], [ 91. , 86.98828 , 78.02344 ]]], dtype=float32)
train_ds = train_ds.map(process_image)
test_ds = test_ds.map(process_image)

for image, label in train_ds.take(1):
    print("****",image)
    print("****",label)
**** tf.Tensor( [[[ 29.873047 38.558594 42.558594 ] [ 39.064453 44.21875 49.67578 ] [ 37.009766 45.01953 51.259766 ] ... [ 45.779297 23.298828 3.0097656] [ 46.509766 22.773438 3.96875 ] [ 45.439453 20.470703 3.15625 ]] [[ 38.34375 45.10547 50.34375 ] [ 41.353516 47.35742 54.916016 ] [ 41.65625 52.347656 60.9375 ] ... [ 47.820312 23.878906 2.9414062] [ 46.86914 21.789062 2.9335938] [ 44.648438 20.861328 2.8125 ]] [[ 45.498047 52.89453 59.041016 ] [ 48.070312 55.191406 63.191406 ] [ 51.035156 58.746094 64.85742 ] ... [ 49.72461 24.107422 4.732422 ] [ 48.439453 24.001953 3. ] [ 47.716797 23.259766 3.0722656]] ... [[107.46094 75.94922 39.396484 ] [106.0293 75.6582 41.5625 ] [106.25 76.85742 43.61133 ] ... [101.078125 64.11328 26.050781 ] [102.02539 66.02539 24.90039 ] [ 97.71875 64.00195 25.667969 ]] [[101.90625 72.40039 35.43164 ] [104.625 73.92383 40.39258 ] [106.44922 76.86328 45.76758 ] ... [102.48633 66.427734 26.548828 ] [ 99.69531 63.570312 22.77539 ] [ 98.49414 64.875 25.74414 ]] [[100.71484 71.05859 35.316406 ] [101.57422 72.24414 36.54492 ] [105.93359 78.13281 45.01758 ] ... [100.22266 66.13672 27.761719 ] [ 97.60547 64.35547 22.88086 ] [ 97.99219 62.5625 24.492188 ]]], shape=(128, 128, 3), dtype=float32) **** tf.Tensor(b'cat', shape=(), dtype=string) 
def scale(image, label):
    return image/255, label

train_ds = train_ds.map(scale)

for image, label in train_ds.take(5):
    print("****Image: ",image.numpy()[0][0])
    print("****Label: ",label.numpy())
****Image: [0.10965074 0.1645527 0.03097427] ****Label: b'dog' ****Image: [0.60398287 0.6628064 0.6510417 ] ****Label: b'dog' ****Image: [0.14935039 0.19651932 0.14156231] ****Label: b'cat' ****Image: [0.8718137 0.91495097 0.9227941 ] ****Label: b'dog' ****Image: [0.9059021 0.9137452 0.85884327] ****Label: b'dog'