CoCalc -- test_ext_func

GitHub Repository: singlestore-labs/singlestoredb-python
Path: blob/main/singlestoredb/tests/test_ext_func_data.py
⁴⁶⁹ views
1
#!/usr/bin/env python
2
# type: ignore
3
"""Test external function data parsing and formatting"""
4
import json
5
import unittest
6

7
import numpy as np
8
import pandas as pd
9
import polars as pl
10
import pyarrow as pa
11
from numpy.testing import assert_array_equal
12
from parameterized import parameterized
13

14
from singlestoredb.functions.ext import json as jsonx
15
from singlestoredb.functions.ext import rowdat_1
16

17

18
TINYINT = 1
19
UNSIGNED_TINYINT = -1
20
SMALLINT = 2
21
UNSIGNED_SMALLINT = -2
22
MEDIUMINT = 9
23
UNSIGNED_MEDIUMINT = -9
24
INT = 3
25
UNSIGNED_INT = -3
26
BIGINT = 8
27
UNSIGNED_BIGINT = -8
28
FLOAT = 4
29
DOUBLE = 5
30
STRING = 254
31
BINARY = -254
32

33
col_spec = [
34
    ('tiny', TINYINT),
35
    ('unsigned_tiny', UNSIGNED_TINYINT),
36
    ('short', SMALLINT),
37
    ('unsigned_short', UNSIGNED_SMALLINT),
38
    ('long', INT),
39
    ('unsigned_long', UNSIGNED_INT),
40
    ('float', FLOAT),
41
    ('double', DOUBLE),
42
    ('longlong', BIGINT),
43
    ('unsigned_longlong', UNSIGNED_BIGINT),
44
    ('int24', MEDIUMINT),
45
    ('unsigned_int24', UNSIGNED_MEDIUMINT),
46
    ('string', STRING),
47
    ('binary', BINARY),
48
]
49

50
col_types = [x[1] for x in col_spec]
51
col_names = [x[0] for x in col_spec]
52

53
numpy_row_ids = np.array([1, 2, 3, 4])
54
numpy_nulls = np.array([False, False, False, True])
55

56
numpy_tiny_arr = np.array([-1, 100, 120, 0], dtype=np.int8)
57
numpy_unsigned_tiny_arr = np.array([1, 100, 130, 0], dtype=np.uint8)
58
numpy_short_arr = np.array([-1, 32700, 254, 0], dtype=np.int16)
59
numpy_unsigned_short_arr = np.array([1, 32800, 254, 0], dtype=np.uint16)
60
numpy_long_arr = np.array([-1, 2147483600, 254, 0], dtype=np.int32)
61
numpy_unsigned_long_arr = np.array([1, 2147483800, 254, 0], dtype=np.uint32)
62
numpy_float_arr = np.array([-1, 100, 3.14159, np.nan], dtype=np.float32)
63
numpy_double_arr = np.array([-1, 100, 3.14159, np.nan], dtype=np.float64)
64
numpy_longlong_arr = np.array([-1, 100, 9223372036854775000, 0], dtype=np.int64)
65
numpy_unsigned_longlong_arr = np.array([1, 100, 9223372036854776000, 0], dtype=np.uint64)
66
numpy_int24_arr = np.array([-1, 8388600, 254, 0], dtype=np.int32)
67
numpy_unsigned_int24_arr = np.array([1, 16777210, 254, 0], dtype=np.uint32)
68
numpy_string_arr = np.array(['hi', 'bye', 'foo', None], dtype=object)
69
numpy_binary_arr = np.array([b'hi', b'bye', b'foo', None], dtype=object)
70

71
numpy_data = [
72
    (numpy_tiny_arr, numpy_nulls),
73
    (numpy_unsigned_tiny_arr, numpy_nulls),
74
    (numpy_short_arr, numpy_nulls),
75
    (numpy_unsigned_short_arr, numpy_nulls),
76
    (numpy_long_arr, numpy_nulls),
77
    (numpy_unsigned_long_arr, numpy_nulls),
78
    (numpy_float_arr, numpy_nulls),
79
    (numpy_double_arr, numpy_nulls),
80
    (numpy_longlong_arr, numpy_nulls),
81
    (numpy_unsigned_longlong_arr, numpy_nulls),
82
    (numpy_int24_arr, numpy_nulls),
83
    (numpy_unsigned_int24_arr, numpy_nulls),
84
    (numpy_string_arr, numpy_nulls),
85
    (numpy_binary_arr, numpy_nulls),
86
]
87

88
polars_row_ids = pl.Series(None, [1, 2, 3, 4], dtype=pl.Int64)
89
polars_nulls = pl.Series(None, numpy_nulls, dtype=pl.Boolean)
90

91
polars_tiny_arr = \
92
    pl.Series(None, numpy_tiny_arr, dtype=pl.Int8)
93
polars_unsigned_tiny_arr = \
94
    pl.Series(None, numpy_unsigned_tiny_arr, dtype=pl.UInt8)
95
polars_short_arr = \
96
    pl.Series(None, numpy_short_arr, dtype=pl.Int16)
97
polars_unsigned_short_arr = \
98
    pl.Series(None, numpy_unsigned_short_arr, dtype=pl.UInt16)
99
polars_long_arr = \
100
    pl.Series(None, numpy_long_arr, dtype=pl.Int32)
101
polars_unsigned_long_arr = \
102
    pl.Series(None, numpy_unsigned_long_arr, dtype=pl.UInt32)
103
polars_float_arr = \
104
    pl.Series(None, numpy_float_arr, dtype=pl.Float32)
105
polars_double_arr = \
106
    pl.Series(None, numpy_double_arr, dtype=pl.Float64)
107
polars_longlong_arr = \
108
    pl.Series(None, numpy_longlong_arr, dtype=pl.Int64)
109
polars_unsigned_longlong_arr = \
110
    pl.Series(None, numpy_unsigned_longlong_arr, dtype=pl.UInt64)
111
polars_int24_arr = \
112
    pl.Series(None, numpy_int24_arr, dtype=pl.Int32)
113
polars_unsigned_int24_arr = \
114
    pl.Series(None, numpy_unsigned_int24_arr, dtype=pl.UInt32)
115
polars_string_arr = \
116
    pl.Series(None, numpy_string_arr.tolist(), dtype=pl.Utf8)
117
polars_binary_arr = \
118
    pl.Series(None, numpy_binary_arr.tolist(), dtype=pl.Binary)
119

120
polars_data = [
121
    (polars_tiny_arr, polars_nulls),
122
    (polars_unsigned_tiny_arr, polars_nulls),
123
    (polars_short_arr, polars_nulls),
124
    (polars_unsigned_short_arr, polars_nulls),
125
    (polars_long_arr, polars_nulls),
126
    (polars_unsigned_long_arr, polars_nulls),
127
    (polars_float_arr, polars_nulls),
128
    (polars_double_arr, polars_nulls),
129
    (polars_longlong_arr, polars_nulls),
130
    (polars_unsigned_longlong_arr, polars_nulls),
131
    (polars_int24_arr, polars_nulls),
132
    (polars_unsigned_int24_arr, polars_nulls),
133
    (polars_string_arr, polars_nulls),
134
    (polars_binary_arr, polars_nulls),
135
]
136

137
pandas_row_ids = pl.Series(None, [1, 2, 3, 4], dtype=pl.Int64)
138
pandas_nulls = pl.Series(None, numpy_nulls, dtype=pl.Boolean)
139

140
pandas_tiny_arr = \
141
    pd.Series(numpy_tiny_arr, dtype=np.int8)
142
pandas_unsigned_tiny_arr = \
143
    pd.Series(numpy_unsigned_tiny_arr, dtype=np.uint8)
144
pandas_short_arr = \
145
    pd.Series(numpy_short_arr, dtype=np.int16)
146
pandas_unsigned_short_arr = \
147
    pd.Series(numpy_unsigned_short_arr, dtype=np.uint16)
148
pandas_long_arr = \
149
    pd.Series(numpy_long_arr, dtype=np.int32)
150
pandas_unsigned_long_arr = \
151
    pd.Series(numpy_unsigned_long_arr, dtype=np.uint32)
152
pandas_float_arr = \
153
    pd.Series(numpy_float_arr, dtype=np.float32)
154
pandas_double_arr = \
155
    pd.Series(numpy_double_arr, dtype=np.float64)
156
pandas_longlong_arr = \
157
    pd.Series(numpy_longlong_arr, dtype=np.int64)
158
pandas_unsigned_longlong_arr = \
159
    pd.Series(numpy_unsigned_longlong_arr, dtype=np.uint64)
160
pandas_int24_arr = \
161
    pd.Series(numpy_int24_arr, dtype=np.int32)
162
pandas_unsigned_int24_arr = \
163
    pd.Series(numpy_unsigned_int24_arr, dtype=np.uint32)
164
pandas_string_arr = \
165
    pd.Series(numpy_string_arr, dtype=object)
166
pandas_binary_arr = \
167
    pd.Series(numpy_binary_arr, dtype=object)
168

169
pandas_data = [
170
    (pandas_tiny_arr, pandas_nulls),
171
    (pandas_unsigned_tiny_arr, pandas_nulls),
172
    (pandas_short_arr, pandas_nulls),
173
    (pandas_unsigned_short_arr, pandas_nulls),
174
    (pandas_long_arr, pandas_nulls),
175
    (pandas_unsigned_long_arr, pandas_nulls),
176
    (pandas_float_arr, pandas_nulls),
177
    (pandas_double_arr, pandas_nulls),
178
    (pandas_longlong_arr, pandas_nulls),
179
    (pandas_unsigned_longlong_arr, pandas_nulls),
180
    (pandas_int24_arr, pandas_nulls),
181
    (pandas_unsigned_int24_arr, pandas_nulls),
182
    (pandas_string_arr, pandas_nulls),
183
    (pandas_binary_arr, pandas_nulls),
184
]
185

186
pyarrow_row_ids = pa.array([1, 2, 3, 4], type=pa.int64())
187
pyarrow_nulls = pa.array(numpy_nulls, type=pa.bool_())
188

189
pyarrow_tiny_arr = \
190
    pa.array(numpy_tiny_arr, type=pa.int8(), mask=numpy_nulls)
191
pyarrow_unsigned_tiny_arr = \
192
    pa.array(numpy_unsigned_tiny_arr, type=pa.uint8(), mask=numpy_nulls)
193
pyarrow_short_arr = \
194
    pa.array(numpy_short_arr, type=pa.int16(), mask=numpy_nulls)
195
pyarrow_unsigned_short_arr = \
196
    pa.array(numpy_unsigned_short_arr, type=pa.uint16(), mask=numpy_nulls)
197
pyarrow_long_arr = \
198
    pa.array(numpy_long_arr, type=pa.int32(), mask=numpy_nulls)
199
pyarrow_unsigned_long_arr = \
200
    pa.array(numpy_unsigned_long_arr, type=pa.uint32(), mask=numpy_nulls)
201
pyarrow_float_arr = \
202
    pa.array(numpy_float_arr, type=pa.float32(), mask=numpy_nulls)
203
pyarrow_double_arr = \
204
    pa.array(numpy_double_arr, type=pa.float64(), mask=numpy_nulls)
205
pyarrow_longlong_arr = \
206
    pa.array(numpy_longlong_arr, type=pa.int64(), mask=numpy_nulls)
207
pyarrow_unsigned_longlong_arr = \
208
    pa.array(numpy_unsigned_longlong_arr, type=pa.uint64(), mask=numpy_nulls)
209
pyarrow_int24_arr = \
210
    pa.array(numpy_int24_arr, type=pa.int32(), mask=numpy_nulls)
211
pyarrow_unsigned_int24_arr = \
212
    pa.array(numpy_unsigned_int24_arr, type=pa.uint32(), mask=numpy_nulls)
213
pyarrow_string_arr = \
214
    pa.array(numpy_string_arr, type=pa.string(), mask=numpy_nulls)
215
pyarrow_binary_arr = \
216
    pa.array(numpy_binary_arr, type=pa.binary(), mask=numpy_nulls)
217

218
pyarrow_data = [
219
    (pyarrow_tiny_arr, pyarrow_nulls),
220
    (pyarrow_unsigned_tiny_arr, pyarrow_nulls),
221
    (pyarrow_short_arr, pyarrow_nulls),
222
    (pyarrow_unsigned_short_arr, pyarrow_nulls),
223
    (pyarrow_long_arr, pyarrow_nulls),
224
    (pyarrow_unsigned_long_arr, pyarrow_nulls),
225
    (pyarrow_float_arr, pyarrow_nulls),
226
    (pyarrow_double_arr, pyarrow_nulls),
227
    (pyarrow_longlong_arr, pyarrow_nulls),
228
    (pyarrow_unsigned_longlong_arr, pyarrow_nulls),
229
    (pyarrow_int24_arr, pyarrow_nulls),
230
    (pyarrow_unsigned_int24_arr, pyarrow_nulls),
231
    (pyarrow_string_arr, pyarrow_nulls),
232
    (pyarrow_binary_arr, pyarrow_nulls),
233
]
234

235
py_row_ids = [1, 2, 3, 4]
236
py_col_data = [
237
    [
238
        -1, 1, -1, 1, -1, 1, -1.0,
239
        -1.0, -1, 1, -1, 1, 'hi', b'hi',
240
    ],
241
    [
242
        100, 100, 32700, 32800, 2147483600, 2147483800, 100.0,
243
        100.0, 100, 100, 8388600, 16777200, 'bye', b'bye',
244
    ],
245
    [
246
        120, 130, 254, 254, 254, 254, 3.14159, 3.14159,
247
        9223372036854775000, 9223372036854776000, 254, 254, 'foo', b'foo',
248
    ],
249
    [
250
        None, None, None, None, None, None, None,
251
        None, None, None, None, None, None, None,
252
    ],
253
]
254

255

256
def assert_py_equal(x, y):
257
    """Compare rows of Python elements."""
258
    for i, (row_x, row_y) in enumerate(zip(x, y)):
259
        for j, (col_x, col_y) in enumerate(zip(row_x, row_y)):
260
            assert type(col_x) is type(col_y), f'{type(col_x)} != {type(col_y)}'
261
            if isinstance(col_x, float):
262
                assert col_x - col_y < 0.0005, f'{i},{j}: {col_x} != {col_y}'
263
            else:
264
                assert col_x == col_y, f'{i},{j}: {col_x} != {col_y}'
265

266

267
class TestRowdat1(unittest.TestCase):
268

269
    def test_numpy_accel(self):
270
        dump_res = rowdat_1._dump_numpy_accel(
271
            col_types, numpy_row_ids, numpy_data,
272
        )
273
        load_res = rowdat_1._load_numpy_accel(col_spec, dump_res)
274

275
        ids = load_res[0]
276
        columns = load_res[1]
277

278
        assert_array_equal(ids, numpy_row_ids)
279
        assert_array_equal(columns[0][0], numpy_tiny_arr, strict=True)
280
        assert_array_equal(columns[1][0], numpy_unsigned_tiny_arr, strict=True)
281
        assert_array_equal(columns[2][0], numpy_short_arr, strict=True)
282
        assert_array_equal(columns[3][0], numpy_unsigned_short_arr, strict=True)
283
        assert_array_equal(columns[4][0], numpy_long_arr, strict=True)
284
        assert_array_equal(columns[5][0], numpy_unsigned_long_arr, strict=True)
285
        assert_array_equal(columns[6][0], numpy_float_arr, strict=True)
286
        assert_array_equal(columns[7][0], numpy_double_arr, strict=True)
287
        assert_array_equal(columns[8][0], numpy_longlong_arr, strict=True)
288
        assert_array_equal(columns[9][0], numpy_unsigned_longlong_arr, strict=True)
289
        assert_array_equal(columns[10][0], numpy_int24_arr, strict=True)
290
        assert_array_equal(columns[11][0], numpy_unsigned_int24_arr, strict=True)
291
        assert_array_equal(columns[12][0], numpy_string_arr, strict=True)
292
        assert_array_equal(columns[13][0], numpy_binary_arr, strict=True)
293

294
    def test_numpy(self):
295
        dump_res = rowdat_1._dump_numpy(
296
            col_types, numpy_row_ids, numpy_data,
297
        )
298
        load_res = rowdat_1._load_numpy(col_spec, dump_res)
299

300
        ids = load_res[0]
301
        columns = load_res[1]
302

303
        assert_array_equal(ids, numpy_row_ids)
304
        assert_array_equal(columns[0][0], numpy_tiny_arr, strict=True)
305
        assert_array_equal(columns[1][0], numpy_unsigned_tiny_arr, strict=True)
306
        assert_array_equal(columns[2][0], numpy_short_arr, strict=True)
307
        assert_array_equal(columns[3][0], numpy_unsigned_short_arr, strict=True)
308
        assert_array_equal(columns[4][0], numpy_long_arr, strict=True)
309
        assert_array_equal(columns[5][0], numpy_unsigned_long_arr, strict=True)
310
        assert_array_equal(columns[6][0], numpy_float_arr, strict=True)
311
        assert_array_equal(columns[7][0], numpy_double_arr, strict=True)
312
        assert_array_equal(columns[8][0], numpy_longlong_arr, strict=True)
313
        assert_array_equal(columns[9][0], numpy_unsigned_longlong_arr, strict=True)
314
        assert_array_equal(columns[10][0], numpy_int24_arr, strict=True)
315
        assert_array_equal(columns[11][0], numpy_unsigned_int24_arr, strict=True)
316
        assert_array_equal(columns[12][0], numpy_string_arr, strict=True)
317
        assert_array_equal(columns[13][0], numpy_binary_arr, strict=True)
318

319
    @parameterized.expand([
320
        ('tinyint exceeds low', TINYINT, -129, ValueError),
321
        ('tinyint low', TINYINT, -128, -128),
322
        ('tinyint high', TINYINT, 127, 127),
323
        ('tinyint exceeds high', TINYINT, 128, ValueError),
324
        ('tinyint zero', TINYINT, 0, 0),
325

326
        ('unsigned tinyint exceeds low', UNSIGNED_TINYINT, -1, ValueError),
327
        ('unsigned tinyint low', UNSIGNED_TINYINT, 0, 0),
328
        ('unsigned tinyint high', UNSIGNED_TINYINT, 255, 255),
329
        ('unsigned tinyint exceeds high', UNSIGNED_TINYINT, 256, ValueError),
330
        ('unsigned tinyint zero', UNSIGNED_TINYINT, 0, 0),
331

332
        ('smallint exceeds low', SMALLINT, -32769, ValueError),
333
        ('smallint low', SMALLINT, -32768, -32768),
334
        ('smallint high', SMALLINT, 32767, 32767),
335
        ('smallint exceeds high', SMALLINT, 32768, ValueError),
336
        ('smallint zero', SMALLINT, 0, 0),
337

338
        ('unsigned smallint exceeds low', UNSIGNED_SMALLINT, -1, ValueError),
339
        ('unsigned smallint low', UNSIGNED_SMALLINT, 0, 0),
340
        ('unsigned smallint high', UNSIGNED_SMALLINT, 65535, 65535),
341
        ('unsigned smallint exceeds high', UNSIGNED_SMALLINT, 65536, ValueError),
342
        ('unsigned smallint zero', UNSIGNED_SMALLINT, 0, 0),
343

344
        ('mediumint exceeds low', MEDIUMINT, -8388609, ValueError),
345
        ('mediumint low', MEDIUMINT, -8388608, -8388608),
346
        ('mediumint high', MEDIUMINT, 8388607, 8388607),
347
        ('mediumint exceeds high', MEDIUMINT, 8388608, ValueError),
348
        ('mediumint zero', MEDIUMINT, 0, 0),
349

350
        ('unsigned mediumint exceeds low', UNSIGNED_MEDIUMINT, -1, ValueError),
351
        ('unsigned mediumint low', UNSIGNED_MEDIUMINT, 0, 0),
352
        ('unsigned mediumint high', UNSIGNED_MEDIUMINT, 16777215, 16777215),
353
        ('unsigned mediumint exceeds high', UNSIGNED_MEDIUMINT, 16777216, ValueError),
354
        ('unsigned mediumint zero', UNSIGNED_MEDIUMINT, 0, 0),
355

356
        ('int exceeds low', INT, -2147483649, ValueError),
357
        ('int low', INT, -2147483648, -2147483648),
358
        ('int high', INT, 2147483647, 2147483647),
359
        ('int exceeds high', INT, 2147483648, ValueError),
360
        ('int zero', INT, 0, 0),
361

362
        ('unsigned int exceeds low', UNSIGNED_INT, -1, ValueError),
363
        ('unsigned int low', UNSIGNED_INT, 0, 0),
364
        ('unsigned int high', UNSIGNED_INT, 4294967295, 4294967295),
365
        ('unsigned int exceeds high', UNSIGNED_INT, 4294967296, ValueError),
366
        ('unsigned int zero', UNSIGNED_INT, 0, 0),
367

368
        ('bigint exceeds low', BIGINT, -2**63 - 1, ValueError),
369
        ('bigint low', BIGINT, -2**63, -2**63),
370
        ('bigint high', BIGINT, 2**63 - 1, 2**63 - 1),
371
        ('bigint exceeds high', BIGINT, 2**63, ValueError),
372
        ('bigint zero', BIGINT, 0, 0),
373

374
        ('unsigned bigint exceeds low', UNSIGNED_BIGINT, -1, ValueError),
375
        ('unsigned bigint low', UNSIGNED_BIGINT, 0, 0),
376
        ('unsigned bigint high', UNSIGNED_BIGINT, 2**64 - 1, 2**64 - 1),
377
        ('unsigned bigint exceeds high', UNSIGNED_BIGINT, 2**64, ValueError),
378
        ('unsigned bigint zero', UNSIGNED_BIGINT, 0, 0),
379
    ])
380
    def test_numpy_accel_limits(self, name, dtype, data, res):
381
        numpy_row_ids = np.array([1])
382

383
        arr = np.array([data])
384

385
        if type(res) is type and issubclass(res, Exception):
386
            # Accelerated
387
            with self.assertRaises(res, msg=f'Expected {res} for {data} in {dtype}'):
388
                rowdat_1._dump_numpy_accel(
389
                    [dtype], numpy_row_ids, [(arr, None)],
390
                )
391

392
            # Pure Python
393
            if 'mediumint exceeds' in name:
394
                pass
395
            else:
396
                with self.assertRaises(res, msg=f'Expected {res} for {data} in {dtype}'):
397
                    rowdat_1._dump_numpy(
398
                        [dtype], numpy_row_ids, [(arr, None)],
399
                    )
400

401
        else:
402
            # Accelerated
403
            dump_res = rowdat_1._dump_numpy_accel(
404
                [dtype], numpy_row_ids, [(arr, None)],
405
            )
406
            load_res = rowdat_1._load_numpy_accel([('x', dtype)], dump_res)
407
            assert load_res[1][0][0] == res, \
408
                f'Expected {res} for {data}, but got {load_res[1][0][0]} in {dtype}'
409

410
            # Pure Python
411
            dump_res = rowdat_1._dump_numpy(
412
                [dtype], numpy_row_ids, [(arr, None)],
413
            )
414
            load_res = rowdat_1._load_numpy([('x', dtype)], dump_res)
415
            assert load_res[1][0][0] == res, \
416
                f'Expected {res} for {data}, but got {load_res[1][0][0]} in {dtype}'
417

418
    @parameterized.expand([
419
        (
420
            'tinyint from int64', TINYINT,
421
            np.array([-128, 0, 127, 67], dtype=np.int64),
422
            np.array([-128, 0, 127, 67], dtype=np.int8),
423
        ),
424
        (
425
            'unsigned tinyint from int64', UNSIGNED_TINYINT,
426
            np.array([0, 255, 241], dtype=np.int64),
427
            np.array([0, 255, 241], dtype=np.uint8),
428
        ),
429
        (
430
            'smallint from int64', SMALLINT,
431
            np.array([-32768, 0, 32767, 25557], dtype=np.int64),
432
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
433
        ),
434
        (
435
            'unsigned smallint from int64', UNSIGNED_SMALLINT,
436
            np.array([0, 65535, 40513], dtype=np.int64),
437
            np.array([0, 65535, 40513], dtype=np.uint16),
438
        ),
439
        (
440
            'mediumint from int64', MEDIUMINT,
441
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int64),
442
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
443
        ),
444
        (
445
            'unsigned mediumint from int64', UNSIGNED_MEDIUMINT,
446
            np.array([0, 16777215, 9996781], dtype=np.int64),
447
            np.array([0, 16777215, 9996781], dtype=np.uint32),
448
        ),
449
        (
450
            'int from int64', INT,
451
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int64),
452
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int32),
453
        ),
454
        (
455
            'unsigned int from int64', UNSIGNED_INT,
456
            np.array([0, 4294967295, 339826098], dtype=np.int64),
457
            np.array([0, 4294967295, 339826098], dtype=np.uint32),
458
        ),
459
        (
460
            'bigint from int64', BIGINT,
461
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.int64),
462
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.int64),
463
        ),
464
        (
465
            'unsigned bigint from int64', UNSIGNED_BIGINT,
466
            np.array([0, 2**63 - 1, 2**63 - 87629], dtype=np.int64),
467
            np.array([0, 2**63 - 1, 2**63 - 87629], dtype=np.uint64),
468
        ),
469
        (
470
            'float from int64', FLOAT,
471
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.int64),
472
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.float32),
473
        ),
474
        (
475
            'double from int64', DOUBLE,
476
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.int64),
477
            np.array([-2**63, 0, 2**63 - 1, 9381123867689], dtype=np.float64),
478
        ),
479

480
        (
481
            'tinyint from int16', TINYINT,
482
            np.array([-128, 0, 127, 67], dtype=np.int16),
483
            np.array([-128, 0, 127, 67], dtype=np.int8),
484
        ),
485
        (
486
            'unsigned tinyint from int16', UNSIGNED_TINYINT,
487
            np.array([0, 255, 241], dtype=np.int16),
488
            np.array([0, 255, 241], dtype=np.uint8),
489
        ),
490
        (
491
            'smallint from int16', SMALLINT,
492
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
493
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
494
        ),
495
        (
496
            'unsigned smallint from int16', UNSIGNED_SMALLINT,
497
            np.array([0, 32767, 25557], dtype=np.int16),
498
            np.array([0, 32767, 25557], dtype=np.uint16),
499
        ),
500
        (
501
            'mediumint from int16', MEDIUMINT,
502
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
503
            np.array([-32768, 0, 32767, 25557], dtype=np.int32),
504
        ),
505
        (
506
            'unsigned mediumint from int16', UNSIGNED_MEDIUMINT,
507
            np.array([0, 32767, 25557], dtype=np.int16),
508
            np.array([0, 32767, 25557], dtype=np.uint32),
509
        ),
510
        (
511
            'int from int16', INT,
512
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
513
            np.array([-32768, 0, 32767, 25557], dtype=np.int32),
514
        ),
515
        (
516
            'unsigned int from int16', UNSIGNED_INT,
517
            np.array([0, 32767, 25557], dtype=np.int16),
518
            np.array([0, 32767, 25557], dtype=np.uint32),
519
        ),
520
        (
521
            'bigint from int16', BIGINT,
522
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
523
            np.array([-32768, 0, 32767, 25557], dtype=np.int64),
524
        ),
525
        (
526
            'unsigned bigint from int16', UNSIGNED_BIGINT,
527
            np.array([0, 32767, 25557], dtype=np.int16),
528
            np.array([0, 32767, 25557], dtype=np.uint64),
529
        ),
530
        (
531
            'float from int16', FLOAT,
532
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
533
            np.array([-32768, 0, 32767, 25557], dtype=np.float32),
534
        ),
535
        (
536
            'double from int16', DOUBLE,
537
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
538
            np.array([-32768, 0, 32767, 25557], dtype=np.float64),
539
        ),
540

541
        (
542
            'tinyint from int32', TINYINT,
543
            np.array([-128, 0, 127, 67], dtype=np.int32),
544
            np.array([-128, 0, 127, 67], dtype=np.int8),
545
        ),
546
        (
547
            'unsigned tinyint from int32', UNSIGNED_TINYINT,
548
            np.array([0, 255, 241], dtype=np.int32),
549
            np.array([0, 255, 241], dtype=np.uint8),
550
        ),
551
        (
552
            'smallint from int32', SMALLINT,
553
            np.array([-32768, 0, 32767, 25557], dtype=np.int32),
554
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
555
        ),
556
        (
557
            'unsigned smallint from int32', UNSIGNED_SMALLINT,
558
            np.array([0, 65535, 40513], dtype=np.int32),
559
            np.array([0, 65535, 40513], dtype=np.uint16),
560
        ),
561
        (
562
            'mediumint from int32', MEDIUMINT,
563
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
564
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
565
        ),
566
        (
567
            'unsigned mediumint from int32', UNSIGNED_MEDIUMINT,
568
            np.array([0, 16777215, 9996781], dtype=np.int32),
569
            np.array([0, 16777215, 9996781], dtype=np.uint32),
570
        ),
571
        (
572
            'int from int32', INT,
573
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int32),
574
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int32),
575
        ),
576
        (
577
            'unsigned int from int32', UNSIGNED_INT,
578
            np.array([0, 2147483647, 3398268], dtype=np.int32),
579
            np.array([0, 2147483647, 3398268], dtype=np.uint32),
580
        ),
581
        (
582
            'bigint from int32', BIGINT,
583
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.int32),
584
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.int64),
585
        ),
586
        (
587
            'unsigned bigint from int32', UNSIGNED_BIGINT,
588
            np.array([0, 2147483647, 987362899], dtype=np.int32),
589
            np.array([0, 2147483647, 987362899], dtype=np.uint64),
590
        ),
591
        (
592
            'float from int32', FLOAT,
593
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.int32),
594
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.float32),
595
        ),
596
        (
597
            'double from int32', DOUBLE,
598
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.int32),
599
            np.array([-2147483648, 0, 2147483647, 789768920], dtype=np.float64),
600
        ),
601

602
        (
603
            'tinyint from int64', TINYINT,
604
            np.array([-128, 0, 127, 67], dtype=np.int64),
605
            np.array([-128, 0, 127, 67], dtype=np.int8),
606
        ),
607
        (
608
            'unsigned tinyint from int64', UNSIGNED_TINYINT,
609
            np.array([0, 255, 241], dtype=np.int64),
610
            np.array([0, 255, 241], dtype=np.uint8),
611
        ),
612
        (
613
            'smallint from int64', SMALLINT,
614
            np.array([-32768, 0, 32767, 25557], dtype=np.int64),
615
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
616
        ),
617
        (
618
            'unsigned smallint from int64', UNSIGNED_SMALLINT,
619
            np.array([0, 65535, 40513], dtype=np.int64),
620
            np.array([0, 65535, 40513], dtype=np.uint16),
621
        ),
622
        (
623
            'mediumint from int64', MEDIUMINT,
624
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int64),
625
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
626
        ),
627
        (
628
            'unsigned mediumint from int64', UNSIGNED_MEDIUMINT,
629
            np.array([0, 16777215, 9996781], dtype=np.int64),
630
            np.array([0, 16777215, 9996781], dtype=np.uint32),
631
        ),
632
        (
633
            'int from int64', INT,
634
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int64),
635
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int32),
636
        ),
637
        (
638
            'unsigned int from int64', UNSIGNED_INT,
639
            np.array([0, 2147483647, 3398268], dtype=np.int64),
640
            np.array([0, 2147483647, 3398268], dtype=np.uint32),
641
        ),
642
        (
643
            'bigint from int64', BIGINT,
644
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.int64),
645
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.int64),
646
        ),
647
        (
648
            'unsigned bigint from int64', UNSIGNED_BIGINT,
649
            np.array([0, 2**63 - 1, 987362899], dtype=np.int64),
650
            np.array([0, 2**63 - 1, 987362899], dtype=np.uint64),
651
        ),
652
        (
653
            'float from int64', FLOAT,
654
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.int64),
655
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float32),
656
        ),
657
        (
658
            'double from int64', DOUBLE,
659
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.int64),
660
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float64),
661
        ),
662

663
        (
664
            'tinyint from float32', TINYINT,
665
            np.array([-128, 0, 127, 67], dtype=np.float32),
666
            np.array([-128, 0, 127, 67], dtype=np.int8),
667
        ),
668
        (
669
            'unsigned tinyint from float32', UNSIGNED_TINYINT,
670
            np.array([0, 255, 241], dtype=np.float32),
671
            np.array([0, 255, 241], dtype=np.uint8),
672
        ),
673
        (
674
            'smallint from float32', SMALLINT,
675
            np.array([-32768, 0, 32767, 25557], dtype=np.float32),
676
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
677
        ),
678
        (
679
            'unsigned smallint from float32', UNSIGNED_SMALLINT,
680
            np.array([0, 65535, 40513], dtype=np.float32),
681
            np.array([0, 65535, 40513], dtype=np.uint16),
682
        ),
683
        (
684
            'mediumint from float32', MEDIUMINT,
685
            np.array([-8388608, 0, 8388607, 999678], dtype=np.float32),
686
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
687
        ),
688
        (
689
            'unsigned mediumint from float32', UNSIGNED_MEDIUMINT,
690
            np.array([0, 16777215, 9996781], dtype=np.float32),
691
            np.array([0, 16777215, 9996781], dtype=np.uint32),
692
        ),
693
        (
694
            'int from float32', INT,
695
            np.array([-2147483648, 0, 214748352, 1123867648], dtype=np.float32),
696
            np.array([-2147483648, 0, 214748352, 1123867648], dtype=np.int32),
697
        ),
698
        (
699
            'unsigned int from float32', UNSIGNED_INT,
700
            np.array([0, 214748368, 3398268], dtype=np.float32),
701
            np.array([0, 214748368, 3398268], dtype=np.uint32),
702
        ),
703
        (
704
            'bigint from float32', BIGINT,
705
            np.array([-2**63, 0, 2**23 - 1, 78976892928], dtype=np.float32),
706
            np.array([-2**63, 0, 2**23 - 1, 78976892928], dtype=np.int64),
707
        ),
708
        (
709
            'unsigned bigint from float32', UNSIGNED_BIGINT,
710
            np.array([0, 2**23 - 1, 987362880], dtype=np.float32),
711
            np.array([0, 2**23 - 1, 987362880], dtype=np.uint64),
712
        ),
713
        (
714
            'float from float32', FLOAT,
715
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float32),
716
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float32),
717
        ),
718
        (
719
            'double from float32', DOUBLE,
720
            np.array([-8388.099609, 0.0, 8388.099609, 1234.567017], dtype=np.float32),
721
            np.array([-8388.099609, 0.0, 8388.099609, 1234.567017], dtype=np.float64),
722
        ),
723

724
        (
725
            'tinyint from float64', TINYINT,
726
            np.array([-128, 0, 127, 67], dtype=np.float64),
727
            np.array([-128, 0, 127, 67], dtype=np.int8),
728
        ),
729
        (
730
            'unsigned tinyint from float64', UNSIGNED_TINYINT,
731
            np.array([0, 255, 241], dtype=np.float64),
732
            np.array([0, 255, 241], dtype=np.uint8),
733
        ),
734
        (
735
            'smallint from float64', SMALLINT,
736
            np.array([-32768, 0, 32767, 25557], dtype=np.float64),
737
            np.array([-32768, 0, 32767, 25557], dtype=np.int16),
738
        ),
739
        (
740
            'unsigned smallint from float64', UNSIGNED_SMALLINT,
741
            np.array([0, 65535, 40513], dtype=np.float64),
742
            np.array([0, 65535, 40513], dtype=np.uint16),
743
        ),
744
        (
745
            'mediumint from float64', MEDIUMINT,
746
            np.array([-8388608, 0, 8388607, 999678], dtype=np.float64),
747
            np.array([-8388608, 0, 8388607, 999678], dtype=np.int32),
748
        ),
749
        (
750
            'unsigned mediumint from float64', UNSIGNED_MEDIUMINT,
751
            np.array([0, 16777215, 9996781], dtype=np.float64),
752
            np.array([0, 16777215, 9996781], dtype=np.uint32),
753
        ),
754
        (
755
            'int from float64', INT,
756
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.float64),
757
            np.array([-2147483648, 0, 2147483647, 1123867689], dtype=np.int32),
758
        ),
759
        (
760
            'unsigned int from float64', UNSIGNED_INT,
761
            np.array([0, 2147483647, 3398268], dtype=np.float64),
762
            np.array([0, 2147483647, 3398268], dtype=np.uint32),
763
        ),
764
        (
765
            'bigint from float64', BIGINT,
766
            np.array([-2**63, 0, 2**53 - 1, 78976892012], dtype=np.float64),
767
            np.array([-2**63, 0, 2**53 - 1, 78976892012], dtype=np.int64),
768
        ),
769
        (
770
            'unsigned bigint from float64', UNSIGNED_BIGINT,
771
            np.array([0, 2**53 - 1, 987362899], dtype=np.float64),
772
            np.array([0, 2**53 - 1, 987362899], dtype=np.uint64),
773
        ),
774
        (
775
            'float from float64', FLOAT,
776
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float64),
777
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float32),
778
        ),
779
        (
780
            'double from float64', DOUBLE,
781
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float64),
782
            np.array([-2**63, 0, 2**63 - 1, 78976892012], dtype=np.float64),
783
        ),
784
    ])
785
    def test_numpy_accel_casts(self, name, dtype, data, res):
786
        numpy_row_ids = np.array(list(range(len(data))))
787

788
        # Accelerated
789
        dump_res = rowdat_1._dump_numpy_accel(
790
            [dtype], numpy_row_ids, [(data, None)],
791
        )
792
        load_res = rowdat_1._load_numpy_accel([('x', dtype)], dump_res)
793

794
        if name == 'double from float32':
795
            assert load_res[1][0][0].dtype is res.dtype
796
            assert (load_res[1][0][0] - res < 0.00005).all()
797
        else:
798
            np.testing.assert_array_equal(load_res[1][0][0], res, strict=True)
799

800
        # Pure Python
801
        dump_res = rowdat_1._dump_numpy(
802
            [dtype], numpy_row_ids, [(data, None)],
803
        )
804
        load_res = rowdat_1._load_numpy([('x', dtype)], dump_res)
805

806
        if name == 'double from float32':
807
            assert load_res[1][0][0].dtype is res.dtype
808
            assert (load_res[1][0][0] - res < 0.00005).all()
809
        else:
810
            np.testing.assert_array_equal(load_res[1][0][0], res, strict=True)
811

812
    def test_python(self):
813
        dump_res = rowdat_1._dump(
814
            col_types, py_row_ids, py_col_data,
815
        )
816
        load_res = rowdat_1._load(col_spec, dump_res)
817

818
        ids = load_res[0]
819
        columns = load_res[1]
820

821
        assert ids == py_row_ids
822
        assert_py_equal(columns, py_col_data)
823

824
    def test_python_accel(self):
825
        dump_res = rowdat_1._dump_accel(
826
            col_types, py_row_ids, py_col_data,
827
        )
828
        load_res = rowdat_1._load_accel(col_spec, dump_res)
829

830
        ids = load_res[0]
831
        columns = load_res[1]
832

833
        assert ids == py_row_ids
834
        assert_py_equal(columns, py_col_data)
835

836
    def test_polars(self):
837
        dump_res = rowdat_1._dump_polars(
838
            col_types, polars_row_ids, polars_data,
839
        )
840
        load_res = rowdat_1._load_polars(col_spec, dump_res)
841

842
        ids = load_res[0]
843
        columns = load_res[1]
844

845
        assert_array_equal(ids, polars_row_ids)
846
        assert_array_equal(columns[0][0], polars_tiny_arr, strict=True)
847
        assert_array_equal(columns[1][0], polars_unsigned_tiny_arr, strict=True)
848
        assert_array_equal(columns[2][0], polars_short_arr, strict=True)
849
        assert_array_equal(columns[3][0], polars_unsigned_short_arr, strict=True)
850
        assert_array_equal(columns[4][0], polars_long_arr, strict=True)
851
        assert_array_equal(columns[5][0], polars_unsigned_long_arr, strict=True)
852
        assert_array_equal(columns[6][0], polars_float_arr, strict=True)
853
        assert_array_equal(columns[7][0], polars_double_arr, strict=True)
854
        assert_array_equal(columns[8][0], polars_longlong_arr, strict=True)
855
        assert_array_equal(columns[9][0], polars_unsigned_longlong_arr, strict=True)
856
        assert_array_equal(columns[10][0], polars_int24_arr, strict=True)
857
        assert_array_equal(columns[11][0], polars_unsigned_int24_arr, strict=True)
858
        assert_array_equal(columns[12][0], polars_string_arr, strict=True)
859
        assert_array_equal(columns[13][0], polars_binary_arr, strict=True)
860

861
    def test_polars_accel(self):
862
        dump_res = rowdat_1._dump_polars_accel(
863
            col_types, polars_row_ids, polars_data,
864
        )
865
        load_res = rowdat_1._load_polars_accel(col_spec, dump_res)
866

867
        ids = load_res[0]
868
        columns = load_res[1]
869

870
        assert_array_equal(ids, polars_row_ids)
871
        assert_array_equal(columns[0][0], polars_tiny_arr, strict=True)
872
        assert_array_equal(columns[1][0], polars_unsigned_tiny_arr, strict=True)
873
        assert_array_equal(columns[2][0], polars_short_arr, strict=True)
874
        assert_array_equal(columns[3][0], polars_unsigned_short_arr, strict=True)
875
        assert_array_equal(columns[4][0], polars_long_arr, strict=True)
876
        assert_array_equal(columns[5][0], polars_unsigned_long_arr, strict=True)
877
        assert_array_equal(columns[6][0], polars_float_arr, strict=True)
878
        assert_array_equal(columns[7][0], polars_double_arr, strict=True)
879
        assert_array_equal(columns[8][0], polars_longlong_arr, strict=True)
880
        assert_array_equal(columns[9][0], polars_unsigned_longlong_arr, strict=True)
881
        assert_array_equal(columns[10][0], polars_int24_arr, strict=True)
882
        assert_array_equal(columns[11][0], polars_unsigned_int24_arr, strict=True)
883
        assert_array_equal(columns[12][0], polars_string_arr, strict=True)
884
        assert_array_equal(columns[13][0], polars_binary_arr, strict=True)
885

886
    def test_pandas(self):
887
        dump_res = rowdat_1._dump_pandas(
888
            col_types, pandas_row_ids, pandas_data,
889
        )
890
        load_res = rowdat_1._load_pandas(col_spec, dump_res)
891

892
        ids = load_res[0]
893
        columns = load_res[1]
894

895
        assert_array_equal(ids, pandas_row_ids)
896
        assert_array_equal(columns[0][0], pandas_tiny_arr, strict=True)
897
        assert_array_equal(columns[1][0], pandas_unsigned_tiny_arr, strict=True)
898
        assert_array_equal(columns[2][0], pandas_short_arr, strict=True)
899
        assert_array_equal(columns[3][0], pandas_unsigned_short_arr, strict=True)
900
        assert_array_equal(columns[4][0], pandas_long_arr, strict=True)
901
        assert_array_equal(columns[5][0], pandas_unsigned_long_arr, strict=True)
902
        assert_array_equal(columns[6][0], pandas_float_arr, strict=True)
903
        assert_array_equal(columns[7][0], pandas_double_arr, strict=True)
904
        assert_array_equal(columns[8][0], pandas_longlong_arr, strict=True)
905
        assert_array_equal(columns[9][0], pandas_unsigned_longlong_arr, strict=True)
906
        assert_array_equal(columns[10][0], pandas_int24_arr, strict=True)
907
        assert_array_equal(columns[11][0], pandas_unsigned_int24_arr, strict=True)
908
        assert_array_equal(columns[12][0], pandas_string_arr, strict=True)
909
        assert_array_equal(columns[13][0], pandas_binary_arr, strict=True)
910

911
    def test_pandas_accel(self):
912
        dump_res = rowdat_1._dump_pandas_accel(
913
            col_types, pandas_row_ids, pandas_data,
914
        )
915
        load_res = rowdat_1._load_pandas_accel(col_spec, dump_res)
916

917
        ids = load_res[0]
918
        columns = load_res[1]
919

920
        assert_array_equal(ids, pandas_row_ids)
921
        assert_array_equal(columns[0][0], pandas_tiny_arr, strict=True)
922
        assert_array_equal(columns[1][0], pandas_unsigned_tiny_arr, strict=True)
923
        assert_array_equal(columns[2][0], pandas_short_arr, strict=True)
924
        assert_array_equal(columns[3][0], pandas_unsigned_short_arr, strict=True)
925
        assert_array_equal(columns[4][0], pandas_long_arr, strict=True)
926
        assert_array_equal(columns[5][0], pandas_unsigned_long_arr, strict=True)
927
        assert_array_equal(columns[6][0], pandas_float_arr, strict=True)
928
        assert_array_equal(columns[7][0], pandas_double_arr, strict=True)
929
        assert_array_equal(columns[8][0], pandas_longlong_arr, strict=True)
930
        assert_array_equal(columns[9][0], pandas_unsigned_longlong_arr, strict=True)
931
        assert_array_equal(columns[10][0], pandas_int24_arr, strict=True)
932
        assert_array_equal(columns[11][0], pandas_unsigned_int24_arr, strict=True)
933
        assert_array_equal(columns[12][0], pandas_string_arr, strict=True)
934
        assert_array_equal(columns[13][0], pandas_binary_arr, strict=True)
935

936
    def test_pyarrow(self):
937
        dump_res = rowdat_1._dump_arrow(
938
            col_types, pyarrow_row_ids, pyarrow_data,
939
        )
940
        load_res = rowdat_1._load_arrow(col_spec, dump_res)
941

942
        ids = load_res[0]
943
        columns = load_res[1]
944

945
        assert_array_equal(ids, pyarrow_row_ids)
946
        assert_array_equal(columns[0][0], pyarrow_tiny_arr, strict=True)
947
        assert_array_equal(columns[1][0], pyarrow_unsigned_tiny_arr, strict=True)
948
        assert_array_equal(columns[2][0], pyarrow_short_arr, strict=True)
949
        assert_array_equal(columns[3][0], pyarrow_unsigned_short_arr, strict=True)
950
        assert_array_equal(columns[4][0], pyarrow_long_arr, strict=True)
951
        assert_array_equal(columns[5][0], pyarrow_unsigned_long_arr, strict=True)
952
        assert_array_equal(columns[6][0], pyarrow_float_arr, strict=True)
953
        assert_array_equal(columns[7][0], pyarrow_double_arr, strict=True)
954
        assert_array_equal(columns[8][0], pyarrow_longlong_arr, strict=True)
955
        assert_array_equal(columns[9][0], pyarrow_unsigned_longlong_arr, strict=True)
956
        assert_array_equal(columns[10][0], pyarrow_int24_arr, strict=True)
957
        assert_array_equal(columns[11][0], pyarrow_unsigned_int24_arr, strict=True)
958
        assert_array_equal(columns[12][0], pyarrow_string_arr, strict=True)
959
        assert_array_equal(columns[13][0], pyarrow_binary_arr, strict=True)
960

961
    def test_pyarrow_accel(self):
962
        dump_res = rowdat_1._dump_arrow_accel(
963
            col_types, pyarrow_row_ids, pyarrow_data,
964
        )
965
        load_res = rowdat_1._load_arrow_accel(col_spec, dump_res)
966

967
        ids = load_res[0]
968
        columns = load_res[1]
969

970
        assert_array_equal(ids, pyarrow_row_ids)
971
        assert_array_equal(columns[0][0], pyarrow_tiny_arr, strict=True)
972
        assert_array_equal(columns[1][0], pyarrow_unsigned_tiny_arr, strict=True)
973
        assert_array_equal(columns[2][0], pyarrow_short_arr, strict=True)
974
        assert_array_equal(columns[3][0], pyarrow_unsigned_short_arr, strict=True)
975
        assert_array_equal(columns[4][0], pyarrow_long_arr, strict=True)
976
        assert_array_equal(columns[5][0], pyarrow_unsigned_long_arr, strict=True)
977
        assert_array_equal(columns[6][0], pyarrow_float_arr, strict=True)
978
        assert_array_equal(columns[7][0], pyarrow_double_arr, strict=True)
979
        assert_array_equal(columns[8][0], pyarrow_longlong_arr, strict=True)
980
        assert_array_equal(columns[9][0], pyarrow_unsigned_longlong_arr, strict=True)
981
        assert_array_equal(columns[10][0], pyarrow_int24_arr, strict=True)
982
        assert_array_equal(columns[11][0], pyarrow_unsigned_int24_arr, strict=True)
983
        assert_array_equal(columns[12][0], pyarrow_string_arr, strict=True)
984
        assert_array_equal(columns[13][0], pyarrow_binary_arr, strict=True)
985

986

987
class TestJSON(unittest.TestCase):
988

989
    def test_numpy(self):
990
        dump_res = jsonx.dump_numpy(
991
            col_types, numpy_row_ids, numpy_data,
992
        )
993
        import pprint
994
        pprint.pprint(json.loads(dump_res))
995
        load_res = jsonx.load_numpy(col_spec, dump_res)
996

997
        ids = load_res[0]
998
        columns = load_res[1]
999

1000
        assert_array_equal(ids, numpy_row_ids)
1001
        assert_array_equal(columns[0][0], numpy_tiny_arr, strict=True)
1002
        assert_array_equal(columns[1][0], numpy_unsigned_tiny_arr, strict=True)
1003
        assert_array_equal(columns[2][0], numpy_short_arr, strict=True)
1004
        assert_array_equal(columns[3][0], numpy_unsigned_short_arr, strict=True)
1005
        assert_array_equal(columns[4][0], numpy_long_arr, strict=True)
1006
        assert_array_equal(columns[5][0], numpy_unsigned_long_arr, strict=True)
1007
        assert_array_equal(columns[6][0], numpy_float_arr, strict=True)
1008
        assert_array_equal(columns[7][0], numpy_double_arr, strict=True)
1009
        assert_array_equal(columns[8][0], numpy_longlong_arr, strict=True)
1010
        assert_array_equal(columns[9][0], numpy_unsigned_longlong_arr, strict=True)
1011
        assert_array_equal(columns[10][0], numpy_int24_arr, strict=True)
1012
        assert_array_equal(columns[11][0], numpy_unsigned_int24_arr, strict=True)
1013
        assert_array_equal(columns[12][0], numpy_string_arr, strict=True)
1014
        assert_array_equal(columns[13][0], numpy_binary_arr, strict=True)
1015

1016
    def test_python(self):
1017
        dump_res = jsonx.dump(
1018
            col_types, py_row_ids, py_col_data,
1019
        )
1020
        load_res = jsonx.load(col_spec, dump_res)
1021

1022
        ids = load_res[0]
1023
        columns = load_res[1]
1024

1025
        assert ids == py_row_ids
1026
        assert_py_equal(columns, py_col_data)
1027

1028
    def test_polars(self):
1029
        dump_res = jsonx.dump_polars(
1030
            col_types, polars_row_ids, polars_data,
1031
        )
1032
        load_res = jsonx.load_polars(col_spec, dump_res)
1033

1034
        ids = load_res[0]
1035
        columns = load_res[1]
1036

1037
        assert_array_equal(ids, polars_row_ids)
1038
        assert_array_equal(columns[0][0], polars_tiny_arr, strict=True)
1039
        assert_array_equal(columns[1][0], polars_unsigned_tiny_arr, strict=True)
1040
        assert_array_equal(columns[2][0], polars_short_arr, strict=True)
1041
        assert_array_equal(columns[3][0], polars_unsigned_short_arr, strict=True)
1042
        assert_array_equal(columns[4][0], polars_long_arr, strict=True)
1043
        assert_array_equal(columns[5][0], polars_unsigned_long_arr, strict=True)
1044
        assert_array_equal(columns[6][0], polars_float_arr, strict=True)
1045
        assert_array_equal(columns[7][0], polars_double_arr, strict=True)
1046
        assert_array_equal(columns[8][0], polars_longlong_arr, strict=True)
1047
        assert_array_equal(columns[9][0], polars_unsigned_longlong_arr, strict=True)
1048
        assert_array_equal(columns[10][0], polars_int24_arr, strict=True)
1049
        assert_array_equal(columns[11][0], polars_unsigned_int24_arr, strict=True)
1050
        assert_array_equal(columns[12][0], polars_string_arr, strict=True)
1051
        assert_array_equal(columns[13][0], polars_binary_arr, strict=True)
1052

1053
    def test_pandas(self):
1054
        dump_res = rowdat_1._dump_pandas(
1055
            col_types, pandas_row_ids, pandas_data,
1056
        )
1057
        load_res = rowdat_1._load_pandas(col_spec, dump_res)
1058

1059
        ids = load_res[0]
1060
        columns = load_res[1]
1061

1062
        assert_array_equal(ids, pandas_row_ids)
1063
        assert_array_equal(columns[0][0], pandas_tiny_arr, strict=True)
1064
        assert_array_equal(columns[1][0], pandas_unsigned_tiny_arr, strict=True)
1065
        assert_array_equal(columns[2][0], pandas_short_arr, strict=True)
1066
        assert_array_equal(columns[3][0], pandas_unsigned_short_arr, strict=True)
1067
        assert_array_equal(columns[4][0], pandas_long_arr, strict=True)
1068
        assert_array_equal(columns[5][0], pandas_unsigned_long_arr, strict=True)
1069
        assert_array_equal(columns[6][0], pandas_float_arr, strict=True)
1070
        assert_array_equal(columns[7][0], pandas_double_arr, strict=True)
1071
        assert_array_equal(columns[8][0], pandas_longlong_arr, strict=True)
1072
        assert_array_equal(columns[9][0], pandas_unsigned_longlong_arr, strict=True)
1073
        assert_array_equal(columns[10][0], pandas_int24_arr, strict=True)
1074
        assert_array_equal(columns[11][0], pandas_unsigned_int24_arr, strict=True)
1075
        assert_array_equal(columns[12][0], pandas_string_arr, strict=True)
1076
        assert_array_equal(columns[13][0], pandas_binary_arr, strict=True)
1077

1078
    def test_pyarrow(self):
1079
        dump_res = rowdat_1._dump_arrow(
1080
            col_types, pyarrow_row_ids, pyarrow_data,
1081
        )
1082
        load_res = rowdat_1._load_arrow(col_spec, dump_res)
1083

1084
        ids = load_res[0]
1085
        columns = load_res[1]
1086

1087
        assert_array_equal(ids, pyarrow_row_ids)
1088
        assert_array_equal(columns[0][0], pyarrow_tiny_arr, strict=True)
1089
        assert_array_equal(columns[1][0], pyarrow_unsigned_tiny_arr, strict=True)
1090
        assert_array_equal(columns[2][0], pyarrow_short_arr, strict=True)
1091
        assert_array_equal(columns[3][0], pyarrow_unsigned_short_arr, strict=True)
1092
        assert_array_equal(columns[4][0], pyarrow_long_arr, strict=True)
1093
        assert_array_equal(columns[5][0], pyarrow_unsigned_long_arr, strict=True)
1094
        assert_array_equal(columns[6][0], pyarrow_float_arr, strict=True)
1095
        assert_array_equal(columns[7][0], pyarrow_double_arr, strict=True)
1096
        assert_array_equal(columns[8][0], pyarrow_longlong_arr, strict=True)
1097
        assert_array_equal(columns[9][0], pyarrow_unsigned_longlong_arr, strict=True)
1098
        assert_array_equal(columns[10][0], pyarrow_int24_arr, strict=True)
1099
        assert_array_equal(columns[11][0], pyarrow_unsigned_int24_arr, strict=True)
1100
        assert_array_equal(columns[12][0], pyarrow_string_arr, strict=True)
1101
        assert_array_equal(columns[13][0], pyarrow_binary_arr, strict=True)
1102

1103
Product

Resources

Company
