CoCalc -- ck

GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/contrib/ck/include/ck_pr.h
⁴⁸²⁵⁵ views
1
/*
2
 * Copyright 2009-2015 Samy Al Bahra.
3
 * Copyright 2011 David Joseph.
4
 * All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without
7
 * modification, are permitted provided that the following conditions
8
 * are met:
9
 * 1. Redistributions of source code must retain the above copyright
10
 *    notice, this list of conditions and the following disclaimer.
11
 * 2. Redistributions in binary form must reproduce the above copyright
12
 *    notice, this list of conditions and the following disclaimer in the
13
 *    documentation and/or other materials provided with the distribution.
14
 *
15
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25
 * SUCH DAMAGE.
26
 */
27

28
#ifndef CK_PR_H
29
#define CK_PR_H
30

31
#include <ck_cc.h>
32
#include <ck_limits.h>
33
#include <ck_md.h>
34
#include <ck_stdint.h>
35
#include <ck_stdbool.h>
36

37
/*
38
 * Default to using builtins for clang analyzer, coverity, and sparse:
39
 * inline assembly is often too opaque for useful analysis.  Override
40
 * the defaults by defining CK_USE_CC_BUILTINS=0 or 1.
41
 */
42
#if !defined(CK_USE_CC_BUILTINS)
43
#if defined(__clang_analyzer__) || defined(__COVERITY__) || defined(__CHECKER__)
44
#define CK_USE_CC_BUILTINS 1
45
#else
46
#define CK_USE_CC_BUILTINS 0
47
#endif
48
#endif
49

50
#if !CK_USE_CC_BUILTINS
51
#if defined(__x86_64__)
52
#include "gcc/x86_64/ck_pr.h"
53
#elif defined(__x86__)
54
#include "gcc/x86/ck_pr.h"
55
#elif defined(__sparcv9__)
56
#include "gcc/sparcv9/ck_pr.h"
57
#elif defined(__ppc64__)
58
#include "gcc/ppc64/ck_pr.h"
59
#elif defined(__s390x__)
60
#include "gcc/s390x/ck_pr.h"
61
#elif defined(__ppc__)
62
#include "gcc/ppc/ck_pr.h"
63
#elif defined(__arm__)
64
#include "gcc/arm/ck_pr.h"
65
#elif defined(__aarch64__)
66
#include "gcc/aarch64/ck_pr.h"
67
#elif !defined(__GNUC__)
68
#error Your platform is unsupported
69
#endif
70
#endif /* !CK_USE_CC_BUILTINS */
71

72
#if defined(__GNUC__)
73
#include "gcc/ck_pr.h"
74
#endif
75

76
#define CK_PR_FENCE_EMIT(T)			\
77
	CK_CC_INLINE static void		\
78
	ck_pr_fence_##T(void)			\
79
	{					\
80
		ck_pr_fence_strict_##T();	\
81
		return;				\
82
	}
83
#define CK_PR_FENCE_NOOP(T)			\
84
	CK_CC_INLINE static void		\
85
	ck_pr_fence_##T(void)			\
86
	{					\
87
		ck_pr_barrier();		\
88
		return;				\
89
	}
90

91
/*
92
 * None of the currently supported platforms allow for data-dependent
93
 * load ordering.
94
 */
95
CK_PR_FENCE_NOOP(load_depends)
96
#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
97

98
/*
99
 * In memory models where atomic operations do not have serializing
100
 * effects, atomic read-modify-write operations are modeled as stores.
101
 */
102
#if defined(CK_MD_RMO)
103
/*
104
 * Only stores to the same location have a global
105
 * ordering.
106
 */
107
CK_PR_FENCE_EMIT(atomic)
108
CK_PR_FENCE_EMIT(atomic_load)
109
CK_PR_FENCE_EMIT(atomic_store)
110
CK_PR_FENCE_EMIT(store_atomic)
111
CK_PR_FENCE_EMIT(load_atomic)
112
CK_PR_FENCE_EMIT(load_store)
113
CK_PR_FENCE_EMIT(store_load)
114
CK_PR_FENCE_EMIT(load)
115
CK_PR_FENCE_EMIT(store)
116
CK_PR_FENCE_EMIT(memory)
117
CK_PR_FENCE_EMIT(acquire)
118
CK_PR_FENCE_EMIT(release)
119
CK_PR_FENCE_EMIT(acqrel)
120
CK_PR_FENCE_EMIT(lock)
121
CK_PR_FENCE_EMIT(unlock)
122
#elif defined(CK_MD_PSO)
123
/*
124
 * Anything can be re-ordered with respect to stores.
125
 * Otherwise, loads are executed in-order.
126
 */
127
CK_PR_FENCE_EMIT(atomic)
128
CK_PR_FENCE_NOOP(atomic_load)
129
CK_PR_FENCE_EMIT(atomic_store)
130
CK_PR_FENCE_EMIT(store_atomic)
131
CK_PR_FENCE_NOOP(load_atomic)
132
CK_PR_FENCE_EMIT(load_store)
133
CK_PR_FENCE_EMIT(store_load)
134
CK_PR_FENCE_NOOP(load)
135
CK_PR_FENCE_EMIT(store)
136
CK_PR_FENCE_EMIT(memory)
137
CK_PR_FENCE_EMIT(acquire)
138
CK_PR_FENCE_EMIT(release)
139
CK_PR_FENCE_EMIT(acqrel)
140
CK_PR_FENCE_EMIT(lock)
141
CK_PR_FENCE_EMIT(unlock)
142
#elif defined(CK_MD_TSO)
143
/*
144
 * Only loads are re-ordered and only with respect to
145
 * prior stores. Atomic operations are serializing.
146
 */
147
CK_PR_FENCE_NOOP(atomic)
148
CK_PR_FENCE_NOOP(atomic_load)
149
CK_PR_FENCE_NOOP(atomic_store)
150
CK_PR_FENCE_NOOP(store_atomic)
151
CK_PR_FENCE_NOOP(load_atomic)
152
CK_PR_FENCE_NOOP(load_store)
153
CK_PR_FENCE_EMIT(store_load)
154
CK_PR_FENCE_NOOP(load)
155
CK_PR_FENCE_NOOP(store)
156
CK_PR_FENCE_EMIT(memory)
157
CK_PR_FENCE_NOOP(acquire)
158
CK_PR_FENCE_NOOP(release)
159
CK_PR_FENCE_NOOP(acqrel)
160
CK_PR_FENCE_NOOP(lock)
161
CK_PR_FENCE_NOOP(unlock)
162
#else
163
#error "No memory model has been defined."
164
#endif /* CK_MD_TSO */
165

166
#undef CK_PR_FENCE_EMIT
167
#undef CK_PR_FENCE_NOOP
168

169
#ifndef CK_F_PR_RFO
170
#define CK_F_PR_RFO
171
CK_CC_INLINE static void
172
ck_pr_rfo(const void *m)
173
{
174

175
	(void)m;
176
	return;
177
}
178
#endif /* CK_F_PR_RFO */
179

180
#define CK_PR_STORE_SAFE(DST, VAL, TYPE)			\
181
    ck_pr_md_store_##TYPE(					\
182
        ((void)sizeof(*(DST) = (VAL)), (DST)),			\
183
        (VAL))
184

185
#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
186
#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
187
#ifndef CK_PR_DISABLE_DOUBLE
188
#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
189
#endif
190
#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
191
#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
192
#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
193
#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
194
#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
195

196
#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
197

198
#ifdef CK_F_PR_LOAD_64
199
#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
200
#endif /* CK_F_PR_LOAD_64 */
201

202
#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
203
#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
204

205
#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
206
#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
207
#ifndef CK_PR_DISABLE_DOUBLE
208
#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
209
#endif
210
#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
211
#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
212
#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
213
#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
214
#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
215

216
#ifdef CK_F_PR_LOAD_64
217
#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
218
#endif /* CK_F_PR_LOAD_64 */
219

220
#define CK_PR_BIN(K, S, M, T, P, C)					\
221
	CK_CC_INLINE static void					\
222
	ck_pr_##K##_##S(M *target, T value)				\
223
	{								\
224
		T previous;						\
225
		C punt;							\
226
		punt = ck_pr_md_load_##S(target);			\
227
		previous = (T)punt;					\
228
		while (ck_pr_cas_##S##_value(target,			\
229
					     (C)previous,		\
230
					     (C)(previous P value),	\
231
					     &previous) == false)	\
232
			ck_pr_stall();					\
233
									\
234
		return;							\
235
	}
236

237
#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
238

239
#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
240

241
#ifndef CK_F_PR_ADD_CHAR
242
#define CK_F_PR_ADD_CHAR
243
CK_PR_BIN_S(add, char, char, +)
244
#endif /* CK_F_PR_ADD_CHAR */
245

246
#ifndef CK_F_PR_SUB_CHAR
247
#define CK_F_PR_SUB_CHAR
248
CK_PR_BIN_S(sub, char, char, -)
249
#endif /* CK_F_PR_SUB_CHAR */
250

251
#ifndef CK_F_PR_AND_CHAR
252
#define CK_F_PR_AND_CHAR
253
CK_PR_BIN_S(and, char, char, &)
254
#endif /* CK_F_PR_AND_CHAR */
255

256
#ifndef CK_F_PR_XOR_CHAR
257
#define CK_F_PR_XOR_CHAR
258
CK_PR_BIN_S(xor, char, char, ^)
259
#endif /* CK_F_PR_XOR_CHAR */
260

261
#ifndef CK_F_PR_OR_CHAR
262
#define CK_F_PR_OR_CHAR
263
CK_PR_BIN_S(or, char, char, |)
264
#endif /* CK_F_PR_OR_CHAR */
265

266
#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
267

268
#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
269

270
#ifndef CK_F_PR_ADD_INT
271
#define CK_F_PR_ADD_INT
272
CK_PR_BIN_S(add, int, int, +)
273
#endif /* CK_F_PR_ADD_INT */
274

275
#ifndef CK_F_PR_SUB_INT
276
#define CK_F_PR_SUB_INT
277
CK_PR_BIN_S(sub, int, int, -)
278
#endif /* CK_F_PR_SUB_INT */
279

280
#ifndef CK_F_PR_AND_INT
281
#define CK_F_PR_AND_INT
282
CK_PR_BIN_S(and, int, int, &)
283
#endif /* CK_F_PR_AND_INT */
284

285
#ifndef CK_F_PR_XOR_INT
286
#define CK_F_PR_XOR_INT
287
CK_PR_BIN_S(xor, int, int, ^)
288
#endif /* CK_F_PR_XOR_INT */
289

290
#ifndef CK_F_PR_OR_INT
291
#define CK_F_PR_OR_INT
292
CK_PR_BIN_S(or, int, int, |)
293
#endif /* CK_F_PR_OR_INT */
294

295
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
296

297
#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
298
	    !defined(CK_PR_DISABLE_DOUBLE)
299

300
#ifndef CK_F_PR_ADD_DOUBLE
301
#define CK_F_PR_ADD_DOUBLE
302
CK_PR_BIN_S(add, double, double, +)
303
#endif /* CK_F_PR_ADD_DOUBLE */
304

305
#ifndef CK_F_PR_SUB_DOUBLE
306
#define CK_F_PR_SUB_DOUBLE
307
CK_PR_BIN_S(sub, double, double, -)
308
#endif /* CK_F_PR_SUB_DOUBLE */
309

310
#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
311

312
#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
313

314
#ifndef CK_F_PR_ADD_UINT
315
#define CK_F_PR_ADD_UINT
316
CK_PR_BIN_S(add, uint, unsigned int, +)
317
#endif /* CK_F_PR_ADD_UINT */
318

319
#ifndef CK_F_PR_SUB_UINT
320
#define CK_F_PR_SUB_UINT
321
CK_PR_BIN_S(sub, uint, unsigned int, -)
322
#endif /* CK_F_PR_SUB_UINT */
323

324
#ifndef CK_F_PR_AND_UINT
325
#define CK_F_PR_AND_UINT
326
CK_PR_BIN_S(and, uint, unsigned int, &)
327
#endif /* CK_F_PR_AND_UINT */
328

329
#ifndef CK_F_PR_XOR_UINT
330
#define CK_F_PR_XOR_UINT
331
CK_PR_BIN_S(xor, uint, unsigned int, ^)
332
#endif /* CK_F_PR_XOR_UINT */
333

334
#ifndef CK_F_PR_OR_UINT
335
#define CK_F_PR_OR_UINT
336
CK_PR_BIN_S(or, uint, unsigned int, |)
337
#endif /* CK_F_PR_OR_UINT */
338

339
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
340

341
#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
342

343
#ifndef CK_F_PR_ADD_PTR
344
#define CK_F_PR_ADD_PTR
345
CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
346
#endif /* CK_F_PR_ADD_PTR */
347

348
#ifndef CK_F_PR_SUB_PTR
349
#define CK_F_PR_SUB_PTR
350
CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
351
#endif /* CK_F_PR_SUB_PTR */
352

353
#ifndef CK_F_PR_AND_PTR
354
#define CK_F_PR_AND_PTR
355
CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
356
#endif /* CK_F_PR_AND_PTR */
357

358
#ifndef CK_F_PR_XOR_PTR
359
#define CK_F_PR_XOR_PTR
360
CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
361
#endif /* CK_F_PR_XOR_PTR */
362

363
#ifndef CK_F_PR_OR_PTR
364
#define CK_F_PR_OR_PTR
365
CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
366
#endif /* CK_F_PR_OR_PTR */
367

368
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
369

370
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
371

372
#ifndef CK_F_PR_ADD_64
373
#define CK_F_PR_ADD_64
374
CK_PR_BIN_S(add, 64, uint64_t, +)
375
#endif /* CK_F_PR_ADD_64 */
376

377
#ifndef CK_F_PR_SUB_64
378
#define CK_F_PR_SUB_64
379
CK_PR_BIN_S(sub, 64, uint64_t, -)
380
#endif /* CK_F_PR_SUB_64 */
381

382
#ifndef CK_F_PR_AND_64
383
#define CK_F_PR_AND_64
384
CK_PR_BIN_S(and, 64, uint64_t, &)
385
#endif /* CK_F_PR_AND_64 */
386

387
#ifndef CK_F_PR_XOR_64
388
#define CK_F_PR_XOR_64
389
CK_PR_BIN_S(xor, 64, uint64_t, ^)
390
#endif /* CK_F_PR_XOR_64 */
391

392
#ifndef CK_F_PR_OR_64
393
#define CK_F_PR_OR_64
394
CK_PR_BIN_S(or, 64, uint64_t, |)
395
#endif /* CK_F_PR_OR_64 */
396

397
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
398

399
#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
400

401
#ifndef CK_F_PR_ADD_32
402
#define CK_F_PR_ADD_32
403
CK_PR_BIN_S(add, 32, uint32_t, +)
404
#endif /* CK_F_PR_ADD_32 */
405

406
#ifndef CK_F_PR_SUB_32
407
#define CK_F_PR_SUB_32
408
CK_PR_BIN_S(sub, 32, uint32_t, -)
409
#endif /* CK_F_PR_SUB_32 */
410

411
#ifndef CK_F_PR_AND_32
412
#define CK_F_PR_AND_32
413
CK_PR_BIN_S(and, 32, uint32_t, &)
414
#endif /* CK_F_PR_AND_32 */
415

416
#ifndef CK_F_PR_XOR_32
417
#define CK_F_PR_XOR_32
418
CK_PR_BIN_S(xor, 32, uint32_t, ^)
419
#endif /* CK_F_PR_XOR_32 */
420

421
#ifndef CK_F_PR_OR_32
422
#define CK_F_PR_OR_32
423
CK_PR_BIN_S(or, 32, uint32_t, |)
424
#endif /* CK_F_PR_OR_32 */
425

426
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
427

428
#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
429

430
#ifndef CK_F_PR_ADD_16
431
#define CK_F_PR_ADD_16
432
CK_PR_BIN_S(add, 16, uint16_t, +)
433
#endif /* CK_F_PR_ADD_16 */
434

435
#ifndef CK_F_PR_SUB_16
436
#define CK_F_PR_SUB_16
437
CK_PR_BIN_S(sub, 16, uint16_t, -)
438
#endif /* CK_F_PR_SUB_16 */
439

440
#ifndef CK_F_PR_AND_16
441
#define CK_F_PR_AND_16
442
CK_PR_BIN_S(and, 16, uint16_t, &)
443
#endif /* CK_F_PR_AND_16 */
444

445
#ifndef CK_F_PR_XOR_16
446
#define CK_F_PR_XOR_16
447
CK_PR_BIN_S(xor, 16, uint16_t, ^)
448
#endif /* CK_F_PR_XOR_16 */
449

450
#ifndef CK_F_PR_OR_16
451
#define CK_F_PR_OR_16
452
CK_PR_BIN_S(or, 16, uint16_t, |)
453
#endif /* CK_F_PR_OR_16 */
454

455
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
456

457
#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
458

459
#ifndef CK_F_PR_ADD_8
460
#define CK_F_PR_ADD_8
461
CK_PR_BIN_S(add, 8, uint8_t, +)
462
#endif /* CK_F_PR_ADD_8 */
463

464
#ifndef CK_F_PR_SUB_8
465
#define CK_F_PR_SUB_8
466
CK_PR_BIN_S(sub, 8, uint8_t, -)
467
#endif /* CK_F_PR_SUB_8 */
468

469
#ifndef CK_F_PR_AND_8
470
#define CK_F_PR_AND_8
471
CK_PR_BIN_S(and, 8, uint8_t, &)
472
#endif /* CK_F_PR_AND_8 */
473

474
#ifndef CK_F_PR_XOR_8
475
#define CK_F_PR_XOR_8
476
CK_PR_BIN_S(xor, 8, uint8_t, ^)
477
#endif /* CK_F_PR_XOR_8 */
478

479
#ifndef CK_F_PR_OR_8
480
#define CK_F_PR_OR_8
481
CK_PR_BIN_S(or, 8, uint8_t, |)
482
#endif /* CK_F_PR_OR_8 */
483

484
#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
485

486
#undef CK_PR_BIN_S
487
#undef CK_PR_BIN
488

489
#define CK_PR_BTX(K, S, M, T, P, C, R)						   \
490
	CK_CC_INLINE static bool						   \
491
	ck_pr_##K##_##S(M *target, unsigned int offset)				   \
492
	{									   \
493
		T previous;							   \
494
		C punt;								   \
495
		punt = ck_pr_md_load_##S(target);				   \
496
		previous = (T)punt;						   \
497
		while (ck_pr_cas_##S##_value(target, (C)previous,		   \
498
			(C)(previous P (R ((T)1 << offset))), &previous) == false) \
499
				ck_pr_stall();					   \
500
		return ((previous >> offset) & 1);				   \
501
	}
502

503
#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
504

505
#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
506

507
#ifndef CK_F_PR_BTC_INT
508
#define CK_F_PR_BTC_INT
509
CK_PR_BTX_S(btc, int, int, ^,)
510
#endif /* CK_F_PR_BTC_INT */
511

512
#ifndef CK_F_PR_BTR_INT
513
#define CK_F_PR_BTR_INT
514
CK_PR_BTX_S(btr, int, int, &, ~)
515
#endif /* CK_F_PR_BTR_INT */
516

517
#ifndef CK_F_PR_BTS_INT
518
#define CK_F_PR_BTS_INT
519
CK_PR_BTX_S(bts, int, int, |,)
520
#endif /* CK_F_PR_BTS_INT */
521

522
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
523

524
#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
525

526
#ifndef CK_F_PR_BTC_UINT
527
#define CK_F_PR_BTC_UINT
528
CK_PR_BTX_S(btc, uint, unsigned int, ^,)
529
#endif /* CK_F_PR_BTC_UINT */
530

531
#ifndef CK_F_PR_BTR_UINT
532
#define CK_F_PR_BTR_UINT
533
CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
534
#endif /* CK_F_PR_BTR_UINT */
535

536
#ifndef CK_F_PR_BTS_UINT
537
#define CK_F_PR_BTS_UINT
538
CK_PR_BTX_S(bts, uint, unsigned int, |,)
539
#endif /* CK_F_PR_BTS_UINT */
540

541
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
542

543
#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
544

545
#ifndef CK_F_PR_BTC_PTR
546
#define CK_F_PR_BTC_PTR
547
CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
548
#endif /* CK_F_PR_BTC_PTR */
549

550
#ifndef CK_F_PR_BTR_PTR
551
#define CK_F_PR_BTR_PTR
552
CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
553
#endif /* CK_F_PR_BTR_PTR */
554

555
#ifndef CK_F_PR_BTS_PTR
556
#define CK_F_PR_BTS_PTR
557
CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
558
#endif /* CK_F_PR_BTS_PTR */
559

560
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
561

562
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
563

564
#ifndef CK_F_PR_BTC_64
565
#define CK_F_PR_BTC_64
566
CK_PR_BTX_S(btc, 64, uint64_t, ^,)
567
#endif /* CK_F_PR_BTC_64 */
568

569
#ifndef CK_F_PR_BTR_64
570
#define CK_F_PR_BTR_64
571
CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
572
#endif /* CK_F_PR_BTR_64 */
573

574
#ifndef CK_F_PR_BTS_64
575
#define CK_F_PR_BTS_64
576
CK_PR_BTX_S(bts, 64, uint64_t, |,)
577
#endif /* CK_F_PR_BTS_64 */
578

579
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
580

581
#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
582

583
#ifndef CK_F_PR_BTC_32
584
#define CK_F_PR_BTC_32
585
CK_PR_BTX_S(btc, 32, uint32_t, ^,)
586
#endif /* CK_F_PR_BTC_32 */
587

588
#ifndef CK_F_PR_BTR_32
589
#define CK_F_PR_BTR_32
590
CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
591
#endif /* CK_F_PR_BTR_32 */
592

593
#ifndef CK_F_PR_BTS_32
594
#define CK_F_PR_BTS_32
595
CK_PR_BTX_S(bts, 32, uint32_t, |,)
596
#endif /* CK_F_PR_BTS_32 */
597

598
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
599

600
#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
601

602
#ifndef CK_F_PR_BTC_16
603
#define CK_F_PR_BTC_16
604
CK_PR_BTX_S(btc, 16, uint16_t, ^,)
605
#endif /* CK_F_PR_BTC_16 */
606

607
#ifndef CK_F_PR_BTR_16
608
#define CK_F_PR_BTR_16
609
CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
610
#endif /* CK_F_PR_BTR_16 */
611

612
#ifndef CK_F_PR_BTS_16
613
#define CK_F_PR_BTS_16
614
CK_PR_BTX_S(bts, 16, uint16_t, |,)
615
#endif /* CK_F_PR_BTS_16 */
616

617
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
618

619
#undef CK_PR_BTX_S
620
#undef CK_PR_BTX
621

622
#define CK_PR_UNARY(K, X, S, M, T)					\
623
	CK_CC_INLINE static void					\
624
	ck_pr_##K##_##S(M *target)					\
625
	{								\
626
		ck_pr_##X##_##S(target, (T)1);				\
627
		return;							\
628
	}
629

630
#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z)				\
631
	CK_CC_INLINE static bool					\
632
	ck_pr_##K##_##S##_is_zero(M *target)				\
633
	{								\
634
		T previous;						\
635
		C punt;							\
636
		punt = (C)ck_pr_md_load_##S(target);			\
637
		previous = (T)punt;					\
638
		while (ck_pr_cas_##S##_value(target,			\
639
					     (C)previous,		\
640
					     (C)(previous P 1),		\
641
					     &previous) == false)	\
642
			ck_pr_stall();					\
643
		return previous == (T)Z;				\
644
        }
645

646
#define CK_PR_UNARY_Z_STUB(K, S, M)					\
647
	CK_CC_INLINE static void					\
648
	ck_pr_##K##_##S##_zero(M *target, bool *zero)			\
649
	{								\
650
		*zero = ck_pr_##K##_##S##_is_zero(target);		\
651
		return;							\
652
	}
653

654
#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
655
#define CK_PR_UNARY_Z_S(K, S, M, P, Z)          \
656
        CK_PR_UNARY_Z(K, S, M, M, P, M, Z)      \
657
        CK_PR_UNARY_Z_STUB(K, S, M)
658

659
#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
660

661
#ifndef CK_F_PR_INC_CHAR
662
#define CK_F_PR_INC_CHAR
663
CK_PR_UNARY_S(inc, add, char, char)
664
#endif /* CK_F_PR_INC_CHAR */
665

666
#ifndef CK_F_PR_INC_CHAR_ZERO
667
#define CK_F_PR_INC_CHAR_ZERO
668
CK_PR_UNARY_Z_S(inc, char, char, +, -1)
669
#else
670
CK_PR_UNARY_Z_STUB(inc, char, char)
671
#endif /* CK_F_PR_INC_CHAR_ZERO */
672

673
#ifndef CK_F_PR_DEC_CHAR
674
#define CK_F_PR_DEC_CHAR
675
CK_PR_UNARY_S(dec, sub, char, char)
676
#endif /* CK_F_PR_DEC_CHAR */
677

678
#ifndef CK_F_PR_DEC_CHAR_ZERO
679
#define CK_F_PR_DEC_CHAR_ZERO
680
CK_PR_UNARY_Z_S(dec, char, char, -, 1)
681
#else
682
CK_PR_UNARY_Z_STUB(dec, char, char)
683
#endif /* CK_F_PR_DEC_CHAR_ZERO */
684

685
#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
686

687
#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
688

689
#ifndef CK_F_PR_INC_INT
690
#define CK_F_PR_INC_INT
691
CK_PR_UNARY_S(inc, add, int, int)
692
#endif /* CK_F_PR_INC_INT */
693

694
#ifndef CK_F_PR_INC_INT_ZERO
695
#define CK_F_PR_INC_INT_ZERO
696
CK_PR_UNARY_Z_S(inc, int, int, +, -1)
697
#else
698
CK_PR_UNARY_Z_STUB(inc, int, int)
699
#endif /* CK_F_PR_INC_INT_ZERO */
700

701
#ifndef CK_F_PR_DEC_INT
702
#define CK_F_PR_DEC_INT
703
CK_PR_UNARY_S(dec, sub, int, int)
704
#endif /* CK_F_PR_DEC_INT */
705

706
#ifndef CK_F_PR_DEC_INT_ZERO
707
#define CK_F_PR_DEC_INT_ZERO
708
CK_PR_UNARY_Z_S(dec, int, int, -, 1)
709
#else
710
CK_PR_UNARY_Z_STUB(dec, int, int)
711
#endif /* CK_F_PR_DEC_INT_ZERO */
712

713
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
714

715
#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
716
	    !defined(CK_PR_DISABLE_DOUBLE)
717

718
#ifndef CK_F_PR_INC_DOUBLE
719
#define CK_F_PR_INC_DOUBLE
720
CK_PR_UNARY_S(inc, add, double, double)
721
#endif /* CK_F_PR_INC_DOUBLE */
722

723
#ifndef CK_F_PR_DEC_DOUBLE
724
#define CK_F_PR_DEC_DOUBLE
725
CK_PR_UNARY_S(dec, sub, double, double)
726
#endif /* CK_F_PR_DEC_DOUBLE */
727

728
#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
729

730
#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
731

732
#ifndef CK_F_PR_INC_UINT
733
#define CK_F_PR_INC_UINT
734
CK_PR_UNARY_S(inc, add, uint, unsigned int)
735
#endif /* CK_F_PR_INC_UINT */
736

737
#ifndef CK_F_PR_INC_UINT_ZERO
738
#define CK_F_PR_INC_UINT_ZERO
739
CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
740
#else
741
CK_PR_UNARY_Z_STUB(inc, uint, unsigned int)
742
#endif /* CK_F_PR_INC_UINT_ZERO */
743

744
#ifndef CK_F_PR_DEC_UINT
745
#define CK_F_PR_DEC_UINT
746
CK_PR_UNARY_S(dec, sub, uint, unsigned int)
747
#endif /* CK_F_PR_DEC_UINT */
748

749
#ifndef CK_F_PR_DEC_UINT_ZERO
750
#define CK_F_PR_DEC_UINT_ZERO
751
CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
752
#else
753
CK_PR_UNARY_Z_STUB(dec, uint, unsigned int)
754
#endif /* CK_F_PR_DEC_UINT_ZERO */
755

756
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
757

758
#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
759

760
#ifndef CK_F_PR_INC_PTR
761
#define CK_F_PR_INC_PTR
762
CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
763
#endif /* CK_F_PR_INC_PTR */
764

765
#ifndef CK_F_PR_INC_PTR_ZERO
766
#define CK_F_PR_INC_PTR_ZERO
767
CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
768
#else
769
CK_PR_UNARY_Z_STUB(inc, ptr, void)
770
#endif /* CK_F_PR_INC_PTR_ZERO */
771

772
#ifndef CK_F_PR_DEC_PTR
773
#define CK_F_PR_DEC_PTR
774
CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
775
#endif /* CK_F_PR_DEC_PTR */
776

777
#ifndef CK_F_PR_DEC_PTR_ZERO
778
#define CK_F_PR_DEC_PTR_ZERO
779
CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
780
#else
781
CK_PR_UNARY_Z_STUB(dec, ptr, void)
782
#endif /* CK_F_PR_DEC_PTR_ZERO */
783

784
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
785

786
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
787

788
#ifndef CK_F_PR_INC_64
789
#define CK_F_PR_INC_64
790
CK_PR_UNARY_S(inc, add, 64, uint64_t)
791
#endif /* CK_F_PR_INC_64 */
792

793
#ifndef CK_F_PR_INC_64_ZERO
794
#define CK_F_PR_INC_64_ZERO
795
CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
796
#else
797
CK_PR_UNARY_Z_STUB(inc, 64, uint64_t)
798
#endif /* CK_F_PR_INC_64_ZERO */
799

800
#ifndef CK_F_PR_DEC_64
801
#define CK_F_PR_DEC_64
802
CK_PR_UNARY_S(dec, sub, 64, uint64_t)
803
#endif /* CK_F_PR_DEC_64 */
804

805
#ifndef CK_F_PR_DEC_64_ZERO
806
#define CK_F_PR_DEC_64_ZERO
807
CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
808
#else
809
CK_PR_UNARY_Z_STUB(dec, 64, uint64_t)
810
#endif /* CK_F_PR_DEC_64_ZERO */
811

812
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
813

814
#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
815

816
#ifndef CK_F_PR_INC_32
817
#define CK_F_PR_INC_32
818
CK_PR_UNARY_S(inc, add, 32, uint32_t)
819
#endif /* CK_F_PR_INC_32 */
820

821
#ifndef CK_F_PR_INC_32_ZERO
822
#define CK_F_PR_INC_32_ZERO
823
CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
824
#else
825
CK_PR_UNARY_Z_STUB(inc, 32, uint32_t)
826
#endif /* CK_F_PR_INC_32_ZERO */
827

828
#ifndef CK_F_PR_DEC_32
829
#define CK_F_PR_DEC_32
830
CK_PR_UNARY_S(dec, sub, 32, uint32_t)
831
#endif /* CK_F_PR_DEC_32 */
832

833
#ifndef CK_F_PR_DEC_32_ZERO
834
#define CK_F_PR_DEC_32_ZERO
835
CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
836
#else
837
CK_PR_UNARY_Z_STUB(dec, 32, uint32_t)
838
#endif /* CK_F_PR_DEC_32_ZERO */
839

840
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
841

842
#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
843

844
#ifndef CK_F_PR_INC_16
845
#define CK_F_PR_INC_16
846
CK_PR_UNARY_S(inc, add, 16, uint16_t)
847
#endif /* CK_F_PR_INC_16 */
848

849
#ifndef CK_F_PR_INC_16_ZERO
850
#define CK_F_PR_INC_16_ZERO
851
CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
852
#else
853
CK_PR_UNARY_Z_STUB(inc, 16, uint16_t)
854
#endif /* CK_F_PR_INC_16_ZERO */
855

856
#ifndef CK_F_PR_DEC_16
857
#define CK_F_PR_DEC_16
858
CK_PR_UNARY_S(dec, sub, 16, uint16_t)
859
#endif /* CK_F_PR_DEC_16 */
860

861
#ifndef CK_F_PR_DEC_16_ZERO
862
#define CK_F_PR_DEC_16_ZERO
863
CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
864
#else
865
CK_PR_UNARY_Z_STUB(dec, 16, uint16_t)
866
#endif /* CK_F_PR_DEC_16_ZERO */
867

868
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
869

870
#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
871

872
#ifndef CK_F_PR_INC_8
873
#define CK_F_PR_INC_8
874
CK_PR_UNARY_S(inc, add, 8, uint8_t)
875
#endif /* CK_F_PR_INC_8 */
876

877
#ifndef CK_F_PR_INC_8_ZERO
878
#define CK_F_PR_INC_8_ZERO
879
CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
880
#else
881
CK_PR_UNARY_Z_STUB(inc, 8, uint8_t)
882
#endif /* CK_F_PR_INC_8_ZERO */
883

884
#ifndef CK_F_PR_DEC_8
885
#define CK_F_PR_DEC_8
886
CK_PR_UNARY_S(dec, sub, 8, uint8_t)
887
#endif /* CK_F_PR_DEC_8 */
888

889
#ifndef CK_F_PR_DEC_8_ZERO
890
#define CK_F_PR_DEC_8_ZERO
891
CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
892
#else
893
CK_PR_UNARY_Z_STUB(dec, 8, uint8_t)
894
#endif /* CK_F_PR_DEC_8_ZERO */
895

896
#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
897

898
#undef CK_PR_UNARY_Z_S
899
#undef CK_PR_UNARY_S
900
#undef CK_PR_UNARY_Z
901
#undef CK_PR_UNARY
902

903
#define CK_PR_N(K, S, M, T, P, C)					\
904
	CK_CC_INLINE static void					\
905
	ck_pr_##K##_##S(M *target)					\
906
	{								\
907
		T previous;						\
908
		C punt;							\
909
		punt = (C)ck_pr_md_load_##S(target);			\
910
		previous = (T)punt;					\
911
		while (ck_pr_cas_##S##_value(target,			\
912
					     (C)previous,		\
913
					     (C)(P previous),		\
914
					     &previous) == false)	\
915
			ck_pr_stall();					\
916
									\
917
		return;							\
918
	}
919

920
#define CK_PR_N_Z(S, M, T, C)						\
921
	CK_CC_INLINE static void					\
922
	ck_pr_neg_##S##_zero(M *target, bool *zero)			\
923
	{								\
924
		T previous;						\
925
		C punt;							\
926
		punt = (C)ck_pr_md_load_##S(target);			\
927
		previous = (T)punt;					\
928
		while (ck_pr_cas_##S##_value(target,			\
929
					     (C)previous,		\
930
					     (C)(-previous),		\
931
					     &previous) == false)	\
932
			ck_pr_stall();					\
933
									\
934
		*zero = previous == 0;					\
935
		return;							\
936
	}
937

938
#define CK_PR_N_S(K, S, M, P)	CK_PR_N(K, S, M, M, P, M)
939
#define CK_PR_N_Z_S(S, M) 	CK_PR_N_Z(S, M, M, M)
940

941
#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
942

943
#ifndef CK_F_PR_NOT_CHAR
944
#define CK_F_PR_NOT_CHAR
945
CK_PR_N_S(not, char, char, ~)
946
#endif /* CK_F_PR_NOT_CHAR */
947

948
#ifndef CK_F_PR_NEG_CHAR
949
#define CK_F_PR_NEG_CHAR
950
CK_PR_N_S(neg, char, char, -)
951
#endif /* CK_F_PR_NEG_CHAR */
952

953
#ifndef CK_F_PR_NEG_CHAR_ZERO
954
#define CK_F_PR_NEG_CHAR_ZERO
955
CK_PR_N_Z_S(char, char)
956
#endif /* CK_F_PR_NEG_CHAR_ZERO */
957

958
#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
959

960
#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
961

962
#ifndef CK_F_PR_NOT_INT
963
#define CK_F_PR_NOT_INT
964
CK_PR_N_S(not, int, int, ~)
965
#endif /* CK_F_PR_NOT_INT */
966

967
#ifndef CK_F_PR_NEG_INT
968
#define CK_F_PR_NEG_INT
969
CK_PR_N_S(neg, int, int, -)
970
#endif /* CK_F_PR_NEG_INT */
971

972
#ifndef CK_F_PR_NEG_INT_ZERO
973
#define CK_F_PR_NEG_INT_ZERO
974
CK_PR_N_Z_S(int, int)
975
#endif /* CK_F_PR_NEG_INT_ZERO */
976

977
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
978

979
#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
980
	    !defined(CK_PR_DISABLE_DOUBLE)
981

982
#ifndef CK_F_PR_NEG_DOUBLE
983
#define CK_F_PR_NEG_DOUBLE
984
CK_PR_N_S(neg, double, double, -)
985
#endif /* CK_F_PR_NEG_DOUBLE */
986

987
#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
988

989
#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
990

991
#ifndef CK_F_PR_NOT_UINT
992
#define CK_F_PR_NOT_UINT
993
CK_PR_N_S(not, uint, unsigned int, ~)
994
#endif /* CK_F_PR_NOT_UINT */
995

996
#ifndef CK_F_PR_NEG_UINT
997
#define CK_F_PR_NEG_UINT
998
CK_PR_N_S(neg, uint, unsigned int, -)
999
#endif /* CK_F_PR_NEG_UINT */
1000

1001
#ifndef CK_F_PR_NEG_UINT_ZERO
1002
#define CK_F_PR_NEG_UINT_ZERO
1003
CK_PR_N_Z_S(uint, unsigned int)
1004
#endif /* CK_F_PR_NEG_UINT_ZERO */
1005

1006
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1007

1008
#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1009

1010
#ifndef CK_F_PR_NOT_PTR
1011
#define CK_F_PR_NOT_PTR
1012
CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
1013
#endif /* CK_F_PR_NOT_PTR */
1014

1015
#ifndef CK_F_PR_NEG_PTR
1016
#define CK_F_PR_NEG_PTR
1017
CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
1018
#endif /* CK_F_PR_NEG_PTR */
1019

1020
#ifndef CK_F_PR_NEG_PTR_ZERO
1021
#define CK_F_PR_NEG_PTR_ZERO
1022
CK_PR_N_Z(ptr, void, uintptr_t, void *)
1023
#endif /* CK_F_PR_NEG_PTR_ZERO */
1024

1025
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1026

1027
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1028

1029
#ifndef CK_F_PR_NOT_64
1030
#define CK_F_PR_NOT_64
1031
CK_PR_N_S(not, 64, uint64_t, ~)
1032
#endif /* CK_F_PR_NOT_64 */
1033

1034
#ifndef CK_F_PR_NEG_64
1035
#define CK_F_PR_NEG_64
1036
CK_PR_N_S(neg, 64, uint64_t, -)
1037
#endif /* CK_F_PR_NEG_64 */
1038

1039
#ifndef CK_F_PR_NEG_64_ZERO
1040
#define CK_F_PR_NEG_64_ZERO
1041
CK_PR_N_Z_S(64, uint64_t)
1042
#endif /* CK_F_PR_NEG_64_ZERO */
1043

1044
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1045

1046
#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1047

1048
#ifndef CK_F_PR_NOT_32
1049
#define CK_F_PR_NOT_32
1050
CK_PR_N_S(not, 32, uint32_t, ~)
1051
#endif /* CK_F_PR_NOT_32 */
1052

1053
#ifndef CK_F_PR_NEG_32
1054
#define CK_F_PR_NEG_32
1055
CK_PR_N_S(neg, 32, uint32_t, -)
1056
#endif /* CK_F_PR_NEG_32 */
1057

1058
#ifndef CK_F_PR_NEG_32_ZERO
1059
#define CK_F_PR_NEG_32_ZERO
1060
CK_PR_N_Z_S(32, uint32_t)
1061
#endif /* CK_F_PR_NEG_32_ZERO */
1062

1063
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1064

1065
#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1066

1067
#ifndef CK_F_PR_NOT_16
1068
#define CK_F_PR_NOT_16
1069
CK_PR_N_S(not, 16, uint16_t, ~)
1070
#endif /* CK_F_PR_NOT_16 */
1071

1072
#ifndef CK_F_PR_NEG_16
1073
#define CK_F_PR_NEG_16
1074
CK_PR_N_S(neg, 16, uint16_t, -)
1075
#endif /* CK_F_PR_NEG_16 */
1076

1077
#ifndef CK_F_PR_NEG_16_ZERO
1078
#define CK_F_PR_NEG_16_ZERO
1079
CK_PR_N_Z_S(16, uint16_t)
1080
#endif /* CK_F_PR_NEG_16_ZERO */
1081

1082
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1083

1084
#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1085

1086
#ifndef CK_F_PR_NOT_8
1087
#define CK_F_PR_NOT_8
1088
CK_PR_N_S(not, 8, uint8_t, ~)
1089
#endif /* CK_F_PR_NOT_8 */
1090

1091
#ifndef CK_F_PR_NEG_8
1092
#define CK_F_PR_NEG_8
1093
CK_PR_N_S(neg, 8, uint8_t, -)
1094
#endif /* CK_F_PR_NEG_8 */
1095

1096
#ifndef CK_F_PR_NEG_8_ZERO
1097
#define CK_F_PR_NEG_8_ZERO
1098
CK_PR_N_Z_S(8, uint8_t)
1099
#endif /* CK_F_PR_NEG_8_ZERO */
1100

1101
#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1102

1103
#undef CK_PR_N_Z_S
1104
#undef CK_PR_N_S
1105
#undef CK_PR_N_Z
1106
#undef CK_PR_N
1107

1108
#define CK_PR_FAA(S, M, T, C)						\
1109
	CK_CC_INLINE static C						\
1110
	ck_pr_faa_##S(M *target, T delta)				\
1111
	{								\
1112
		T previous;						\
1113
		C punt;							\
1114
		punt = (C)ck_pr_md_load_##S(target);			\
1115
		previous = (T)punt;					\
1116
		while (ck_pr_cas_##S##_value(target,			\
1117
					     (C)previous,		\
1118
					     (C)(previous + delta),	\
1119
					     &previous) == false)	\
1120
			ck_pr_stall();					\
1121
									\
1122
		return ((C)previous);					\
1123
	}
1124

1125
#define CK_PR_FAS(S, M, C)						\
1126
	CK_CC_INLINE static C						\
1127
	ck_pr_fas_##S(M *target, C update)				\
1128
	{								\
1129
		C previous;						\
1130
		previous = ck_pr_md_load_##S(target);			\
1131
		while (ck_pr_cas_##S##_value(target,			\
1132
					     previous,			\
1133
					     update,			\
1134
					     &previous) == false)	\
1135
			ck_pr_stall();					\
1136
									\
1137
		return (previous);					\
1138
	}
1139

1140
#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1141
#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1142

1143
#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1144

1145
#ifndef CK_F_PR_FAA_CHAR
1146
#define CK_F_PR_FAA_CHAR
1147
CK_PR_FAA_S(char, char)
1148
#endif /* CK_F_PR_FAA_CHAR */
1149

1150
#ifndef CK_F_PR_FAS_CHAR
1151
#define CK_F_PR_FAS_CHAR
1152
CK_PR_FAS_S(char, char)
1153
#endif /* CK_F_PR_FAS_CHAR */
1154

1155
#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1156

1157
#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1158

1159
#ifndef CK_F_PR_FAA_INT
1160
#define CK_F_PR_FAA_INT
1161
CK_PR_FAA_S(int, int)
1162
#endif /* CK_F_PR_FAA_INT */
1163

1164
#ifndef CK_F_PR_FAS_INT
1165
#define CK_F_PR_FAS_INT
1166
CK_PR_FAS_S(int, int)
1167
#endif /* CK_F_PR_FAS_INT */
1168

1169
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1170

1171
#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
1172
	    !defined(CK_PR_DISABLE_DOUBLE)
1173

1174
#ifndef CK_F_PR_FAA_DOUBLE
1175
#define CK_F_PR_FAA_DOUBLE
1176
CK_PR_FAA_S(double, double)
1177
#endif /* CK_F_PR_FAA_DOUBLE */
1178

1179
#ifndef CK_F_PR_FAS_DOUBLE
1180
#define CK_F_PR_FAS_DOUBLE
1181
CK_PR_FAS_S(double, double)
1182
#endif /* CK_F_PR_FAS_DOUBLE */
1183

1184
#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
1185

1186
#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1187

1188
#ifndef CK_F_PR_FAA_UINT
1189
#define CK_F_PR_FAA_UINT
1190
CK_PR_FAA_S(uint, unsigned int)
1191
#endif /* CK_F_PR_FAA_UINT */
1192

1193
#ifndef CK_F_PR_FAS_UINT
1194
#define CK_F_PR_FAS_UINT
1195
CK_PR_FAS_S(uint, unsigned int)
1196
#endif /* CK_F_PR_FAS_UINT */
1197

1198
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1199

1200
#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1201

1202
#ifndef CK_F_PR_FAA_PTR
1203
#define CK_F_PR_FAA_PTR
1204
CK_PR_FAA(ptr, void, uintptr_t, void *)
1205
#endif /* CK_F_PR_FAA_PTR */
1206

1207
#ifndef CK_F_PR_FAS_PTR
1208
#define CK_F_PR_FAS_PTR
1209
CK_PR_FAS(ptr, void, void *)
1210
#endif /* CK_F_PR_FAS_PTR */
1211

1212
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1213

1214
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1215

1216
#ifndef CK_F_PR_FAA_64
1217
#define CK_F_PR_FAA_64
1218
CK_PR_FAA_S(64, uint64_t)
1219
#endif /* CK_F_PR_FAA_64 */
1220

1221
#ifndef CK_F_PR_FAS_64
1222
#define CK_F_PR_FAS_64
1223
CK_PR_FAS_S(64, uint64_t)
1224
#endif /* CK_F_PR_FAS_64 */
1225

1226
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1227

1228
#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1229

1230
#ifndef CK_F_PR_FAA_32
1231
#define CK_F_PR_FAA_32
1232
CK_PR_FAA_S(32, uint32_t)
1233
#endif /* CK_F_PR_FAA_32 */
1234

1235
#ifndef CK_F_PR_FAS_32
1236
#define CK_F_PR_FAS_32
1237
CK_PR_FAS_S(32, uint32_t)
1238
#endif /* CK_F_PR_FAS_32 */
1239

1240
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1241

1242
#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1243

1244
#ifndef CK_F_PR_FAA_16
1245
#define CK_F_PR_FAA_16
1246
CK_PR_FAA_S(16, uint16_t)
1247
#endif /* CK_F_PR_FAA_16 */
1248

1249
#ifndef CK_F_PR_FAS_16
1250
#define CK_F_PR_FAS_16
1251
CK_PR_FAS_S(16, uint16_t)
1252
#endif /* CK_F_PR_FAS_16 */
1253

1254
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1255

1256
#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1257

1258
#ifndef CK_F_PR_FAA_8
1259
#define CK_F_PR_FAA_8
1260
CK_PR_FAA_S(8, uint8_t)
1261
#endif /* CK_F_PR_FAA_8 */
1262

1263
#ifndef CK_F_PR_FAS_8
1264
#define CK_F_PR_FAS_8
1265
CK_PR_FAS_S(8, uint8_t)
1266
#endif /* CK_F_PR_FAS_8 */
1267

1268
#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1269

1270
#undef CK_PR_FAA_S
1271
#undef CK_PR_FAS_S
1272
#undef CK_PR_FAA
1273
#undef CK_PR_FAS
1274

1275
#endif /* CK_PR_H */
1276

1277
Product

Resources

Company
