Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
freebsd
GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/crypto/openssl/amd64/aesni-x86_64.S
39482 views
1
/* Do not modify. This file is auto-generated from aesni-x86_64.pl. */
2
.text
3
4
.globl aesni_encrypt
5
.type aesni_encrypt,@function
6
.align 16
7
aesni_encrypt:
8
.cfi_startproc
9
.byte 243,15,30,250
10
movups (%rdi),%xmm2
11
movl 240(%rdx),%eax
12
movups (%rdx),%xmm0
13
movups 16(%rdx),%xmm1
14
leaq 32(%rdx),%rdx
15
xorps %xmm0,%xmm2
16
.Loop_enc1_1:
17
.byte 102,15,56,220,209
18
decl %eax
19
movups (%rdx),%xmm1
20
leaq 16(%rdx),%rdx
21
jnz .Loop_enc1_1
22
.byte 102,15,56,221,209
23
pxor %xmm0,%xmm0
24
pxor %xmm1,%xmm1
25
movups %xmm2,(%rsi)
26
pxor %xmm2,%xmm2
27
.byte 0xf3,0xc3
28
.cfi_endproc
29
.size aesni_encrypt,.-aesni_encrypt
30
31
.globl aesni_decrypt
32
.type aesni_decrypt,@function
33
.align 16
34
aesni_decrypt:
35
.cfi_startproc
36
.byte 243,15,30,250
37
movups (%rdi),%xmm2
38
movl 240(%rdx),%eax
39
movups (%rdx),%xmm0
40
movups 16(%rdx),%xmm1
41
leaq 32(%rdx),%rdx
42
xorps %xmm0,%xmm2
43
.Loop_dec1_2:
44
.byte 102,15,56,222,209
45
decl %eax
46
movups (%rdx),%xmm1
47
leaq 16(%rdx),%rdx
48
jnz .Loop_dec1_2
49
.byte 102,15,56,223,209
50
pxor %xmm0,%xmm0
51
pxor %xmm1,%xmm1
52
movups %xmm2,(%rsi)
53
pxor %xmm2,%xmm2
54
.byte 0xf3,0xc3
55
.cfi_endproc
56
.size aesni_decrypt, .-aesni_decrypt
57
.type _aesni_encrypt2,@function
58
.align 16
59
_aesni_encrypt2:
60
.cfi_startproc
61
movups (%rcx),%xmm0
62
shll $4,%eax
63
movups 16(%rcx),%xmm1
64
xorps %xmm0,%xmm2
65
xorps %xmm0,%xmm3
66
movups 32(%rcx),%xmm0
67
leaq 32(%rcx,%rax,1),%rcx
68
negq %rax
69
addq $16,%rax
70
71
.Lenc_loop2:
72
.byte 102,15,56,220,209
73
.byte 102,15,56,220,217
74
movups (%rcx,%rax,1),%xmm1
75
addq $32,%rax
76
.byte 102,15,56,220,208
77
.byte 102,15,56,220,216
78
movups -16(%rcx,%rax,1),%xmm0
79
jnz .Lenc_loop2
80
81
.byte 102,15,56,220,209
82
.byte 102,15,56,220,217
83
.byte 102,15,56,221,208
84
.byte 102,15,56,221,216
85
.byte 0xf3,0xc3
86
.cfi_endproc
87
.size _aesni_encrypt2,.-_aesni_encrypt2
88
.type _aesni_decrypt2,@function
89
.align 16
90
_aesni_decrypt2:
91
.cfi_startproc
92
movups (%rcx),%xmm0
93
shll $4,%eax
94
movups 16(%rcx),%xmm1
95
xorps %xmm0,%xmm2
96
xorps %xmm0,%xmm3
97
movups 32(%rcx),%xmm0
98
leaq 32(%rcx,%rax,1),%rcx
99
negq %rax
100
addq $16,%rax
101
102
.Ldec_loop2:
103
.byte 102,15,56,222,209
104
.byte 102,15,56,222,217
105
movups (%rcx,%rax,1),%xmm1
106
addq $32,%rax
107
.byte 102,15,56,222,208
108
.byte 102,15,56,222,216
109
movups -16(%rcx,%rax,1),%xmm0
110
jnz .Ldec_loop2
111
112
.byte 102,15,56,222,209
113
.byte 102,15,56,222,217
114
.byte 102,15,56,223,208
115
.byte 102,15,56,223,216
116
.byte 0xf3,0xc3
117
.cfi_endproc
118
.size _aesni_decrypt2,.-_aesni_decrypt2
119
.type _aesni_encrypt3,@function
120
.align 16
121
_aesni_encrypt3:
122
.cfi_startproc
123
movups (%rcx),%xmm0
124
shll $4,%eax
125
movups 16(%rcx),%xmm1
126
xorps %xmm0,%xmm2
127
xorps %xmm0,%xmm3
128
xorps %xmm0,%xmm4
129
movups 32(%rcx),%xmm0
130
leaq 32(%rcx,%rax,1),%rcx
131
negq %rax
132
addq $16,%rax
133
134
.Lenc_loop3:
135
.byte 102,15,56,220,209
136
.byte 102,15,56,220,217
137
.byte 102,15,56,220,225
138
movups (%rcx,%rax,1),%xmm1
139
addq $32,%rax
140
.byte 102,15,56,220,208
141
.byte 102,15,56,220,216
142
.byte 102,15,56,220,224
143
movups -16(%rcx,%rax,1),%xmm0
144
jnz .Lenc_loop3
145
146
.byte 102,15,56,220,209
147
.byte 102,15,56,220,217
148
.byte 102,15,56,220,225
149
.byte 102,15,56,221,208
150
.byte 102,15,56,221,216
151
.byte 102,15,56,221,224
152
.byte 0xf3,0xc3
153
.cfi_endproc
154
.size _aesni_encrypt3,.-_aesni_encrypt3
155
.type _aesni_decrypt3,@function
156
.align 16
157
_aesni_decrypt3:
158
.cfi_startproc
159
movups (%rcx),%xmm0
160
shll $4,%eax
161
movups 16(%rcx),%xmm1
162
xorps %xmm0,%xmm2
163
xorps %xmm0,%xmm3
164
xorps %xmm0,%xmm4
165
movups 32(%rcx),%xmm0
166
leaq 32(%rcx,%rax,1),%rcx
167
negq %rax
168
addq $16,%rax
169
170
.Ldec_loop3:
171
.byte 102,15,56,222,209
172
.byte 102,15,56,222,217
173
.byte 102,15,56,222,225
174
movups (%rcx,%rax,1),%xmm1
175
addq $32,%rax
176
.byte 102,15,56,222,208
177
.byte 102,15,56,222,216
178
.byte 102,15,56,222,224
179
movups -16(%rcx,%rax,1),%xmm0
180
jnz .Ldec_loop3
181
182
.byte 102,15,56,222,209
183
.byte 102,15,56,222,217
184
.byte 102,15,56,222,225
185
.byte 102,15,56,223,208
186
.byte 102,15,56,223,216
187
.byte 102,15,56,223,224
188
.byte 0xf3,0xc3
189
.cfi_endproc
190
.size _aesni_decrypt3,.-_aesni_decrypt3
191
.type _aesni_encrypt4,@function
192
.align 16
193
_aesni_encrypt4:
194
.cfi_startproc
195
movups (%rcx),%xmm0
196
shll $4,%eax
197
movups 16(%rcx),%xmm1
198
xorps %xmm0,%xmm2
199
xorps %xmm0,%xmm3
200
xorps %xmm0,%xmm4
201
xorps %xmm0,%xmm5
202
movups 32(%rcx),%xmm0
203
leaq 32(%rcx,%rax,1),%rcx
204
negq %rax
205
.byte 0x0f,0x1f,0x00
206
addq $16,%rax
207
208
.Lenc_loop4:
209
.byte 102,15,56,220,209
210
.byte 102,15,56,220,217
211
.byte 102,15,56,220,225
212
.byte 102,15,56,220,233
213
movups (%rcx,%rax,1),%xmm1
214
addq $32,%rax
215
.byte 102,15,56,220,208
216
.byte 102,15,56,220,216
217
.byte 102,15,56,220,224
218
.byte 102,15,56,220,232
219
movups -16(%rcx,%rax,1),%xmm0
220
jnz .Lenc_loop4
221
222
.byte 102,15,56,220,209
223
.byte 102,15,56,220,217
224
.byte 102,15,56,220,225
225
.byte 102,15,56,220,233
226
.byte 102,15,56,221,208
227
.byte 102,15,56,221,216
228
.byte 102,15,56,221,224
229
.byte 102,15,56,221,232
230
.byte 0xf3,0xc3
231
.cfi_endproc
232
.size _aesni_encrypt4,.-_aesni_encrypt4
233
.type _aesni_decrypt4,@function
234
.align 16
235
_aesni_decrypt4:
236
.cfi_startproc
237
movups (%rcx),%xmm0
238
shll $4,%eax
239
movups 16(%rcx),%xmm1
240
xorps %xmm0,%xmm2
241
xorps %xmm0,%xmm3
242
xorps %xmm0,%xmm4
243
xorps %xmm0,%xmm5
244
movups 32(%rcx),%xmm0
245
leaq 32(%rcx,%rax,1),%rcx
246
negq %rax
247
.byte 0x0f,0x1f,0x00
248
addq $16,%rax
249
250
.Ldec_loop4:
251
.byte 102,15,56,222,209
252
.byte 102,15,56,222,217
253
.byte 102,15,56,222,225
254
.byte 102,15,56,222,233
255
movups (%rcx,%rax,1),%xmm1
256
addq $32,%rax
257
.byte 102,15,56,222,208
258
.byte 102,15,56,222,216
259
.byte 102,15,56,222,224
260
.byte 102,15,56,222,232
261
movups -16(%rcx,%rax,1),%xmm0
262
jnz .Ldec_loop4
263
264
.byte 102,15,56,222,209
265
.byte 102,15,56,222,217
266
.byte 102,15,56,222,225
267
.byte 102,15,56,222,233
268
.byte 102,15,56,223,208
269
.byte 102,15,56,223,216
270
.byte 102,15,56,223,224
271
.byte 102,15,56,223,232
272
.byte 0xf3,0xc3
273
.cfi_endproc
274
.size _aesni_decrypt4,.-_aesni_decrypt4
275
.type _aesni_encrypt6,@function
276
.align 16
277
_aesni_encrypt6:
278
.cfi_startproc
279
movups (%rcx),%xmm0
280
shll $4,%eax
281
movups 16(%rcx),%xmm1
282
xorps %xmm0,%xmm2
283
pxor %xmm0,%xmm3
284
pxor %xmm0,%xmm4
285
.byte 102,15,56,220,209
286
leaq 32(%rcx,%rax,1),%rcx
287
negq %rax
288
.byte 102,15,56,220,217
289
pxor %xmm0,%xmm5
290
pxor %xmm0,%xmm6
291
.byte 102,15,56,220,225
292
pxor %xmm0,%xmm7
293
movups (%rcx,%rax,1),%xmm0
294
addq $16,%rax
295
jmp .Lenc_loop6_enter
296
.align 16
297
.Lenc_loop6:
298
.byte 102,15,56,220,209
299
.byte 102,15,56,220,217
300
.byte 102,15,56,220,225
301
.Lenc_loop6_enter:
302
.byte 102,15,56,220,233
303
.byte 102,15,56,220,241
304
.byte 102,15,56,220,249
305
movups (%rcx,%rax,1),%xmm1
306
addq $32,%rax
307
.byte 102,15,56,220,208
308
.byte 102,15,56,220,216
309
.byte 102,15,56,220,224
310
.byte 102,15,56,220,232
311
.byte 102,15,56,220,240
312
.byte 102,15,56,220,248
313
movups -16(%rcx,%rax,1),%xmm0
314
jnz .Lenc_loop6
315
316
.byte 102,15,56,220,209
317
.byte 102,15,56,220,217
318
.byte 102,15,56,220,225
319
.byte 102,15,56,220,233
320
.byte 102,15,56,220,241
321
.byte 102,15,56,220,249
322
.byte 102,15,56,221,208
323
.byte 102,15,56,221,216
324
.byte 102,15,56,221,224
325
.byte 102,15,56,221,232
326
.byte 102,15,56,221,240
327
.byte 102,15,56,221,248
328
.byte 0xf3,0xc3
329
.cfi_endproc
330
.size _aesni_encrypt6,.-_aesni_encrypt6
331
.type _aesni_decrypt6,@function
332
.align 16
333
_aesni_decrypt6:
334
.cfi_startproc
335
movups (%rcx),%xmm0
336
shll $4,%eax
337
movups 16(%rcx),%xmm1
338
xorps %xmm0,%xmm2
339
pxor %xmm0,%xmm3
340
pxor %xmm0,%xmm4
341
.byte 102,15,56,222,209
342
leaq 32(%rcx,%rax,1),%rcx
343
negq %rax
344
.byte 102,15,56,222,217
345
pxor %xmm0,%xmm5
346
pxor %xmm0,%xmm6
347
.byte 102,15,56,222,225
348
pxor %xmm0,%xmm7
349
movups (%rcx,%rax,1),%xmm0
350
addq $16,%rax
351
jmp .Ldec_loop6_enter
352
.align 16
353
.Ldec_loop6:
354
.byte 102,15,56,222,209
355
.byte 102,15,56,222,217
356
.byte 102,15,56,222,225
357
.Ldec_loop6_enter:
358
.byte 102,15,56,222,233
359
.byte 102,15,56,222,241
360
.byte 102,15,56,222,249
361
movups (%rcx,%rax,1),%xmm1
362
addq $32,%rax
363
.byte 102,15,56,222,208
364
.byte 102,15,56,222,216
365
.byte 102,15,56,222,224
366
.byte 102,15,56,222,232
367
.byte 102,15,56,222,240
368
.byte 102,15,56,222,248
369
movups -16(%rcx,%rax,1),%xmm0
370
jnz .Ldec_loop6
371
372
.byte 102,15,56,222,209
373
.byte 102,15,56,222,217
374
.byte 102,15,56,222,225
375
.byte 102,15,56,222,233
376
.byte 102,15,56,222,241
377
.byte 102,15,56,222,249
378
.byte 102,15,56,223,208
379
.byte 102,15,56,223,216
380
.byte 102,15,56,223,224
381
.byte 102,15,56,223,232
382
.byte 102,15,56,223,240
383
.byte 102,15,56,223,248
384
.byte 0xf3,0xc3
385
.cfi_endproc
386
.size _aesni_decrypt6,.-_aesni_decrypt6
387
.type _aesni_encrypt8,@function
388
.align 16
389
_aesni_encrypt8:
390
.cfi_startproc
391
movups (%rcx),%xmm0
392
shll $4,%eax
393
movups 16(%rcx),%xmm1
394
xorps %xmm0,%xmm2
395
xorps %xmm0,%xmm3
396
pxor %xmm0,%xmm4
397
pxor %xmm0,%xmm5
398
pxor %xmm0,%xmm6
399
leaq 32(%rcx,%rax,1),%rcx
400
negq %rax
401
.byte 102,15,56,220,209
402
pxor %xmm0,%xmm7
403
pxor %xmm0,%xmm8
404
.byte 102,15,56,220,217
405
pxor %xmm0,%xmm9
406
movups (%rcx,%rax,1),%xmm0
407
addq $16,%rax
408
jmp .Lenc_loop8_inner
409
.align 16
410
.Lenc_loop8:
411
.byte 102,15,56,220,209
412
.byte 102,15,56,220,217
413
.Lenc_loop8_inner:
414
.byte 102,15,56,220,225
415
.byte 102,15,56,220,233
416
.byte 102,15,56,220,241
417
.byte 102,15,56,220,249
418
.byte 102,68,15,56,220,193
419
.byte 102,68,15,56,220,201
420
.Lenc_loop8_enter:
421
movups (%rcx,%rax,1),%xmm1
422
addq $32,%rax
423
.byte 102,15,56,220,208
424
.byte 102,15,56,220,216
425
.byte 102,15,56,220,224
426
.byte 102,15,56,220,232
427
.byte 102,15,56,220,240
428
.byte 102,15,56,220,248
429
.byte 102,68,15,56,220,192
430
.byte 102,68,15,56,220,200
431
movups -16(%rcx,%rax,1),%xmm0
432
jnz .Lenc_loop8
433
434
.byte 102,15,56,220,209
435
.byte 102,15,56,220,217
436
.byte 102,15,56,220,225
437
.byte 102,15,56,220,233
438
.byte 102,15,56,220,241
439
.byte 102,15,56,220,249
440
.byte 102,68,15,56,220,193
441
.byte 102,68,15,56,220,201
442
.byte 102,15,56,221,208
443
.byte 102,15,56,221,216
444
.byte 102,15,56,221,224
445
.byte 102,15,56,221,232
446
.byte 102,15,56,221,240
447
.byte 102,15,56,221,248
448
.byte 102,68,15,56,221,192
449
.byte 102,68,15,56,221,200
450
.byte 0xf3,0xc3
451
.cfi_endproc
452
.size _aesni_encrypt8,.-_aesni_encrypt8
453
.type _aesni_decrypt8,@function
454
.align 16
455
_aesni_decrypt8:
456
.cfi_startproc
457
movups (%rcx),%xmm0
458
shll $4,%eax
459
movups 16(%rcx),%xmm1
460
xorps %xmm0,%xmm2
461
xorps %xmm0,%xmm3
462
pxor %xmm0,%xmm4
463
pxor %xmm0,%xmm5
464
pxor %xmm0,%xmm6
465
leaq 32(%rcx,%rax,1),%rcx
466
negq %rax
467
.byte 102,15,56,222,209
468
pxor %xmm0,%xmm7
469
pxor %xmm0,%xmm8
470
.byte 102,15,56,222,217
471
pxor %xmm0,%xmm9
472
movups (%rcx,%rax,1),%xmm0
473
addq $16,%rax
474
jmp .Ldec_loop8_inner
475
.align 16
476
.Ldec_loop8:
477
.byte 102,15,56,222,209
478
.byte 102,15,56,222,217
479
.Ldec_loop8_inner:
480
.byte 102,15,56,222,225
481
.byte 102,15,56,222,233
482
.byte 102,15,56,222,241
483
.byte 102,15,56,222,249
484
.byte 102,68,15,56,222,193
485
.byte 102,68,15,56,222,201
486
.Ldec_loop8_enter:
487
movups (%rcx,%rax,1),%xmm1
488
addq $32,%rax
489
.byte 102,15,56,222,208
490
.byte 102,15,56,222,216
491
.byte 102,15,56,222,224
492
.byte 102,15,56,222,232
493
.byte 102,15,56,222,240
494
.byte 102,15,56,222,248
495
.byte 102,68,15,56,222,192
496
.byte 102,68,15,56,222,200
497
movups -16(%rcx,%rax,1),%xmm0
498
jnz .Ldec_loop8
499
500
.byte 102,15,56,222,209
501
.byte 102,15,56,222,217
502
.byte 102,15,56,222,225
503
.byte 102,15,56,222,233
504
.byte 102,15,56,222,241
505
.byte 102,15,56,222,249
506
.byte 102,68,15,56,222,193
507
.byte 102,68,15,56,222,201
508
.byte 102,15,56,223,208
509
.byte 102,15,56,223,216
510
.byte 102,15,56,223,224
511
.byte 102,15,56,223,232
512
.byte 102,15,56,223,240
513
.byte 102,15,56,223,248
514
.byte 102,68,15,56,223,192
515
.byte 102,68,15,56,223,200
516
.byte 0xf3,0xc3
517
.cfi_endproc
518
.size _aesni_decrypt8,.-_aesni_decrypt8
519
.globl aesni_ecb_encrypt
520
.type aesni_ecb_encrypt,@function
521
.align 16
522
aesni_ecb_encrypt:
523
.cfi_startproc
524
.byte 243,15,30,250
525
andq $-16,%rdx
526
jz .Lecb_ret
527
528
movl 240(%rcx),%eax
529
movups (%rcx),%xmm0
530
movq %rcx,%r11
531
movl %eax,%r10d
532
testl %r8d,%r8d
533
jz .Lecb_decrypt
534
535
cmpq $0x80,%rdx
536
jb .Lecb_enc_tail
537
538
movdqu (%rdi),%xmm2
539
movdqu 16(%rdi),%xmm3
540
movdqu 32(%rdi),%xmm4
541
movdqu 48(%rdi),%xmm5
542
movdqu 64(%rdi),%xmm6
543
movdqu 80(%rdi),%xmm7
544
movdqu 96(%rdi),%xmm8
545
movdqu 112(%rdi),%xmm9
546
leaq 128(%rdi),%rdi
547
subq $0x80,%rdx
548
jmp .Lecb_enc_loop8_enter
549
.align 16
550
.Lecb_enc_loop8:
551
movups %xmm2,(%rsi)
552
movq %r11,%rcx
553
movdqu (%rdi),%xmm2
554
movl %r10d,%eax
555
movups %xmm3,16(%rsi)
556
movdqu 16(%rdi),%xmm3
557
movups %xmm4,32(%rsi)
558
movdqu 32(%rdi),%xmm4
559
movups %xmm5,48(%rsi)
560
movdqu 48(%rdi),%xmm5
561
movups %xmm6,64(%rsi)
562
movdqu 64(%rdi),%xmm6
563
movups %xmm7,80(%rsi)
564
movdqu 80(%rdi),%xmm7
565
movups %xmm8,96(%rsi)
566
movdqu 96(%rdi),%xmm8
567
movups %xmm9,112(%rsi)
568
leaq 128(%rsi),%rsi
569
movdqu 112(%rdi),%xmm9
570
leaq 128(%rdi),%rdi
571
.Lecb_enc_loop8_enter:
572
573
call _aesni_encrypt8
574
575
subq $0x80,%rdx
576
jnc .Lecb_enc_loop8
577
578
movups %xmm2,(%rsi)
579
movq %r11,%rcx
580
movups %xmm3,16(%rsi)
581
movl %r10d,%eax
582
movups %xmm4,32(%rsi)
583
movups %xmm5,48(%rsi)
584
movups %xmm6,64(%rsi)
585
movups %xmm7,80(%rsi)
586
movups %xmm8,96(%rsi)
587
movups %xmm9,112(%rsi)
588
leaq 128(%rsi),%rsi
589
addq $0x80,%rdx
590
jz .Lecb_ret
591
592
.Lecb_enc_tail:
593
movups (%rdi),%xmm2
594
cmpq $0x20,%rdx
595
jb .Lecb_enc_one
596
movups 16(%rdi),%xmm3
597
je .Lecb_enc_two
598
movups 32(%rdi),%xmm4
599
cmpq $0x40,%rdx
600
jb .Lecb_enc_three
601
movups 48(%rdi),%xmm5
602
je .Lecb_enc_four
603
movups 64(%rdi),%xmm6
604
cmpq $0x60,%rdx
605
jb .Lecb_enc_five
606
movups 80(%rdi),%xmm7
607
je .Lecb_enc_six
608
movdqu 96(%rdi),%xmm8
609
xorps %xmm9,%xmm9
610
call _aesni_encrypt8
611
movups %xmm2,(%rsi)
612
movups %xmm3,16(%rsi)
613
movups %xmm4,32(%rsi)
614
movups %xmm5,48(%rsi)
615
movups %xmm6,64(%rsi)
616
movups %xmm7,80(%rsi)
617
movups %xmm8,96(%rsi)
618
jmp .Lecb_ret
619
.align 16
620
.Lecb_enc_one:
621
movups (%rcx),%xmm0
622
movups 16(%rcx),%xmm1
623
leaq 32(%rcx),%rcx
624
xorps %xmm0,%xmm2
625
.Loop_enc1_3:
626
.byte 102,15,56,220,209
627
decl %eax
628
movups (%rcx),%xmm1
629
leaq 16(%rcx),%rcx
630
jnz .Loop_enc1_3
631
.byte 102,15,56,221,209
632
movups %xmm2,(%rsi)
633
jmp .Lecb_ret
634
.align 16
635
.Lecb_enc_two:
636
call _aesni_encrypt2
637
movups %xmm2,(%rsi)
638
movups %xmm3,16(%rsi)
639
jmp .Lecb_ret
640
.align 16
641
.Lecb_enc_three:
642
call _aesni_encrypt3
643
movups %xmm2,(%rsi)
644
movups %xmm3,16(%rsi)
645
movups %xmm4,32(%rsi)
646
jmp .Lecb_ret
647
.align 16
648
.Lecb_enc_four:
649
call _aesni_encrypt4
650
movups %xmm2,(%rsi)
651
movups %xmm3,16(%rsi)
652
movups %xmm4,32(%rsi)
653
movups %xmm5,48(%rsi)
654
jmp .Lecb_ret
655
.align 16
656
.Lecb_enc_five:
657
xorps %xmm7,%xmm7
658
call _aesni_encrypt6
659
movups %xmm2,(%rsi)
660
movups %xmm3,16(%rsi)
661
movups %xmm4,32(%rsi)
662
movups %xmm5,48(%rsi)
663
movups %xmm6,64(%rsi)
664
jmp .Lecb_ret
665
.align 16
666
.Lecb_enc_six:
667
call _aesni_encrypt6
668
movups %xmm2,(%rsi)
669
movups %xmm3,16(%rsi)
670
movups %xmm4,32(%rsi)
671
movups %xmm5,48(%rsi)
672
movups %xmm6,64(%rsi)
673
movups %xmm7,80(%rsi)
674
jmp .Lecb_ret
675
676
.align 16
677
.Lecb_decrypt:
678
cmpq $0x80,%rdx
679
jb .Lecb_dec_tail
680
681
movdqu (%rdi),%xmm2
682
movdqu 16(%rdi),%xmm3
683
movdqu 32(%rdi),%xmm4
684
movdqu 48(%rdi),%xmm5
685
movdqu 64(%rdi),%xmm6
686
movdqu 80(%rdi),%xmm7
687
movdqu 96(%rdi),%xmm8
688
movdqu 112(%rdi),%xmm9
689
leaq 128(%rdi),%rdi
690
subq $0x80,%rdx
691
jmp .Lecb_dec_loop8_enter
692
.align 16
693
.Lecb_dec_loop8:
694
movups %xmm2,(%rsi)
695
movq %r11,%rcx
696
movdqu (%rdi),%xmm2
697
movl %r10d,%eax
698
movups %xmm3,16(%rsi)
699
movdqu 16(%rdi),%xmm3
700
movups %xmm4,32(%rsi)
701
movdqu 32(%rdi),%xmm4
702
movups %xmm5,48(%rsi)
703
movdqu 48(%rdi),%xmm5
704
movups %xmm6,64(%rsi)
705
movdqu 64(%rdi),%xmm6
706
movups %xmm7,80(%rsi)
707
movdqu 80(%rdi),%xmm7
708
movups %xmm8,96(%rsi)
709
movdqu 96(%rdi),%xmm8
710
movups %xmm9,112(%rsi)
711
leaq 128(%rsi),%rsi
712
movdqu 112(%rdi),%xmm9
713
leaq 128(%rdi),%rdi
714
.Lecb_dec_loop8_enter:
715
716
call _aesni_decrypt8
717
718
movups (%r11),%xmm0
719
subq $0x80,%rdx
720
jnc .Lecb_dec_loop8
721
722
movups %xmm2,(%rsi)
723
pxor %xmm2,%xmm2
724
movq %r11,%rcx
725
movups %xmm3,16(%rsi)
726
pxor %xmm3,%xmm3
727
movl %r10d,%eax
728
movups %xmm4,32(%rsi)
729
pxor %xmm4,%xmm4
730
movups %xmm5,48(%rsi)
731
pxor %xmm5,%xmm5
732
movups %xmm6,64(%rsi)
733
pxor %xmm6,%xmm6
734
movups %xmm7,80(%rsi)
735
pxor %xmm7,%xmm7
736
movups %xmm8,96(%rsi)
737
pxor %xmm8,%xmm8
738
movups %xmm9,112(%rsi)
739
pxor %xmm9,%xmm9
740
leaq 128(%rsi),%rsi
741
addq $0x80,%rdx
742
jz .Lecb_ret
743
744
.Lecb_dec_tail:
745
movups (%rdi),%xmm2
746
cmpq $0x20,%rdx
747
jb .Lecb_dec_one
748
movups 16(%rdi),%xmm3
749
je .Lecb_dec_two
750
movups 32(%rdi),%xmm4
751
cmpq $0x40,%rdx
752
jb .Lecb_dec_three
753
movups 48(%rdi),%xmm5
754
je .Lecb_dec_four
755
movups 64(%rdi),%xmm6
756
cmpq $0x60,%rdx
757
jb .Lecb_dec_five
758
movups 80(%rdi),%xmm7
759
je .Lecb_dec_six
760
movups 96(%rdi),%xmm8
761
movups (%rcx),%xmm0
762
xorps %xmm9,%xmm9
763
call _aesni_decrypt8
764
movups %xmm2,(%rsi)
765
pxor %xmm2,%xmm2
766
movups %xmm3,16(%rsi)
767
pxor %xmm3,%xmm3
768
movups %xmm4,32(%rsi)
769
pxor %xmm4,%xmm4
770
movups %xmm5,48(%rsi)
771
pxor %xmm5,%xmm5
772
movups %xmm6,64(%rsi)
773
pxor %xmm6,%xmm6
774
movups %xmm7,80(%rsi)
775
pxor %xmm7,%xmm7
776
movups %xmm8,96(%rsi)
777
pxor %xmm8,%xmm8
778
pxor %xmm9,%xmm9
779
jmp .Lecb_ret
780
.align 16
781
.Lecb_dec_one:
782
movups (%rcx),%xmm0
783
movups 16(%rcx),%xmm1
784
leaq 32(%rcx),%rcx
785
xorps %xmm0,%xmm2
786
.Loop_dec1_4:
787
.byte 102,15,56,222,209
788
decl %eax
789
movups (%rcx),%xmm1
790
leaq 16(%rcx),%rcx
791
jnz .Loop_dec1_4
792
.byte 102,15,56,223,209
793
movups %xmm2,(%rsi)
794
pxor %xmm2,%xmm2
795
jmp .Lecb_ret
796
.align 16
797
.Lecb_dec_two:
798
call _aesni_decrypt2
799
movups %xmm2,(%rsi)
800
pxor %xmm2,%xmm2
801
movups %xmm3,16(%rsi)
802
pxor %xmm3,%xmm3
803
jmp .Lecb_ret
804
.align 16
805
.Lecb_dec_three:
806
call _aesni_decrypt3
807
movups %xmm2,(%rsi)
808
pxor %xmm2,%xmm2
809
movups %xmm3,16(%rsi)
810
pxor %xmm3,%xmm3
811
movups %xmm4,32(%rsi)
812
pxor %xmm4,%xmm4
813
jmp .Lecb_ret
814
.align 16
815
.Lecb_dec_four:
816
call _aesni_decrypt4
817
movups %xmm2,(%rsi)
818
pxor %xmm2,%xmm2
819
movups %xmm3,16(%rsi)
820
pxor %xmm3,%xmm3
821
movups %xmm4,32(%rsi)
822
pxor %xmm4,%xmm4
823
movups %xmm5,48(%rsi)
824
pxor %xmm5,%xmm5
825
jmp .Lecb_ret
826
.align 16
827
.Lecb_dec_five:
828
xorps %xmm7,%xmm7
829
call _aesni_decrypt6
830
movups %xmm2,(%rsi)
831
pxor %xmm2,%xmm2
832
movups %xmm3,16(%rsi)
833
pxor %xmm3,%xmm3
834
movups %xmm4,32(%rsi)
835
pxor %xmm4,%xmm4
836
movups %xmm5,48(%rsi)
837
pxor %xmm5,%xmm5
838
movups %xmm6,64(%rsi)
839
pxor %xmm6,%xmm6
840
pxor %xmm7,%xmm7
841
jmp .Lecb_ret
842
.align 16
843
.Lecb_dec_six:
844
call _aesni_decrypt6
845
movups %xmm2,(%rsi)
846
pxor %xmm2,%xmm2
847
movups %xmm3,16(%rsi)
848
pxor %xmm3,%xmm3
849
movups %xmm4,32(%rsi)
850
pxor %xmm4,%xmm4
851
movups %xmm5,48(%rsi)
852
pxor %xmm5,%xmm5
853
movups %xmm6,64(%rsi)
854
pxor %xmm6,%xmm6
855
movups %xmm7,80(%rsi)
856
pxor %xmm7,%xmm7
857
858
.Lecb_ret:
859
xorps %xmm0,%xmm0
860
pxor %xmm1,%xmm1
861
.byte 0xf3,0xc3
862
.cfi_endproc
863
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
864
.globl aesni_ccm64_encrypt_blocks
865
.type aesni_ccm64_encrypt_blocks,@function
866
.align 16
867
aesni_ccm64_encrypt_blocks:
868
.cfi_startproc
869
.byte 243,15,30,250
870
movl 240(%rcx),%eax
871
movdqu (%r8),%xmm6
872
movdqa .Lincrement64(%rip),%xmm9
873
movdqa .Lbswap_mask(%rip),%xmm7
874
875
shll $4,%eax
876
movl $16,%r10d
877
leaq 0(%rcx),%r11
878
movdqu (%r9),%xmm3
879
movdqa %xmm6,%xmm2
880
leaq 32(%rcx,%rax,1),%rcx
881
.byte 102,15,56,0,247
882
subq %rax,%r10
883
jmp .Lccm64_enc_outer
884
.align 16
885
.Lccm64_enc_outer:
886
movups (%r11),%xmm0
887
movq %r10,%rax
888
movups (%rdi),%xmm8
889
890
xorps %xmm0,%xmm2
891
movups 16(%r11),%xmm1
892
xorps %xmm8,%xmm0
893
xorps %xmm0,%xmm3
894
movups 32(%r11),%xmm0
895
896
.Lccm64_enc2_loop:
897
.byte 102,15,56,220,209
898
.byte 102,15,56,220,217
899
movups (%rcx,%rax,1),%xmm1
900
addq $32,%rax
901
.byte 102,15,56,220,208
902
.byte 102,15,56,220,216
903
movups -16(%rcx,%rax,1),%xmm0
904
jnz .Lccm64_enc2_loop
905
.byte 102,15,56,220,209
906
.byte 102,15,56,220,217
907
paddq %xmm9,%xmm6
908
decq %rdx
909
.byte 102,15,56,221,208
910
.byte 102,15,56,221,216
911
912
leaq 16(%rdi),%rdi
913
xorps %xmm2,%xmm8
914
movdqa %xmm6,%xmm2
915
movups %xmm8,(%rsi)
916
.byte 102,15,56,0,215
917
leaq 16(%rsi),%rsi
918
jnz .Lccm64_enc_outer
919
920
pxor %xmm0,%xmm0
921
pxor %xmm1,%xmm1
922
pxor %xmm2,%xmm2
923
movups %xmm3,(%r9)
924
pxor %xmm3,%xmm3
925
pxor %xmm8,%xmm8
926
pxor %xmm6,%xmm6
927
.byte 0xf3,0xc3
928
.cfi_endproc
929
.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
930
.globl aesni_ccm64_decrypt_blocks
931
.type aesni_ccm64_decrypt_blocks,@function
932
.align 16
933
aesni_ccm64_decrypt_blocks:
934
.cfi_startproc
935
.byte 243,15,30,250
936
movl 240(%rcx),%eax
937
movups (%r8),%xmm6
938
movdqu (%r9),%xmm3
939
movdqa .Lincrement64(%rip),%xmm9
940
movdqa .Lbswap_mask(%rip),%xmm7
941
942
movaps %xmm6,%xmm2
943
movl %eax,%r10d
944
movq %rcx,%r11
945
.byte 102,15,56,0,247
946
movups (%rcx),%xmm0
947
movups 16(%rcx),%xmm1
948
leaq 32(%rcx),%rcx
949
xorps %xmm0,%xmm2
950
.Loop_enc1_5:
951
.byte 102,15,56,220,209
952
decl %eax
953
movups (%rcx),%xmm1
954
leaq 16(%rcx),%rcx
955
jnz .Loop_enc1_5
956
.byte 102,15,56,221,209
957
shll $4,%r10d
958
movl $16,%eax
959
movups (%rdi),%xmm8
960
paddq %xmm9,%xmm6
961
leaq 16(%rdi),%rdi
962
subq %r10,%rax
963
leaq 32(%r11,%r10,1),%rcx
964
movq %rax,%r10
965
jmp .Lccm64_dec_outer
966
.align 16
967
.Lccm64_dec_outer:
968
xorps %xmm2,%xmm8
969
movdqa %xmm6,%xmm2
970
movups %xmm8,(%rsi)
971
leaq 16(%rsi),%rsi
972
.byte 102,15,56,0,215
973
974
subq $1,%rdx
975
jz .Lccm64_dec_break
976
977
movups (%r11),%xmm0
978
movq %r10,%rax
979
movups 16(%r11),%xmm1
980
xorps %xmm0,%xmm8
981
xorps %xmm0,%xmm2
982
xorps %xmm8,%xmm3
983
movups 32(%r11),%xmm0
984
jmp .Lccm64_dec2_loop
985
.align 16
986
.Lccm64_dec2_loop:
987
.byte 102,15,56,220,209
988
.byte 102,15,56,220,217
989
movups (%rcx,%rax,1),%xmm1
990
addq $32,%rax
991
.byte 102,15,56,220,208
992
.byte 102,15,56,220,216
993
movups -16(%rcx,%rax,1),%xmm0
994
jnz .Lccm64_dec2_loop
995
movups (%rdi),%xmm8
996
paddq %xmm9,%xmm6
997
.byte 102,15,56,220,209
998
.byte 102,15,56,220,217
999
.byte 102,15,56,221,208
1000
.byte 102,15,56,221,216
1001
leaq 16(%rdi),%rdi
1002
jmp .Lccm64_dec_outer
1003
1004
.align 16
1005
.Lccm64_dec_break:
1006
1007
movl 240(%r11),%eax
1008
movups (%r11),%xmm0
1009
movups 16(%r11),%xmm1
1010
xorps %xmm0,%xmm8
1011
leaq 32(%r11),%r11
1012
xorps %xmm8,%xmm3
1013
.Loop_enc1_6:
1014
.byte 102,15,56,220,217
1015
decl %eax
1016
movups (%r11),%xmm1
1017
leaq 16(%r11),%r11
1018
jnz .Loop_enc1_6
1019
.byte 102,15,56,221,217
1020
pxor %xmm0,%xmm0
1021
pxor %xmm1,%xmm1
1022
pxor %xmm2,%xmm2
1023
movups %xmm3,(%r9)
1024
pxor %xmm3,%xmm3
1025
pxor %xmm8,%xmm8
1026
pxor %xmm6,%xmm6
1027
.byte 0xf3,0xc3
1028
.cfi_endproc
1029
.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
1030
.globl aesni_ctr32_encrypt_blocks
1031
.type aesni_ctr32_encrypt_blocks,@function
1032
.align 16
1033
aesni_ctr32_encrypt_blocks:
1034
.cfi_startproc
1035
.byte 243,15,30,250
1036
cmpq $1,%rdx
1037
jne .Lctr32_bulk
1038
1039
1040
1041
movups (%r8),%xmm2
1042
movups (%rdi),%xmm3
1043
movl 240(%rcx),%edx
1044
movups (%rcx),%xmm0
1045
movups 16(%rcx),%xmm1
1046
leaq 32(%rcx),%rcx
1047
xorps %xmm0,%xmm2
1048
.Loop_enc1_7:
1049
.byte 102,15,56,220,209
1050
decl %edx
1051
movups (%rcx),%xmm1
1052
leaq 16(%rcx),%rcx
1053
jnz .Loop_enc1_7
1054
.byte 102,15,56,221,209
1055
pxor %xmm0,%xmm0
1056
pxor %xmm1,%xmm1
1057
xorps %xmm3,%xmm2
1058
pxor %xmm3,%xmm3
1059
movups %xmm2,(%rsi)
1060
xorps %xmm2,%xmm2
1061
jmp .Lctr32_epilogue
1062
1063
.align 16
1064
.Lctr32_bulk:
1065
leaq (%rsp),%r11
1066
.cfi_def_cfa_register %r11
1067
pushq %rbp
1068
.cfi_offset %rbp,-16
1069
subq $128,%rsp
1070
andq $-16,%rsp
1071
1072
1073
1074
1075
movdqu (%r8),%xmm2
1076
movdqu (%rcx),%xmm0
1077
movl 12(%r8),%r8d
1078
pxor %xmm0,%xmm2
1079
movl 12(%rcx),%ebp
1080
movdqa %xmm2,0(%rsp)
1081
bswapl %r8d
1082
movdqa %xmm2,%xmm3
1083
movdqa %xmm2,%xmm4
1084
movdqa %xmm2,%xmm5
1085
movdqa %xmm2,64(%rsp)
1086
movdqa %xmm2,80(%rsp)
1087
movdqa %xmm2,96(%rsp)
1088
movq %rdx,%r10
1089
movdqa %xmm2,112(%rsp)
1090
1091
leaq 1(%r8),%rax
1092
leaq 2(%r8),%rdx
1093
bswapl %eax
1094
bswapl %edx
1095
xorl %ebp,%eax
1096
xorl %ebp,%edx
1097
.byte 102,15,58,34,216,3
1098
leaq 3(%r8),%rax
1099
movdqa %xmm3,16(%rsp)
1100
.byte 102,15,58,34,226,3
1101
bswapl %eax
1102
movq %r10,%rdx
1103
leaq 4(%r8),%r10
1104
movdqa %xmm4,32(%rsp)
1105
xorl %ebp,%eax
1106
bswapl %r10d
1107
.byte 102,15,58,34,232,3
1108
xorl %ebp,%r10d
1109
movdqa %xmm5,48(%rsp)
1110
leaq 5(%r8),%r9
1111
movl %r10d,64+12(%rsp)
1112
bswapl %r9d
1113
leaq 6(%r8),%r10
1114
movl 240(%rcx),%eax
1115
xorl %ebp,%r9d
1116
bswapl %r10d
1117
movl %r9d,80+12(%rsp)
1118
xorl %ebp,%r10d
1119
leaq 7(%r8),%r9
1120
movl %r10d,96+12(%rsp)
1121
bswapl %r9d
1122
movl OPENSSL_ia32cap_P+4(%rip),%r10d
1123
xorl %ebp,%r9d
1124
andl $71303168,%r10d
1125
movl %r9d,112+12(%rsp)
1126
1127
movups 16(%rcx),%xmm1
1128
1129
movdqa 64(%rsp),%xmm6
1130
movdqa 80(%rsp),%xmm7
1131
1132
cmpq $8,%rdx
1133
jb .Lctr32_tail
1134
1135
subq $6,%rdx
1136
cmpl $4194304,%r10d
1137
je .Lctr32_6x
1138
1139
leaq 128(%rcx),%rcx
1140
subq $2,%rdx
1141
jmp .Lctr32_loop8
1142
1143
.align 16
1144
.Lctr32_6x:
1145
shll $4,%eax
1146
movl $48,%r10d
1147
bswapl %ebp
1148
leaq 32(%rcx,%rax,1),%rcx
1149
subq %rax,%r10
1150
jmp .Lctr32_loop6
1151
1152
.align 16
1153
.Lctr32_loop6:
1154
addl $6,%r8d
1155
movups -48(%rcx,%r10,1),%xmm0
1156
.byte 102,15,56,220,209
1157
movl %r8d,%eax
1158
xorl %ebp,%eax
1159
.byte 102,15,56,220,217
1160
.byte 0x0f,0x38,0xf1,0x44,0x24,12
1161
leal 1(%r8),%eax
1162
.byte 102,15,56,220,225
1163
xorl %ebp,%eax
1164
.byte 0x0f,0x38,0xf1,0x44,0x24,28
1165
.byte 102,15,56,220,233
1166
leal 2(%r8),%eax
1167
xorl %ebp,%eax
1168
.byte 102,15,56,220,241
1169
.byte 0x0f,0x38,0xf1,0x44,0x24,44
1170
leal 3(%r8),%eax
1171
.byte 102,15,56,220,249
1172
movups -32(%rcx,%r10,1),%xmm1
1173
xorl %ebp,%eax
1174
1175
.byte 102,15,56,220,208
1176
.byte 0x0f,0x38,0xf1,0x44,0x24,60
1177
leal 4(%r8),%eax
1178
.byte 102,15,56,220,216
1179
xorl %ebp,%eax
1180
.byte 0x0f,0x38,0xf1,0x44,0x24,76
1181
.byte 102,15,56,220,224
1182
leal 5(%r8),%eax
1183
xorl %ebp,%eax
1184
.byte 102,15,56,220,232
1185
.byte 0x0f,0x38,0xf1,0x44,0x24,92
1186
movq %r10,%rax
1187
.byte 102,15,56,220,240
1188
.byte 102,15,56,220,248
1189
movups -16(%rcx,%r10,1),%xmm0
1190
1191
call .Lenc_loop6
1192
1193
movdqu (%rdi),%xmm8
1194
movdqu 16(%rdi),%xmm9
1195
movdqu 32(%rdi),%xmm10
1196
movdqu 48(%rdi),%xmm11
1197
movdqu 64(%rdi),%xmm12
1198
movdqu 80(%rdi),%xmm13
1199
leaq 96(%rdi),%rdi
1200
movups -64(%rcx,%r10,1),%xmm1
1201
pxor %xmm2,%xmm8
1202
movaps 0(%rsp),%xmm2
1203
pxor %xmm3,%xmm9
1204
movaps 16(%rsp),%xmm3
1205
pxor %xmm4,%xmm10
1206
movaps 32(%rsp),%xmm4
1207
pxor %xmm5,%xmm11
1208
movaps 48(%rsp),%xmm5
1209
pxor %xmm6,%xmm12
1210
movaps 64(%rsp),%xmm6
1211
pxor %xmm7,%xmm13
1212
movaps 80(%rsp),%xmm7
1213
movdqu %xmm8,(%rsi)
1214
movdqu %xmm9,16(%rsi)
1215
movdqu %xmm10,32(%rsi)
1216
movdqu %xmm11,48(%rsi)
1217
movdqu %xmm12,64(%rsi)
1218
movdqu %xmm13,80(%rsi)
1219
leaq 96(%rsi),%rsi
1220
1221
subq $6,%rdx
1222
jnc .Lctr32_loop6
1223
1224
addq $6,%rdx
1225
jz .Lctr32_done
1226
1227
leal -48(%r10),%eax
1228
leaq -80(%rcx,%r10,1),%rcx
1229
negl %eax
1230
shrl $4,%eax
1231
jmp .Lctr32_tail
1232
1233
.align 32
1234
.Lctr32_loop8:
1235
addl $8,%r8d
1236
movdqa 96(%rsp),%xmm8
1237
.byte 102,15,56,220,209
1238
movl %r8d,%r9d
1239
movdqa 112(%rsp),%xmm9
1240
.byte 102,15,56,220,217
1241
bswapl %r9d
1242
movups 32-128(%rcx),%xmm0
1243
.byte 102,15,56,220,225
1244
xorl %ebp,%r9d
1245
nop
1246
.byte 102,15,56,220,233
1247
movl %r9d,0+12(%rsp)
1248
leaq 1(%r8),%r9
1249
.byte 102,15,56,220,241
1250
.byte 102,15,56,220,249
1251
.byte 102,68,15,56,220,193
1252
.byte 102,68,15,56,220,201
1253
movups 48-128(%rcx),%xmm1
1254
bswapl %r9d
1255
.byte 102,15,56,220,208
1256
.byte 102,15,56,220,216
1257
xorl %ebp,%r9d
1258
.byte 0x66,0x90
1259
.byte 102,15,56,220,224
1260
.byte 102,15,56,220,232
1261
movl %r9d,16+12(%rsp)
1262
leaq 2(%r8),%r9
1263
.byte 102,15,56,220,240
1264
.byte 102,15,56,220,248
1265
.byte 102,68,15,56,220,192
1266
.byte 102,68,15,56,220,200
1267
movups 64-128(%rcx),%xmm0
1268
bswapl %r9d
1269
.byte 102,15,56,220,209
1270
.byte 102,15,56,220,217
1271
xorl %ebp,%r9d
1272
.byte 0x66,0x90
1273
.byte 102,15,56,220,225
1274
.byte 102,15,56,220,233
1275
movl %r9d,32+12(%rsp)
1276
leaq 3(%r8),%r9
1277
.byte 102,15,56,220,241
1278
.byte 102,15,56,220,249
1279
.byte 102,68,15,56,220,193
1280
.byte 102,68,15,56,220,201
1281
movups 80-128(%rcx),%xmm1
1282
bswapl %r9d
1283
.byte 102,15,56,220,208
1284
.byte 102,15,56,220,216
1285
xorl %ebp,%r9d
1286
.byte 0x66,0x90
1287
.byte 102,15,56,220,224
1288
.byte 102,15,56,220,232
1289
movl %r9d,48+12(%rsp)
1290
leaq 4(%r8),%r9
1291
.byte 102,15,56,220,240
1292
.byte 102,15,56,220,248
1293
.byte 102,68,15,56,220,192
1294
.byte 102,68,15,56,220,200
1295
movups 96-128(%rcx),%xmm0
1296
bswapl %r9d
1297
.byte 102,15,56,220,209
1298
.byte 102,15,56,220,217
1299
xorl %ebp,%r9d
1300
.byte 0x66,0x90
1301
.byte 102,15,56,220,225
1302
.byte 102,15,56,220,233
1303
movl %r9d,64+12(%rsp)
1304
leaq 5(%r8),%r9
1305
.byte 102,15,56,220,241
1306
.byte 102,15,56,220,249
1307
.byte 102,68,15,56,220,193
1308
.byte 102,68,15,56,220,201
1309
movups 112-128(%rcx),%xmm1
1310
bswapl %r9d
1311
.byte 102,15,56,220,208
1312
.byte 102,15,56,220,216
1313
xorl %ebp,%r9d
1314
.byte 0x66,0x90
1315
.byte 102,15,56,220,224
1316
.byte 102,15,56,220,232
1317
movl %r9d,80+12(%rsp)
1318
leaq 6(%r8),%r9
1319
.byte 102,15,56,220,240
1320
.byte 102,15,56,220,248
1321
.byte 102,68,15,56,220,192
1322
.byte 102,68,15,56,220,200
1323
movups 128-128(%rcx),%xmm0
1324
bswapl %r9d
1325
.byte 102,15,56,220,209
1326
.byte 102,15,56,220,217
1327
xorl %ebp,%r9d
1328
.byte 0x66,0x90
1329
.byte 102,15,56,220,225
1330
.byte 102,15,56,220,233
1331
movl %r9d,96+12(%rsp)
1332
leaq 7(%r8),%r9
1333
.byte 102,15,56,220,241
1334
.byte 102,15,56,220,249
1335
.byte 102,68,15,56,220,193
1336
.byte 102,68,15,56,220,201
1337
movups 144-128(%rcx),%xmm1
1338
bswapl %r9d
1339
.byte 102,15,56,220,208
1340
.byte 102,15,56,220,216
1341
.byte 102,15,56,220,224
1342
xorl %ebp,%r9d
1343
movdqu 0(%rdi),%xmm10
1344
.byte 102,15,56,220,232
1345
movl %r9d,112+12(%rsp)
1346
cmpl $11,%eax
1347
.byte 102,15,56,220,240
1348
.byte 102,15,56,220,248
1349
.byte 102,68,15,56,220,192
1350
.byte 102,68,15,56,220,200
1351
movups 160-128(%rcx),%xmm0
1352
1353
jb .Lctr32_enc_done
1354
1355
.byte 102,15,56,220,209
1356
.byte 102,15,56,220,217
1357
.byte 102,15,56,220,225
1358
.byte 102,15,56,220,233
1359
.byte 102,15,56,220,241
1360
.byte 102,15,56,220,249
1361
.byte 102,68,15,56,220,193
1362
.byte 102,68,15,56,220,201
1363
movups 176-128(%rcx),%xmm1
1364
1365
.byte 102,15,56,220,208
1366
.byte 102,15,56,220,216
1367
.byte 102,15,56,220,224
1368
.byte 102,15,56,220,232
1369
.byte 102,15,56,220,240
1370
.byte 102,15,56,220,248
1371
.byte 102,68,15,56,220,192
1372
.byte 102,68,15,56,220,200
1373
movups 192-128(%rcx),%xmm0
1374
je .Lctr32_enc_done
1375
1376
.byte 102,15,56,220,209
1377
.byte 102,15,56,220,217
1378
.byte 102,15,56,220,225
1379
.byte 102,15,56,220,233
1380
.byte 102,15,56,220,241
1381
.byte 102,15,56,220,249
1382
.byte 102,68,15,56,220,193
1383
.byte 102,68,15,56,220,201
1384
movups 208-128(%rcx),%xmm1
1385
1386
.byte 102,15,56,220,208
1387
.byte 102,15,56,220,216
1388
.byte 102,15,56,220,224
1389
.byte 102,15,56,220,232
1390
.byte 102,15,56,220,240
1391
.byte 102,15,56,220,248
1392
.byte 102,68,15,56,220,192
1393
.byte 102,68,15,56,220,200
1394
movups 224-128(%rcx),%xmm0
1395
jmp .Lctr32_enc_done
1396
1397
.align 16
1398
.Lctr32_enc_done:
1399
movdqu 16(%rdi),%xmm11
1400
pxor %xmm0,%xmm10
1401
movdqu 32(%rdi),%xmm12
1402
pxor %xmm0,%xmm11
1403
movdqu 48(%rdi),%xmm13
1404
pxor %xmm0,%xmm12
1405
movdqu 64(%rdi),%xmm14
1406
pxor %xmm0,%xmm13
1407
movdqu 80(%rdi),%xmm15
1408
pxor %xmm0,%xmm14
1409
pxor %xmm0,%xmm15
1410
.byte 102,15,56,220,209
1411
.byte 102,15,56,220,217
1412
.byte 102,15,56,220,225
1413
.byte 102,15,56,220,233
1414
.byte 102,15,56,220,241
1415
.byte 102,15,56,220,249
1416
.byte 102,68,15,56,220,193
1417
.byte 102,68,15,56,220,201
1418
movdqu 96(%rdi),%xmm1
1419
leaq 128(%rdi),%rdi
1420
1421
.byte 102,65,15,56,221,210
1422
pxor %xmm0,%xmm1
1423
movdqu 112-128(%rdi),%xmm10
1424
.byte 102,65,15,56,221,219
1425
pxor %xmm0,%xmm10
1426
movdqa 0(%rsp),%xmm11
1427
.byte 102,65,15,56,221,228
1428
.byte 102,65,15,56,221,237
1429
movdqa 16(%rsp),%xmm12
1430
movdqa 32(%rsp),%xmm13
1431
.byte 102,65,15,56,221,246
1432
.byte 102,65,15,56,221,255
1433
movdqa 48(%rsp),%xmm14
1434
movdqa 64(%rsp),%xmm15
1435
.byte 102,68,15,56,221,193
1436
movdqa 80(%rsp),%xmm0
1437
movups 16-128(%rcx),%xmm1
1438
.byte 102,69,15,56,221,202
1439
1440
movups %xmm2,(%rsi)
1441
movdqa %xmm11,%xmm2
1442
movups %xmm3,16(%rsi)
1443
movdqa %xmm12,%xmm3
1444
movups %xmm4,32(%rsi)
1445
movdqa %xmm13,%xmm4
1446
movups %xmm5,48(%rsi)
1447
movdqa %xmm14,%xmm5
1448
movups %xmm6,64(%rsi)
1449
movdqa %xmm15,%xmm6
1450
movups %xmm7,80(%rsi)
1451
movdqa %xmm0,%xmm7
1452
movups %xmm8,96(%rsi)
1453
movups %xmm9,112(%rsi)
1454
leaq 128(%rsi),%rsi
1455
1456
subq $8,%rdx
1457
jnc .Lctr32_loop8
1458
1459
addq $8,%rdx
1460
jz .Lctr32_done
1461
leaq -128(%rcx),%rcx
1462
1463
.Lctr32_tail:
1464
1465
1466
leaq 16(%rcx),%rcx
1467
cmpq $4,%rdx
1468
jb .Lctr32_loop3
1469
je .Lctr32_loop4
1470
1471
1472
shll $4,%eax
1473
movdqa 96(%rsp),%xmm8
1474
pxor %xmm9,%xmm9
1475
1476
movups 16(%rcx),%xmm0
1477
.byte 102,15,56,220,209
1478
.byte 102,15,56,220,217
1479
leaq 32-16(%rcx,%rax,1),%rcx
1480
negq %rax
1481
.byte 102,15,56,220,225
1482
addq $16,%rax
1483
movups (%rdi),%xmm10
1484
.byte 102,15,56,220,233
1485
.byte 102,15,56,220,241
1486
movups 16(%rdi),%xmm11
1487
movups 32(%rdi),%xmm12
1488
.byte 102,15,56,220,249
1489
.byte 102,68,15,56,220,193
1490
1491
call .Lenc_loop8_enter
1492
1493
movdqu 48(%rdi),%xmm13
1494
pxor %xmm10,%xmm2
1495
movdqu 64(%rdi),%xmm10
1496
pxor %xmm11,%xmm3
1497
movdqu %xmm2,(%rsi)
1498
pxor %xmm12,%xmm4
1499
movdqu %xmm3,16(%rsi)
1500
pxor %xmm13,%xmm5
1501
movdqu %xmm4,32(%rsi)
1502
pxor %xmm10,%xmm6
1503
movdqu %xmm5,48(%rsi)
1504
movdqu %xmm6,64(%rsi)
1505
cmpq $6,%rdx
1506
jb .Lctr32_done
1507
1508
movups 80(%rdi),%xmm11
1509
xorps %xmm11,%xmm7
1510
movups %xmm7,80(%rsi)
1511
je .Lctr32_done
1512
1513
movups 96(%rdi),%xmm12
1514
xorps %xmm12,%xmm8
1515
movups %xmm8,96(%rsi)
1516
jmp .Lctr32_done
1517
1518
.align 32
1519
.Lctr32_loop4:
1520
.byte 102,15,56,220,209
1521
leaq 16(%rcx),%rcx
1522
decl %eax
1523
.byte 102,15,56,220,217
1524
.byte 102,15,56,220,225
1525
.byte 102,15,56,220,233
1526
movups (%rcx),%xmm1
1527
jnz .Lctr32_loop4
1528
.byte 102,15,56,221,209
1529
.byte 102,15,56,221,217
1530
movups (%rdi),%xmm10
1531
movups 16(%rdi),%xmm11
1532
.byte 102,15,56,221,225
1533
.byte 102,15,56,221,233
1534
movups 32(%rdi),%xmm12
1535
movups 48(%rdi),%xmm13
1536
1537
xorps %xmm10,%xmm2
1538
movups %xmm2,(%rsi)
1539
xorps %xmm11,%xmm3
1540
movups %xmm3,16(%rsi)
1541
pxor %xmm12,%xmm4
1542
movdqu %xmm4,32(%rsi)
1543
pxor %xmm13,%xmm5
1544
movdqu %xmm5,48(%rsi)
1545
jmp .Lctr32_done
1546
1547
.align 32
1548
.Lctr32_loop3:
1549
.byte 102,15,56,220,209
1550
leaq 16(%rcx),%rcx
1551
decl %eax
1552
.byte 102,15,56,220,217
1553
.byte 102,15,56,220,225
1554
movups (%rcx),%xmm1
1555
jnz .Lctr32_loop3
1556
.byte 102,15,56,221,209
1557
.byte 102,15,56,221,217
1558
.byte 102,15,56,221,225
1559
1560
movups (%rdi),%xmm10
1561
xorps %xmm10,%xmm2
1562
movups %xmm2,(%rsi)
1563
cmpq $2,%rdx
1564
jb .Lctr32_done
1565
1566
movups 16(%rdi),%xmm11
1567
xorps %xmm11,%xmm3
1568
movups %xmm3,16(%rsi)
1569
je .Lctr32_done
1570
1571
movups 32(%rdi),%xmm12
1572
xorps %xmm12,%xmm4
1573
movups %xmm4,32(%rsi)
1574
1575
.Lctr32_done:
1576
xorps %xmm0,%xmm0
1577
xorl %ebp,%ebp
1578
pxor %xmm1,%xmm1
1579
pxor %xmm2,%xmm2
1580
pxor %xmm3,%xmm3
1581
pxor %xmm4,%xmm4
1582
pxor %xmm5,%xmm5
1583
pxor %xmm6,%xmm6
1584
pxor %xmm7,%xmm7
1585
movaps %xmm0,0(%rsp)
1586
pxor %xmm8,%xmm8
1587
movaps %xmm0,16(%rsp)
1588
pxor %xmm9,%xmm9
1589
movaps %xmm0,32(%rsp)
1590
pxor %xmm10,%xmm10
1591
movaps %xmm0,48(%rsp)
1592
pxor %xmm11,%xmm11
1593
movaps %xmm0,64(%rsp)
1594
pxor %xmm12,%xmm12
1595
movaps %xmm0,80(%rsp)
1596
pxor %xmm13,%xmm13
1597
movaps %xmm0,96(%rsp)
1598
pxor %xmm14,%xmm14
1599
movaps %xmm0,112(%rsp)
1600
pxor %xmm15,%xmm15
1601
movq -8(%r11),%rbp
1602
.cfi_restore %rbp
1603
leaq (%r11),%rsp
1604
.cfi_def_cfa_register %rsp
1605
.Lctr32_epilogue:
1606
.byte 0xf3,0xc3
1607
.cfi_endproc
1608
.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
1609
.globl aesni_xts_encrypt
1610
.type aesni_xts_encrypt,@function
1611
.align 16
1612
aesni_xts_encrypt:
1613
.cfi_startproc
1614
.byte 243,15,30,250
1615
leaq (%rsp),%r11
1616
.cfi_def_cfa_register %r11
1617
pushq %rbp
1618
.cfi_offset %rbp,-16
1619
subq $112,%rsp
1620
andq $-16,%rsp
1621
movups (%r9),%xmm2
1622
movl 240(%r8),%eax
1623
movl 240(%rcx),%r10d
1624
movups (%r8),%xmm0
1625
movups 16(%r8),%xmm1
1626
leaq 32(%r8),%r8
1627
xorps %xmm0,%xmm2
1628
.Loop_enc1_8:
1629
.byte 102,15,56,220,209
1630
decl %eax
1631
movups (%r8),%xmm1
1632
leaq 16(%r8),%r8
1633
jnz .Loop_enc1_8
1634
.byte 102,15,56,221,209
1635
movups (%rcx),%xmm0
1636
movq %rcx,%rbp
1637
movl %r10d,%eax
1638
shll $4,%r10d
1639
movq %rdx,%r9
1640
andq $-16,%rdx
1641
1642
movups 16(%rcx,%r10,1),%xmm1
1643
1644
movdqa .Lxts_magic(%rip),%xmm8
1645
movdqa %xmm2,%xmm15
1646
pshufd $0x5f,%xmm2,%xmm9
1647
pxor %xmm0,%xmm1
1648
movdqa %xmm9,%xmm14
1649
paddd %xmm9,%xmm9
1650
movdqa %xmm15,%xmm10
1651
psrad $31,%xmm14
1652
paddq %xmm15,%xmm15
1653
pand %xmm8,%xmm14
1654
pxor %xmm0,%xmm10
1655
pxor %xmm14,%xmm15
1656
movdqa %xmm9,%xmm14
1657
paddd %xmm9,%xmm9
1658
movdqa %xmm15,%xmm11
1659
psrad $31,%xmm14
1660
paddq %xmm15,%xmm15
1661
pand %xmm8,%xmm14
1662
pxor %xmm0,%xmm11
1663
pxor %xmm14,%xmm15
1664
movdqa %xmm9,%xmm14
1665
paddd %xmm9,%xmm9
1666
movdqa %xmm15,%xmm12
1667
psrad $31,%xmm14
1668
paddq %xmm15,%xmm15
1669
pand %xmm8,%xmm14
1670
pxor %xmm0,%xmm12
1671
pxor %xmm14,%xmm15
1672
movdqa %xmm9,%xmm14
1673
paddd %xmm9,%xmm9
1674
movdqa %xmm15,%xmm13
1675
psrad $31,%xmm14
1676
paddq %xmm15,%xmm15
1677
pand %xmm8,%xmm14
1678
pxor %xmm0,%xmm13
1679
pxor %xmm14,%xmm15
1680
movdqa %xmm15,%xmm14
1681
psrad $31,%xmm9
1682
paddq %xmm15,%xmm15
1683
pand %xmm8,%xmm9
1684
pxor %xmm0,%xmm14
1685
pxor %xmm9,%xmm15
1686
movaps %xmm1,96(%rsp)
1687
1688
subq $96,%rdx
1689
jc .Lxts_enc_short
1690
1691
movl $16+96,%eax
1692
leaq 32(%rbp,%r10,1),%rcx
1693
subq %r10,%rax
1694
movups 16(%rbp),%xmm1
1695
movq %rax,%r10
1696
leaq .Lxts_magic(%rip),%r8
1697
jmp .Lxts_enc_grandloop
1698
1699
.align 32
1700
.Lxts_enc_grandloop:
1701
movdqu 0(%rdi),%xmm2
1702
movdqa %xmm0,%xmm8
1703
movdqu 16(%rdi),%xmm3
1704
pxor %xmm10,%xmm2
1705
movdqu 32(%rdi),%xmm4
1706
pxor %xmm11,%xmm3
1707
.byte 102,15,56,220,209
1708
movdqu 48(%rdi),%xmm5
1709
pxor %xmm12,%xmm4
1710
.byte 102,15,56,220,217
1711
movdqu 64(%rdi),%xmm6
1712
pxor %xmm13,%xmm5
1713
.byte 102,15,56,220,225
1714
movdqu 80(%rdi),%xmm7
1715
pxor %xmm15,%xmm8
1716
movdqa 96(%rsp),%xmm9
1717
pxor %xmm14,%xmm6
1718
.byte 102,15,56,220,233
1719
movups 32(%rbp),%xmm0
1720
leaq 96(%rdi),%rdi
1721
pxor %xmm8,%xmm7
1722
1723
pxor %xmm9,%xmm10
1724
.byte 102,15,56,220,241
1725
pxor %xmm9,%xmm11
1726
movdqa %xmm10,0(%rsp)
1727
.byte 102,15,56,220,249
1728
movups 48(%rbp),%xmm1
1729
pxor %xmm9,%xmm12
1730
1731
.byte 102,15,56,220,208
1732
pxor %xmm9,%xmm13
1733
movdqa %xmm11,16(%rsp)
1734
.byte 102,15,56,220,216
1735
pxor %xmm9,%xmm14
1736
movdqa %xmm12,32(%rsp)
1737
.byte 102,15,56,220,224
1738
.byte 102,15,56,220,232
1739
pxor %xmm9,%xmm8
1740
movdqa %xmm14,64(%rsp)
1741
.byte 102,15,56,220,240
1742
.byte 102,15,56,220,248
1743
movups 64(%rbp),%xmm0
1744
movdqa %xmm8,80(%rsp)
1745
pshufd $0x5f,%xmm15,%xmm9
1746
jmp .Lxts_enc_loop6
1747
.align 32
1748
.Lxts_enc_loop6:
1749
.byte 102,15,56,220,209
1750
.byte 102,15,56,220,217
1751
.byte 102,15,56,220,225
1752
.byte 102,15,56,220,233
1753
.byte 102,15,56,220,241
1754
.byte 102,15,56,220,249
1755
movups -64(%rcx,%rax,1),%xmm1
1756
addq $32,%rax
1757
1758
.byte 102,15,56,220,208
1759
.byte 102,15,56,220,216
1760
.byte 102,15,56,220,224
1761
.byte 102,15,56,220,232
1762
.byte 102,15,56,220,240
1763
.byte 102,15,56,220,248
1764
movups -80(%rcx,%rax,1),%xmm0
1765
jnz .Lxts_enc_loop6
1766
1767
movdqa (%r8),%xmm8
1768
movdqa %xmm9,%xmm14
1769
paddd %xmm9,%xmm9
1770
.byte 102,15,56,220,209
1771
paddq %xmm15,%xmm15
1772
psrad $31,%xmm14
1773
.byte 102,15,56,220,217
1774
pand %xmm8,%xmm14
1775
movups (%rbp),%xmm10
1776
.byte 102,15,56,220,225
1777
.byte 102,15,56,220,233
1778
.byte 102,15,56,220,241
1779
pxor %xmm14,%xmm15
1780
movaps %xmm10,%xmm11
1781
.byte 102,15,56,220,249
1782
movups -64(%rcx),%xmm1
1783
1784
movdqa %xmm9,%xmm14
1785
.byte 102,15,56,220,208
1786
paddd %xmm9,%xmm9
1787
pxor %xmm15,%xmm10
1788
.byte 102,15,56,220,216
1789
psrad $31,%xmm14
1790
paddq %xmm15,%xmm15
1791
.byte 102,15,56,220,224
1792
.byte 102,15,56,220,232
1793
pand %xmm8,%xmm14
1794
movaps %xmm11,%xmm12
1795
.byte 102,15,56,220,240
1796
pxor %xmm14,%xmm15
1797
movdqa %xmm9,%xmm14
1798
.byte 102,15,56,220,248
1799
movups -48(%rcx),%xmm0
1800
1801
paddd %xmm9,%xmm9
1802
.byte 102,15,56,220,209
1803
pxor %xmm15,%xmm11
1804
psrad $31,%xmm14
1805
.byte 102,15,56,220,217
1806
paddq %xmm15,%xmm15
1807
pand %xmm8,%xmm14
1808
.byte 102,15,56,220,225
1809
.byte 102,15,56,220,233
1810
movdqa %xmm13,48(%rsp)
1811
pxor %xmm14,%xmm15
1812
.byte 102,15,56,220,241
1813
movaps %xmm12,%xmm13
1814
movdqa %xmm9,%xmm14
1815
.byte 102,15,56,220,249
1816
movups -32(%rcx),%xmm1
1817
1818
paddd %xmm9,%xmm9
1819
.byte 102,15,56,220,208
1820
pxor %xmm15,%xmm12
1821
psrad $31,%xmm14
1822
.byte 102,15,56,220,216
1823
paddq %xmm15,%xmm15
1824
pand %xmm8,%xmm14
1825
.byte 102,15,56,220,224
1826
.byte 102,15,56,220,232
1827
.byte 102,15,56,220,240
1828
pxor %xmm14,%xmm15
1829
movaps %xmm13,%xmm14
1830
.byte 102,15,56,220,248
1831
1832
movdqa %xmm9,%xmm0
1833
paddd %xmm9,%xmm9
1834
.byte 102,15,56,220,209
1835
pxor %xmm15,%xmm13
1836
psrad $31,%xmm0
1837
.byte 102,15,56,220,217
1838
paddq %xmm15,%xmm15
1839
pand %xmm8,%xmm0
1840
.byte 102,15,56,220,225
1841
.byte 102,15,56,220,233
1842
pxor %xmm0,%xmm15
1843
movups (%rbp),%xmm0
1844
.byte 102,15,56,220,241
1845
.byte 102,15,56,220,249
1846
movups 16(%rbp),%xmm1
1847
1848
pxor %xmm15,%xmm14
1849
.byte 102,15,56,221,84,36,0
1850
psrad $31,%xmm9
1851
paddq %xmm15,%xmm15
1852
.byte 102,15,56,221,92,36,16
1853
.byte 102,15,56,221,100,36,32
1854
pand %xmm8,%xmm9
1855
movq %r10,%rax
1856
.byte 102,15,56,221,108,36,48
1857
.byte 102,15,56,221,116,36,64
1858
.byte 102,15,56,221,124,36,80
1859
pxor %xmm9,%xmm15
1860
1861
leaq 96(%rsi),%rsi
1862
movups %xmm2,-96(%rsi)
1863
movups %xmm3,-80(%rsi)
1864
movups %xmm4,-64(%rsi)
1865
movups %xmm5,-48(%rsi)
1866
movups %xmm6,-32(%rsi)
1867
movups %xmm7,-16(%rsi)
1868
subq $96,%rdx
1869
jnc .Lxts_enc_grandloop
1870
1871
movl $16+96,%eax
1872
subl %r10d,%eax
1873
movq %rbp,%rcx
1874
shrl $4,%eax
1875
1876
.Lxts_enc_short:
1877
1878
movl %eax,%r10d
1879
pxor %xmm0,%xmm10
1880
addq $96,%rdx
1881
jz .Lxts_enc_done
1882
1883
pxor %xmm0,%xmm11
1884
cmpq $0x20,%rdx
1885
jb .Lxts_enc_one
1886
pxor %xmm0,%xmm12
1887
je .Lxts_enc_two
1888
1889
pxor %xmm0,%xmm13
1890
cmpq $0x40,%rdx
1891
jb .Lxts_enc_three
1892
pxor %xmm0,%xmm14
1893
je .Lxts_enc_four
1894
1895
movdqu (%rdi),%xmm2
1896
movdqu 16(%rdi),%xmm3
1897
movdqu 32(%rdi),%xmm4
1898
pxor %xmm10,%xmm2
1899
movdqu 48(%rdi),%xmm5
1900
pxor %xmm11,%xmm3
1901
movdqu 64(%rdi),%xmm6
1902
leaq 80(%rdi),%rdi
1903
pxor %xmm12,%xmm4
1904
pxor %xmm13,%xmm5
1905
pxor %xmm14,%xmm6
1906
pxor %xmm7,%xmm7
1907
1908
call _aesni_encrypt6
1909
1910
xorps %xmm10,%xmm2
1911
movdqa %xmm15,%xmm10
1912
xorps %xmm11,%xmm3
1913
xorps %xmm12,%xmm4
1914
movdqu %xmm2,(%rsi)
1915
xorps %xmm13,%xmm5
1916
movdqu %xmm3,16(%rsi)
1917
xorps %xmm14,%xmm6
1918
movdqu %xmm4,32(%rsi)
1919
movdqu %xmm5,48(%rsi)
1920
movdqu %xmm6,64(%rsi)
1921
leaq 80(%rsi),%rsi
1922
jmp .Lxts_enc_done
1923
1924
.align 16
1925
.Lxts_enc_one:
1926
movups (%rdi),%xmm2
1927
leaq 16(%rdi),%rdi
1928
xorps %xmm10,%xmm2
1929
movups (%rcx),%xmm0
1930
movups 16(%rcx),%xmm1
1931
leaq 32(%rcx),%rcx
1932
xorps %xmm0,%xmm2
1933
.Loop_enc1_9:
1934
.byte 102,15,56,220,209
1935
decl %eax
1936
movups (%rcx),%xmm1
1937
leaq 16(%rcx),%rcx
1938
jnz .Loop_enc1_9
1939
.byte 102,15,56,221,209
1940
xorps %xmm10,%xmm2
1941
movdqa %xmm11,%xmm10
1942
movups %xmm2,(%rsi)
1943
leaq 16(%rsi),%rsi
1944
jmp .Lxts_enc_done
1945
1946
.align 16
1947
.Lxts_enc_two:
1948
movups (%rdi),%xmm2
1949
movups 16(%rdi),%xmm3
1950
leaq 32(%rdi),%rdi
1951
xorps %xmm10,%xmm2
1952
xorps %xmm11,%xmm3
1953
1954
call _aesni_encrypt2
1955
1956
xorps %xmm10,%xmm2
1957
movdqa %xmm12,%xmm10
1958
xorps %xmm11,%xmm3
1959
movups %xmm2,(%rsi)
1960
movups %xmm3,16(%rsi)
1961
leaq 32(%rsi),%rsi
1962
jmp .Lxts_enc_done
1963
1964
.align 16
1965
.Lxts_enc_three:
1966
movups (%rdi),%xmm2
1967
movups 16(%rdi),%xmm3
1968
movups 32(%rdi),%xmm4
1969
leaq 48(%rdi),%rdi
1970
xorps %xmm10,%xmm2
1971
xorps %xmm11,%xmm3
1972
xorps %xmm12,%xmm4
1973
1974
call _aesni_encrypt3
1975
1976
xorps %xmm10,%xmm2
1977
movdqa %xmm13,%xmm10
1978
xorps %xmm11,%xmm3
1979
xorps %xmm12,%xmm4
1980
movups %xmm2,(%rsi)
1981
movups %xmm3,16(%rsi)
1982
movups %xmm4,32(%rsi)
1983
leaq 48(%rsi),%rsi
1984
jmp .Lxts_enc_done
1985
1986
.align 16
1987
.Lxts_enc_four:
1988
movups (%rdi),%xmm2
1989
movups 16(%rdi),%xmm3
1990
movups 32(%rdi),%xmm4
1991
xorps %xmm10,%xmm2
1992
movups 48(%rdi),%xmm5
1993
leaq 64(%rdi),%rdi
1994
xorps %xmm11,%xmm3
1995
xorps %xmm12,%xmm4
1996
xorps %xmm13,%xmm5
1997
1998
call _aesni_encrypt4
1999
2000
pxor %xmm10,%xmm2
2001
movdqa %xmm14,%xmm10
2002
pxor %xmm11,%xmm3
2003
pxor %xmm12,%xmm4
2004
movdqu %xmm2,(%rsi)
2005
pxor %xmm13,%xmm5
2006
movdqu %xmm3,16(%rsi)
2007
movdqu %xmm4,32(%rsi)
2008
movdqu %xmm5,48(%rsi)
2009
leaq 64(%rsi),%rsi
2010
jmp .Lxts_enc_done
2011
2012
.align 16
2013
.Lxts_enc_done:
2014
andq $15,%r9
2015
jz .Lxts_enc_ret
2016
movq %r9,%rdx
2017
2018
.Lxts_enc_steal:
2019
movzbl (%rdi),%eax
2020
movzbl -16(%rsi),%ecx
2021
leaq 1(%rdi),%rdi
2022
movb %al,-16(%rsi)
2023
movb %cl,0(%rsi)
2024
leaq 1(%rsi),%rsi
2025
subq $1,%rdx
2026
jnz .Lxts_enc_steal
2027
2028
subq %r9,%rsi
2029
movq %rbp,%rcx
2030
movl %r10d,%eax
2031
2032
movups -16(%rsi),%xmm2
2033
xorps %xmm10,%xmm2
2034
movups (%rcx),%xmm0
2035
movups 16(%rcx),%xmm1
2036
leaq 32(%rcx),%rcx
2037
xorps %xmm0,%xmm2
2038
.Loop_enc1_10:
2039
.byte 102,15,56,220,209
2040
decl %eax
2041
movups (%rcx),%xmm1
2042
leaq 16(%rcx),%rcx
2043
jnz .Loop_enc1_10
2044
.byte 102,15,56,221,209
2045
xorps %xmm10,%xmm2
2046
movups %xmm2,-16(%rsi)
2047
2048
.Lxts_enc_ret:
2049
xorps %xmm0,%xmm0
2050
pxor %xmm1,%xmm1
2051
pxor %xmm2,%xmm2
2052
pxor %xmm3,%xmm3
2053
pxor %xmm4,%xmm4
2054
pxor %xmm5,%xmm5
2055
pxor %xmm6,%xmm6
2056
pxor %xmm7,%xmm7
2057
movaps %xmm0,0(%rsp)
2058
pxor %xmm8,%xmm8
2059
movaps %xmm0,16(%rsp)
2060
pxor %xmm9,%xmm9
2061
movaps %xmm0,32(%rsp)
2062
pxor %xmm10,%xmm10
2063
movaps %xmm0,48(%rsp)
2064
pxor %xmm11,%xmm11
2065
movaps %xmm0,64(%rsp)
2066
pxor %xmm12,%xmm12
2067
movaps %xmm0,80(%rsp)
2068
pxor %xmm13,%xmm13
2069
movaps %xmm0,96(%rsp)
2070
pxor %xmm14,%xmm14
2071
pxor %xmm15,%xmm15
2072
movq -8(%r11),%rbp
2073
.cfi_restore %rbp
2074
leaq (%r11),%rsp
2075
.cfi_def_cfa_register %rsp
2076
.Lxts_enc_epilogue:
2077
.byte 0xf3,0xc3
2078
.cfi_endproc
2079
.size aesni_xts_encrypt,.-aesni_xts_encrypt
2080
.globl aesni_xts_decrypt
2081
.type aesni_xts_decrypt,@function
2082
.align 16
2083
aesni_xts_decrypt:
2084
.cfi_startproc
2085
.byte 243,15,30,250
2086
leaq (%rsp),%r11
2087
.cfi_def_cfa_register %r11
2088
pushq %rbp
2089
.cfi_offset %rbp,-16
2090
subq $112,%rsp
2091
andq $-16,%rsp
2092
movups (%r9),%xmm2
2093
movl 240(%r8),%eax
2094
movl 240(%rcx),%r10d
2095
movups (%r8),%xmm0
2096
movups 16(%r8),%xmm1
2097
leaq 32(%r8),%r8
2098
xorps %xmm0,%xmm2
2099
.Loop_enc1_11:
2100
.byte 102,15,56,220,209
2101
decl %eax
2102
movups (%r8),%xmm1
2103
leaq 16(%r8),%r8
2104
jnz .Loop_enc1_11
2105
.byte 102,15,56,221,209
2106
xorl %eax,%eax
2107
testq $15,%rdx
2108
setnz %al
2109
shlq $4,%rax
2110
subq %rax,%rdx
2111
2112
movups (%rcx),%xmm0
2113
movq %rcx,%rbp
2114
movl %r10d,%eax
2115
shll $4,%r10d
2116
movq %rdx,%r9
2117
andq $-16,%rdx
2118
2119
movups 16(%rcx,%r10,1),%xmm1
2120
2121
movdqa .Lxts_magic(%rip),%xmm8
2122
movdqa %xmm2,%xmm15
2123
pshufd $0x5f,%xmm2,%xmm9
2124
pxor %xmm0,%xmm1
2125
movdqa %xmm9,%xmm14
2126
paddd %xmm9,%xmm9
2127
movdqa %xmm15,%xmm10
2128
psrad $31,%xmm14
2129
paddq %xmm15,%xmm15
2130
pand %xmm8,%xmm14
2131
pxor %xmm0,%xmm10
2132
pxor %xmm14,%xmm15
2133
movdqa %xmm9,%xmm14
2134
paddd %xmm9,%xmm9
2135
movdqa %xmm15,%xmm11
2136
psrad $31,%xmm14
2137
paddq %xmm15,%xmm15
2138
pand %xmm8,%xmm14
2139
pxor %xmm0,%xmm11
2140
pxor %xmm14,%xmm15
2141
movdqa %xmm9,%xmm14
2142
paddd %xmm9,%xmm9
2143
movdqa %xmm15,%xmm12
2144
psrad $31,%xmm14
2145
paddq %xmm15,%xmm15
2146
pand %xmm8,%xmm14
2147
pxor %xmm0,%xmm12
2148
pxor %xmm14,%xmm15
2149
movdqa %xmm9,%xmm14
2150
paddd %xmm9,%xmm9
2151
movdqa %xmm15,%xmm13
2152
psrad $31,%xmm14
2153
paddq %xmm15,%xmm15
2154
pand %xmm8,%xmm14
2155
pxor %xmm0,%xmm13
2156
pxor %xmm14,%xmm15
2157
movdqa %xmm15,%xmm14
2158
psrad $31,%xmm9
2159
paddq %xmm15,%xmm15
2160
pand %xmm8,%xmm9
2161
pxor %xmm0,%xmm14
2162
pxor %xmm9,%xmm15
2163
movaps %xmm1,96(%rsp)
2164
2165
subq $96,%rdx
2166
jc .Lxts_dec_short
2167
2168
movl $16+96,%eax
2169
leaq 32(%rbp,%r10,1),%rcx
2170
subq %r10,%rax
2171
movups 16(%rbp),%xmm1
2172
movq %rax,%r10
2173
leaq .Lxts_magic(%rip),%r8
2174
jmp .Lxts_dec_grandloop
2175
2176
.align 32
2177
.Lxts_dec_grandloop:
2178
movdqu 0(%rdi),%xmm2
2179
movdqa %xmm0,%xmm8
2180
movdqu 16(%rdi),%xmm3
2181
pxor %xmm10,%xmm2
2182
movdqu 32(%rdi),%xmm4
2183
pxor %xmm11,%xmm3
2184
.byte 102,15,56,222,209
2185
movdqu 48(%rdi),%xmm5
2186
pxor %xmm12,%xmm4
2187
.byte 102,15,56,222,217
2188
movdqu 64(%rdi),%xmm6
2189
pxor %xmm13,%xmm5
2190
.byte 102,15,56,222,225
2191
movdqu 80(%rdi),%xmm7
2192
pxor %xmm15,%xmm8
2193
movdqa 96(%rsp),%xmm9
2194
pxor %xmm14,%xmm6
2195
.byte 102,15,56,222,233
2196
movups 32(%rbp),%xmm0
2197
leaq 96(%rdi),%rdi
2198
pxor %xmm8,%xmm7
2199
2200
pxor %xmm9,%xmm10
2201
.byte 102,15,56,222,241
2202
pxor %xmm9,%xmm11
2203
movdqa %xmm10,0(%rsp)
2204
.byte 102,15,56,222,249
2205
movups 48(%rbp),%xmm1
2206
pxor %xmm9,%xmm12
2207
2208
.byte 102,15,56,222,208
2209
pxor %xmm9,%xmm13
2210
movdqa %xmm11,16(%rsp)
2211
.byte 102,15,56,222,216
2212
pxor %xmm9,%xmm14
2213
movdqa %xmm12,32(%rsp)
2214
.byte 102,15,56,222,224
2215
.byte 102,15,56,222,232
2216
pxor %xmm9,%xmm8
2217
movdqa %xmm14,64(%rsp)
2218
.byte 102,15,56,222,240
2219
.byte 102,15,56,222,248
2220
movups 64(%rbp),%xmm0
2221
movdqa %xmm8,80(%rsp)
2222
pshufd $0x5f,%xmm15,%xmm9
2223
jmp .Lxts_dec_loop6
2224
.align 32
2225
.Lxts_dec_loop6:
2226
.byte 102,15,56,222,209
2227
.byte 102,15,56,222,217
2228
.byte 102,15,56,222,225
2229
.byte 102,15,56,222,233
2230
.byte 102,15,56,222,241
2231
.byte 102,15,56,222,249
2232
movups -64(%rcx,%rax,1),%xmm1
2233
addq $32,%rax
2234
2235
.byte 102,15,56,222,208
2236
.byte 102,15,56,222,216
2237
.byte 102,15,56,222,224
2238
.byte 102,15,56,222,232
2239
.byte 102,15,56,222,240
2240
.byte 102,15,56,222,248
2241
movups -80(%rcx,%rax,1),%xmm0
2242
jnz .Lxts_dec_loop6
2243
2244
movdqa (%r8),%xmm8
2245
movdqa %xmm9,%xmm14
2246
paddd %xmm9,%xmm9
2247
.byte 102,15,56,222,209
2248
paddq %xmm15,%xmm15
2249
psrad $31,%xmm14
2250
.byte 102,15,56,222,217
2251
pand %xmm8,%xmm14
2252
movups (%rbp),%xmm10
2253
.byte 102,15,56,222,225
2254
.byte 102,15,56,222,233
2255
.byte 102,15,56,222,241
2256
pxor %xmm14,%xmm15
2257
movaps %xmm10,%xmm11
2258
.byte 102,15,56,222,249
2259
movups -64(%rcx),%xmm1
2260
2261
movdqa %xmm9,%xmm14
2262
.byte 102,15,56,222,208
2263
paddd %xmm9,%xmm9
2264
pxor %xmm15,%xmm10
2265
.byte 102,15,56,222,216
2266
psrad $31,%xmm14
2267
paddq %xmm15,%xmm15
2268
.byte 102,15,56,222,224
2269
.byte 102,15,56,222,232
2270
pand %xmm8,%xmm14
2271
movaps %xmm11,%xmm12
2272
.byte 102,15,56,222,240
2273
pxor %xmm14,%xmm15
2274
movdqa %xmm9,%xmm14
2275
.byte 102,15,56,222,248
2276
movups -48(%rcx),%xmm0
2277
2278
paddd %xmm9,%xmm9
2279
.byte 102,15,56,222,209
2280
pxor %xmm15,%xmm11
2281
psrad $31,%xmm14
2282
.byte 102,15,56,222,217
2283
paddq %xmm15,%xmm15
2284
pand %xmm8,%xmm14
2285
.byte 102,15,56,222,225
2286
.byte 102,15,56,222,233
2287
movdqa %xmm13,48(%rsp)
2288
pxor %xmm14,%xmm15
2289
.byte 102,15,56,222,241
2290
movaps %xmm12,%xmm13
2291
movdqa %xmm9,%xmm14
2292
.byte 102,15,56,222,249
2293
movups -32(%rcx),%xmm1
2294
2295
paddd %xmm9,%xmm9
2296
.byte 102,15,56,222,208
2297
pxor %xmm15,%xmm12
2298
psrad $31,%xmm14
2299
.byte 102,15,56,222,216
2300
paddq %xmm15,%xmm15
2301
pand %xmm8,%xmm14
2302
.byte 102,15,56,222,224
2303
.byte 102,15,56,222,232
2304
.byte 102,15,56,222,240
2305
pxor %xmm14,%xmm15
2306
movaps %xmm13,%xmm14
2307
.byte 102,15,56,222,248
2308
2309
movdqa %xmm9,%xmm0
2310
paddd %xmm9,%xmm9
2311
.byte 102,15,56,222,209
2312
pxor %xmm15,%xmm13
2313
psrad $31,%xmm0
2314
.byte 102,15,56,222,217
2315
paddq %xmm15,%xmm15
2316
pand %xmm8,%xmm0
2317
.byte 102,15,56,222,225
2318
.byte 102,15,56,222,233
2319
pxor %xmm0,%xmm15
2320
movups (%rbp),%xmm0
2321
.byte 102,15,56,222,241
2322
.byte 102,15,56,222,249
2323
movups 16(%rbp),%xmm1
2324
2325
pxor %xmm15,%xmm14
2326
.byte 102,15,56,223,84,36,0
2327
psrad $31,%xmm9
2328
paddq %xmm15,%xmm15
2329
.byte 102,15,56,223,92,36,16
2330
.byte 102,15,56,223,100,36,32
2331
pand %xmm8,%xmm9
2332
movq %r10,%rax
2333
.byte 102,15,56,223,108,36,48
2334
.byte 102,15,56,223,116,36,64
2335
.byte 102,15,56,223,124,36,80
2336
pxor %xmm9,%xmm15
2337
2338
leaq 96(%rsi),%rsi
2339
movups %xmm2,-96(%rsi)
2340
movups %xmm3,-80(%rsi)
2341
movups %xmm4,-64(%rsi)
2342
movups %xmm5,-48(%rsi)
2343
movups %xmm6,-32(%rsi)
2344
movups %xmm7,-16(%rsi)
2345
subq $96,%rdx
2346
jnc .Lxts_dec_grandloop
2347
2348
movl $16+96,%eax
2349
subl %r10d,%eax
2350
movq %rbp,%rcx
2351
shrl $4,%eax
2352
2353
.Lxts_dec_short:
2354
2355
movl %eax,%r10d
2356
pxor %xmm0,%xmm10
2357
pxor %xmm0,%xmm11
2358
addq $96,%rdx
2359
jz .Lxts_dec_done
2360
2361
pxor %xmm0,%xmm12
2362
cmpq $0x20,%rdx
2363
jb .Lxts_dec_one
2364
pxor %xmm0,%xmm13
2365
je .Lxts_dec_two
2366
2367
pxor %xmm0,%xmm14
2368
cmpq $0x40,%rdx
2369
jb .Lxts_dec_three
2370
je .Lxts_dec_four
2371
2372
movdqu (%rdi),%xmm2
2373
movdqu 16(%rdi),%xmm3
2374
movdqu 32(%rdi),%xmm4
2375
pxor %xmm10,%xmm2
2376
movdqu 48(%rdi),%xmm5
2377
pxor %xmm11,%xmm3
2378
movdqu 64(%rdi),%xmm6
2379
leaq 80(%rdi),%rdi
2380
pxor %xmm12,%xmm4
2381
pxor %xmm13,%xmm5
2382
pxor %xmm14,%xmm6
2383
2384
call _aesni_decrypt6
2385
2386
xorps %xmm10,%xmm2
2387
xorps %xmm11,%xmm3
2388
xorps %xmm12,%xmm4
2389
movdqu %xmm2,(%rsi)
2390
xorps %xmm13,%xmm5
2391
movdqu %xmm3,16(%rsi)
2392
xorps %xmm14,%xmm6
2393
movdqu %xmm4,32(%rsi)
2394
pxor %xmm14,%xmm14
2395
movdqu %xmm5,48(%rsi)
2396
pcmpgtd %xmm15,%xmm14
2397
movdqu %xmm6,64(%rsi)
2398
leaq 80(%rsi),%rsi
2399
pshufd $0x13,%xmm14,%xmm11
2400
andq $15,%r9
2401
jz .Lxts_dec_ret
2402
2403
movdqa %xmm15,%xmm10
2404
paddq %xmm15,%xmm15
2405
pand %xmm8,%xmm11
2406
pxor %xmm15,%xmm11
2407
jmp .Lxts_dec_done2
2408
2409
.align 16
2410
.Lxts_dec_one:
2411
movups (%rdi),%xmm2
2412
leaq 16(%rdi),%rdi
2413
xorps %xmm10,%xmm2
2414
movups (%rcx),%xmm0
2415
movups 16(%rcx),%xmm1
2416
leaq 32(%rcx),%rcx
2417
xorps %xmm0,%xmm2
2418
.Loop_dec1_12:
2419
.byte 102,15,56,222,209
2420
decl %eax
2421
movups (%rcx),%xmm1
2422
leaq 16(%rcx),%rcx
2423
jnz .Loop_dec1_12
2424
.byte 102,15,56,223,209
2425
xorps %xmm10,%xmm2
2426
movdqa %xmm11,%xmm10
2427
movups %xmm2,(%rsi)
2428
movdqa %xmm12,%xmm11
2429
leaq 16(%rsi),%rsi
2430
jmp .Lxts_dec_done
2431
2432
.align 16
2433
.Lxts_dec_two:
2434
movups (%rdi),%xmm2
2435
movups 16(%rdi),%xmm3
2436
leaq 32(%rdi),%rdi
2437
xorps %xmm10,%xmm2
2438
xorps %xmm11,%xmm3
2439
2440
call _aesni_decrypt2
2441
2442
xorps %xmm10,%xmm2
2443
movdqa %xmm12,%xmm10
2444
xorps %xmm11,%xmm3
2445
movdqa %xmm13,%xmm11
2446
movups %xmm2,(%rsi)
2447
movups %xmm3,16(%rsi)
2448
leaq 32(%rsi),%rsi
2449
jmp .Lxts_dec_done
2450
2451
.align 16
2452
.Lxts_dec_three:
2453
movups (%rdi),%xmm2
2454
movups 16(%rdi),%xmm3
2455
movups 32(%rdi),%xmm4
2456
leaq 48(%rdi),%rdi
2457
xorps %xmm10,%xmm2
2458
xorps %xmm11,%xmm3
2459
xorps %xmm12,%xmm4
2460
2461
call _aesni_decrypt3
2462
2463
xorps %xmm10,%xmm2
2464
movdqa %xmm13,%xmm10
2465
xorps %xmm11,%xmm3
2466
movdqa %xmm14,%xmm11
2467
xorps %xmm12,%xmm4
2468
movups %xmm2,(%rsi)
2469
movups %xmm3,16(%rsi)
2470
movups %xmm4,32(%rsi)
2471
leaq 48(%rsi),%rsi
2472
jmp .Lxts_dec_done
2473
2474
.align 16
2475
.Lxts_dec_four:
2476
movups (%rdi),%xmm2
2477
movups 16(%rdi),%xmm3
2478
movups 32(%rdi),%xmm4
2479
xorps %xmm10,%xmm2
2480
movups 48(%rdi),%xmm5
2481
leaq 64(%rdi),%rdi
2482
xorps %xmm11,%xmm3
2483
xorps %xmm12,%xmm4
2484
xorps %xmm13,%xmm5
2485
2486
call _aesni_decrypt4
2487
2488
pxor %xmm10,%xmm2
2489
movdqa %xmm14,%xmm10
2490
pxor %xmm11,%xmm3
2491
movdqa %xmm15,%xmm11
2492
pxor %xmm12,%xmm4
2493
movdqu %xmm2,(%rsi)
2494
pxor %xmm13,%xmm5
2495
movdqu %xmm3,16(%rsi)
2496
movdqu %xmm4,32(%rsi)
2497
movdqu %xmm5,48(%rsi)
2498
leaq 64(%rsi),%rsi
2499
jmp .Lxts_dec_done
2500
2501
.align 16
2502
.Lxts_dec_done:
2503
andq $15,%r9
2504
jz .Lxts_dec_ret
2505
.Lxts_dec_done2:
2506
movq %r9,%rdx
2507
movq %rbp,%rcx
2508
movl %r10d,%eax
2509
2510
movups (%rdi),%xmm2
2511
xorps %xmm11,%xmm2
2512
movups (%rcx),%xmm0
2513
movups 16(%rcx),%xmm1
2514
leaq 32(%rcx),%rcx
2515
xorps %xmm0,%xmm2
2516
.Loop_dec1_13:
2517
.byte 102,15,56,222,209
2518
decl %eax
2519
movups (%rcx),%xmm1
2520
leaq 16(%rcx),%rcx
2521
jnz .Loop_dec1_13
2522
.byte 102,15,56,223,209
2523
xorps %xmm11,%xmm2
2524
movups %xmm2,(%rsi)
2525
2526
.Lxts_dec_steal:
2527
movzbl 16(%rdi),%eax
2528
movzbl (%rsi),%ecx
2529
leaq 1(%rdi),%rdi
2530
movb %al,(%rsi)
2531
movb %cl,16(%rsi)
2532
leaq 1(%rsi),%rsi
2533
subq $1,%rdx
2534
jnz .Lxts_dec_steal
2535
2536
subq %r9,%rsi
2537
movq %rbp,%rcx
2538
movl %r10d,%eax
2539
2540
movups (%rsi),%xmm2
2541
xorps %xmm10,%xmm2
2542
movups (%rcx),%xmm0
2543
movups 16(%rcx),%xmm1
2544
leaq 32(%rcx),%rcx
2545
xorps %xmm0,%xmm2
2546
.Loop_dec1_14:
2547
.byte 102,15,56,222,209
2548
decl %eax
2549
movups (%rcx),%xmm1
2550
leaq 16(%rcx),%rcx
2551
jnz .Loop_dec1_14
2552
.byte 102,15,56,223,209
2553
xorps %xmm10,%xmm2
2554
movups %xmm2,(%rsi)
2555
2556
.Lxts_dec_ret:
2557
xorps %xmm0,%xmm0
2558
pxor %xmm1,%xmm1
2559
pxor %xmm2,%xmm2
2560
pxor %xmm3,%xmm3
2561
pxor %xmm4,%xmm4
2562
pxor %xmm5,%xmm5
2563
pxor %xmm6,%xmm6
2564
pxor %xmm7,%xmm7
2565
movaps %xmm0,0(%rsp)
2566
pxor %xmm8,%xmm8
2567
movaps %xmm0,16(%rsp)
2568
pxor %xmm9,%xmm9
2569
movaps %xmm0,32(%rsp)
2570
pxor %xmm10,%xmm10
2571
movaps %xmm0,48(%rsp)
2572
pxor %xmm11,%xmm11
2573
movaps %xmm0,64(%rsp)
2574
pxor %xmm12,%xmm12
2575
movaps %xmm0,80(%rsp)
2576
pxor %xmm13,%xmm13
2577
movaps %xmm0,96(%rsp)
2578
pxor %xmm14,%xmm14
2579
pxor %xmm15,%xmm15
2580
movq -8(%r11),%rbp
2581
.cfi_restore %rbp
2582
leaq (%r11),%rsp
2583
.cfi_def_cfa_register %rsp
2584
.Lxts_dec_epilogue:
2585
.byte 0xf3,0xc3
2586
.cfi_endproc
2587
.size aesni_xts_decrypt,.-aesni_xts_decrypt
2588
.globl aesni_ocb_encrypt
2589
.type aesni_ocb_encrypt,@function
2590
.align 32
2591
aesni_ocb_encrypt:
2592
.cfi_startproc
2593
.byte 243,15,30,250
2594
leaq (%rsp),%rax
2595
pushq %rbx
2596
.cfi_adjust_cfa_offset 8
2597
.cfi_offset %rbx,-16
2598
pushq %rbp
2599
.cfi_adjust_cfa_offset 8
2600
.cfi_offset %rbp,-24
2601
pushq %r12
2602
.cfi_adjust_cfa_offset 8
2603
.cfi_offset %r12,-32
2604
pushq %r13
2605
.cfi_adjust_cfa_offset 8
2606
.cfi_offset %r13,-40
2607
pushq %r14
2608
.cfi_adjust_cfa_offset 8
2609
.cfi_offset %r14,-48
2610
movq 8(%rax),%rbx
2611
movq 8+8(%rax),%rbp
2612
2613
movl 240(%rcx),%r10d
2614
movq %rcx,%r11
2615
shll $4,%r10d
2616
movups (%rcx),%xmm9
2617
movups 16(%rcx,%r10,1),%xmm1
2618
2619
movdqu (%r9),%xmm15
2620
pxor %xmm1,%xmm9
2621
pxor %xmm1,%xmm15
2622
2623
movl $16+32,%eax
2624
leaq 32(%r11,%r10,1),%rcx
2625
movups 16(%r11),%xmm1
2626
subq %r10,%rax
2627
movq %rax,%r10
2628
2629
movdqu (%rbx),%xmm10
2630
movdqu (%rbp),%xmm8
2631
2632
testq $1,%r8
2633
jnz .Locb_enc_odd
2634
2635
bsfq %r8,%r12
2636
addq $1,%r8
2637
shlq $4,%r12
2638
movdqu (%rbx,%r12,1),%xmm7
2639
movdqu (%rdi),%xmm2
2640
leaq 16(%rdi),%rdi
2641
2642
call __ocb_encrypt1
2643
2644
movdqa %xmm7,%xmm15
2645
movups %xmm2,(%rsi)
2646
leaq 16(%rsi),%rsi
2647
subq $1,%rdx
2648
jz .Locb_enc_done
2649
2650
.Locb_enc_odd:
2651
leaq 1(%r8),%r12
2652
leaq 3(%r8),%r13
2653
leaq 5(%r8),%r14
2654
leaq 6(%r8),%r8
2655
bsfq %r12,%r12
2656
bsfq %r13,%r13
2657
bsfq %r14,%r14
2658
shlq $4,%r12
2659
shlq $4,%r13
2660
shlq $4,%r14
2661
2662
subq $6,%rdx
2663
jc .Locb_enc_short
2664
jmp .Locb_enc_grandloop
2665
2666
.align 32
2667
.Locb_enc_grandloop:
2668
movdqu 0(%rdi),%xmm2
2669
movdqu 16(%rdi),%xmm3
2670
movdqu 32(%rdi),%xmm4
2671
movdqu 48(%rdi),%xmm5
2672
movdqu 64(%rdi),%xmm6
2673
movdqu 80(%rdi),%xmm7
2674
leaq 96(%rdi),%rdi
2675
2676
call __ocb_encrypt6
2677
2678
movups %xmm2,0(%rsi)
2679
movups %xmm3,16(%rsi)
2680
movups %xmm4,32(%rsi)
2681
movups %xmm5,48(%rsi)
2682
movups %xmm6,64(%rsi)
2683
movups %xmm7,80(%rsi)
2684
leaq 96(%rsi),%rsi
2685
subq $6,%rdx
2686
jnc .Locb_enc_grandloop
2687
2688
.Locb_enc_short:
2689
addq $6,%rdx
2690
jz .Locb_enc_done
2691
2692
movdqu 0(%rdi),%xmm2
2693
cmpq $2,%rdx
2694
jb .Locb_enc_one
2695
movdqu 16(%rdi),%xmm3
2696
je .Locb_enc_two
2697
2698
movdqu 32(%rdi),%xmm4
2699
cmpq $4,%rdx
2700
jb .Locb_enc_three
2701
movdqu 48(%rdi),%xmm5
2702
je .Locb_enc_four
2703
2704
movdqu 64(%rdi),%xmm6
2705
pxor %xmm7,%xmm7
2706
2707
call __ocb_encrypt6
2708
2709
movdqa %xmm14,%xmm15
2710
movups %xmm2,0(%rsi)
2711
movups %xmm3,16(%rsi)
2712
movups %xmm4,32(%rsi)
2713
movups %xmm5,48(%rsi)
2714
movups %xmm6,64(%rsi)
2715
2716
jmp .Locb_enc_done
2717
2718
.align 16
2719
.Locb_enc_one:
2720
movdqa %xmm10,%xmm7
2721
2722
call __ocb_encrypt1
2723
2724
movdqa %xmm7,%xmm15
2725
movups %xmm2,0(%rsi)
2726
jmp .Locb_enc_done
2727
2728
.align 16
2729
.Locb_enc_two:
2730
pxor %xmm4,%xmm4
2731
pxor %xmm5,%xmm5
2732
2733
call __ocb_encrypt4
2734
2735
movdqa %xmm11,%xmm15
2736
movups %xmm2,0(%rsi)
2737
movups %xmm3,16(%rsi)
2738
2739
jmp .Locb_enc_done
2740
2741
.align 16
2742
.Locb_enc_three:
2743
pxor %xmm5,%xmm5
2744
2745
call __ocb_encrypt4
2746
2747
movdqa %xmm12,%xmm15
2748
movups %xmm2,0(%rsi)
2749
movups %xmm3,16(%rsi)
2750
movups %xmm4,32(%rsi)
2751
2752
jmp .Locb_enc_done
2753
2754
.align 16
2755
.Locb_enc_four:
2756
call __ocb_encrypt4
2757
2758
movdqa %xmm13,%xmm15
2759
movups %xmm2,0(%rsi)
2760
movups %xmm3,16(%rsi)
2761
movups %xmm4,32(%rsi)
2762
movups %xmm5,48(%rsi)
2763
2764
.Locb_enc_done:
2765
pxor %xmm0,%xmm15
2766
movdqu %xmm8,(%rbp)
2767
movdqu %xmm15,(%r9)
2768
2769
xorps %xmm0,%xmm0
2770
pxor %xmm1,%xmm1
2771
pxor %xmm2,%xmm2
2772
pxor %xmm3,%xmm3
2773
pxor %xmm4,%xmm4
2774
pxor %xmm5,%xmm5
2775
pxor %xmm6,%xmm6
2776
pxor %xmm7,%xmm7
2777
pxor %xmm8,%xmm8
2778
pxor %xmm9,%xmm9
2779
pxor %xmm10,%xmm10
2780
pxor %xmm11,%xmm11
2781
pxor %xmm12,%xmm12
2782
pxor %xmm13,%xmm13
2783
pxor %xmm14,%xmm14
2784
pxor %xmm15,%xmm15
2785
leaq 40(%rsp),%rax
2786
.cfi_def_cfa %rax,8
2787
movq -40(%rax),%r14
2788
.cfi_restore %r14
2789
movq -32(%rax),%r13
2790
.cfi_restore %r13
2791
movq -24(%rax),%r12
2792
.cfi_restore %r12
2793
movq -16(%rax),%rbp
2794
.cfi_restore %rbp
2795
movq -8(%rax),%rbx
2796
.cfi_restore %rbx
2797
leaq (%rax),%rsp
2798
.cfi_def_cfa_register %rsp
2799
.Locb_enc_epilogue:
2800
.byte 0xf3,0xc3
2801
.cfi_endproc
2802
.size aesni_ocb_encrypt,.-aesni_ocb_encrypt
2803
2804
.type __ocb_encrypt6,@function
2805
.align 32
2806
__ocb_encrypt6:
2807
.cfi_startproc
2808
pxor %xmm9,%xmm15
2809
movdqu (%rbx,%r12,1),%xmm11
2810
movdqa %xmm10,%xmm12
2811
movdqu (%rbx,%r13,1),%xmm13
2812
movdqa %xmm10,%xmm14
2813
pxor %xmm15,%xmm10
2814
movdqu (%rbx,%r14,1),%xmm15
2815
pxor %xmm10,%xmm11
2816
pxor %xmm2,%xmm8
2817
pxor %xmm10,%xmm2
2818
pxor %xmm11,%xmm12
2819
pxor %xmm3,%xmm8
2820
pxor %xmm11,%xmm3
2821
pxor %xmm12,%xmm13
2822
pxor %xmm4,%xmm8
2823
pxor %xmm12,%xmm4
2824
pxor %xmm13,%xmm14
2825
pxor %xmm5,%xmm8
2826
pxor %xmm13,%xmm5
2827
pxor %xmm14,%xmm15
2828
pxor %xmm6,%xmm8
2829
pxor %xmm14,%xmm6
2830
pxor %xmm7,%xmm8
2831
pxor %xmm15,%xmm7
2832
movups 32(%r11),%xmm0
2833
2834
leaq 1(%r8),%r12
2835
leaq 3(%r8),%r13
2836
leaq 5(%r8),%r14
2837
addq $6,%r8
2838
pxor %xmm9,%xmm10
2839
bsfq %r12,%r12
2840
bsfq %r13,%r13
2841
bsfq %r14,%r14
2842
2843
.byte 102,15,56,220,209
2844
.byte 102,15,56,220,217
2845
.byte 102,15,56,220,225
2846
.byte 102,15,56,220,233
2847
pxor %xmm9,%xmm11
2848
pxor %xmm9,%xmm12
2849
.byte 102,15,56,220,241
2850
pxor %xmm9,%xmm13
2851
pxor %xmm9,%xmm14
2852
.byte 102,15,56,220,249
2853
movups 48(%r11),%xmm1
2854
pxor %xmm9,%xmm15
2855
2856
.byte 102,15,56,220,208
2857
.byte 102,15,56,220,216
2858
.byte 102,15,56,220,224
2859
.byte 102,15,56,220,232
2860
.byte 102,15,56,220,240
2861
.byte 102,15,56,220,248
2862
movups 64(%r11),%xmm0
2863
shlq $4,%r12
2864
shlq $4,%r13
2865
jmp .Locb_enc_loop6
2866
2867
.align 32
2868
.Locb_enc_loop6:
2869
.byte 102,15,56,220,209
2870
.byte 102,15,56,220,217
2871
.byte 102,15,56,220,225
2872
.byte 102,15,56,220,233
2873
.byte 102,15,56,220,241
2874
.byte 102,15,56,220,249
2875
movups (%rcx,%rax,1),%xmm1
2876
addq $32,%rax
2877
2878
.byte 102,15,56,220,208
2879
.byte 102,15,56,220,216
2880
.byte 102,15,56,220,224
2881
.byte 102,15,56,220,232
2882
.byte 102,15,56,220,240
2883
.byte 102,15,56,220,248
2884
movups -16(%rcx,%rax,1),%xmm0
2885
jnz .Locb_enc_loop6
2886
2887
.byte 102,15,56,220,209
2888
.byte 102,15,56,220,217
2889
.byte 102,15,56,220,225
2890
.byte 102,15,56,220,233
2891
.byte 102,15,56,220,241
2892
.byte 102,15,56,220,249
2893
movups 16(%r11),%xmm1
2894
shlq $4,%r14
2895
2896
.byte 102,65,15,56,221,210
2897
movdqu (%rbx),%xmm10
2898
movq %r10,%rax
2899
.byte 102,65,15,56,221,219
2900
.byte 102,65,15,56,221,228
2901
.byte 102,65,15,56,221,237
2902
.byte 102,65,15,56,221,246
2903
.byte 102,65,15,56,221,255
2904
.byte 0xf3,0xc3
2905
.cfi_endproc
2906
.size __ocb_encrypt6,.-__ocb_encrypt6
2907
2908
.type __ocb_encrypt4,@function
2909
.align 32
2910
__ocb_encrypt4:
2911
.cfi_startproc
2912
pxor %xmm9,%xmm15
2913
movdqu (%rbx,%r12,1),%xmm11
2914
movdqa %xmm10,%xmm12
2915
movdqu (%rbx,%r13,1),%xmm13
2916
pxor %xmm15,%xmm10
2917
pxor %xmm10,%xmm11
2918
pxor %xmm2,%xmm8
2919
pxor %xmm10,%xmm2
2920
pxor %xmm11,%xmm12
2921
pxor %xmm3,%xmm8
2922
pxor %xmm11,%xmm3
2923
pxor %xmm12,%xmm13
2924
pxor %xmm4,%xmm8
2925
pxor %xmm12,%xmm4
2926
pxor %xmm5,%xmm8
2927
pxor %xmm13,%xmm5
2928
movups 32(%r11),%xmm0
2929
2930
pxor %xmm9,%xmm10
2931
pxor %xmm9,%xmm11
2932
pxor %xmm9,%xmm12
2933
pxor %xmm9,%xmm13
2934
2935
.byte 102,15,56,220,209
2936
.byte 102,15,56,220,217
2937
.byte 102,15,56,220,225
2938
.byte 102,15,56,220,233
2939
movups 48(%r11),%xmm1
2940
2941
.byte 102,15,56,220,208
2942
.byte 102,15,56,220,216
2943
.byte 102,15,56,220,224
2944
.byte 102,15,56,220,232
2945
movups 64(%r11),%xmm0
2946
jmp .Locb_enc_loop4
2947
2948
.align 32
2949
.Locb_enc_loop4:
2950
.byte 102,15,56,220,209
2951
.byte 102,15,56,220,217
2952
.byte 102,15,56,220,225
2953
.byte 102,15,56,220,233
2954
movups (%rcx,%rax,1),%xmm1
2955
addq $32,%rax
2956
2957
.byte 102,15,56,220,208
2958
.byte 102,15,56,220,216
2959
.byte 102,15,56,220,224
2960
.byte 102,15,56,220,232
2961
movups -16(%rcx,%rax,1),%xmm0
2962
jnz .Locb_enc_loop4
2963
2964
.byte 102,15,56,220,209
2965
.byte 102,15,56,220,217
2966
.byte 102,15,56,220,225
2967
.byte 102,15,56,220,233
2968
movups 16(%r11),%xmm1
2969
movq %r10,%rax
2970
2971
.byte 102,65,15,56,221,210
2972
.byte 102,65,15,56,221,219
2973
.byte 102,65,15,56,221,228
2974
.byte 102,65,15,56,221,237
2975
.byte 0xf3,0xc3
2976
.cfi_endproc
2977
.size __ocb_encrypt4,.-__ocb_encrypt4
2978
2979
.type __ocb_encrypt1,@function
2980
.align 32
2981
__ocb_encrypt1:
2982
.cfi_startproc
2983
pxor %xmm15,%xmm7
2984
pxor %xmm9,%xmm7
2985
pxor %xmm2,%xmm8
2986
pxor %xmm7,%xmm2
2987
movups 32(%r11),%xmm0
2988
2989
.byte 102,15,56,220,209
2990
movups 48(%r11),%xmm1
2991
pxor %xmm9,%xmm7
2992
2993
.byte 102,15,56,220,208
2994
movups 64(%r11),%xmm0
2995
jmp .Locb_enc_loop1
2996
2997
.align 32
2998
.Locb_enc_loop1:
2999
.byte 102,15,56,220,209
3000
movups (%rcx,%rax,1),%xmm1
3001
addq $32,%rax
3002
3003
.byte 102,15,56,220,208
3004
movups -16(%rcx,%rax,1),%xmm0
3005
jnz .Locb_enc_loop1
3006
3007
.byte 102,15,56,220,209
3008
movups 16(%r11),%xmm1
3009
movq %r10,%rax
3010
3011
.byte 102,15,56,221,215
3012
.byte 0xf3,0xc3
3013
.cfi_endproc
3014
.size __ocb_encrypt1,.-__ocb_encrypt1
3015
3016
.globl aesni_ocb_decrypt
3017
.type aesni_ocb_decrypt,@function
3018
.align 32
3019
aesni_ocb_decrypt:
3020
.cfi_startproc
3021
.byte 243,15,30,250
3022
leaq (%rsp),%rax
3023
pushq %rbx
3024
.cfi_adjust_cfa_offset 8
3025
.cfi_offset %rbx,-16
3026
pushq %rbp
3027
.cfi_adjust_cfa_offset 8
3028
.cfi_offset %rbp,-24
3029
pushq %r12
3030
.cfi_adjust_cfa_offset 8
3031
.cfi_offset %r12,-32
3032
pushq %r13
3033
.cfi_adjust_cfa_offset 8
3034
.cfi_offset %r13,-40
3035
pushq %r14
3036
.cfi_adjust_cfa_offset 8
3037
.cfi_offset %r14,-48
3038
movq 8(%rax),%rbx
3039
movq 8+8(%rax),%rbp
3040
3041
movl 240(%rcx),%r10d
3042
movq %rcx,%r11
3043
shll $4,%r10d
3044
movups (%rcx),%xmm9
3045
movups 16(%rcx,%r10,1),%xmm1
3046
3047
movdqu (%r9),%xmm15
3048
pxor %xmm1,%xmm9
3049
pxor %xmm1,%xmm15
3050
3051
movl $16+32,%eax
3052
leaq 32(%r11,%r10,1),%rcx
3053
movups 16(%r11),%xmm1
3054
subq %r10,%rax
3055
movq %rax,%r10
3056
3057
movdqu (%rbx),%xmm10
3058
movdqu (%rbp),%xmm8
3059
3060
testq $1,%r8
3061
jnz .Locb_dec_odd
3062
3063
bsfq %r8,%r12
3064
addq $1,%r8
3065
shlq $4,%r12
3066
movdqu (%rbx,%r12,1),%xmm7
3067
movdqu (%rdi),%xmm2
3068
leaq 16(%rdi),%rdi
3069
3070
call __ocb_decrypt1
3071
3072
movdqa %xmm7,%xmm15
3073
movups %xmm2,(%rsi)
3074
xorps %xmm2,%xmm8
3075
leaq 16(%rsi),%rsi
3076
subq $1,%rdx
3077
jz .Locb_dec_done
3078
3079
.Locb_dec_odd:
3080
leaq 1(%r8),%r12
3081
leaq 3(%r8),%r13
3082
leaq 5(%r8),%r14
3083
leaq 6(%r8),%r8
3084
bsfq %r12,%r12
3085
bsfq %r13,%r13
3086
bsfq %r14,%r14
3087
shlq $4,%r12
3088
shlq $4,%r13
3089
shlq $4,%r14
3090
3091
subq $6,%rdx
3092
jc .Locb_dec_short
3093
jmp .Locb_dec_grandloop
3094
3095
.align 32
3096
.Locb_dec_grandloop:
3097
movdqu 0(%rdi),%xmm2
3098
movdqu 16(%rdi),%xmm3
3099
movdqu 32(%rdi),%xmm4
3100
movdqu 48(%rdi),%xmm5
3101
movdqu 64(%rdi),%xmm6
3102
movdqu 80(%rdi),%xmm7
3103
leaq 96(%rdi),%rdi
3104
3105
call __ocb_decrypt6
3106
3107
movups %xmm2,0(%rsi)
3108
pxor %xmm2,%xmm8
3109
movups %xmm3,16(%rsi)
3110
pxor %xmm3,%xmm8
3111
movups %xmm4,32(%rsi)
3112
pxor %xmm4,%xmm8
3113
movups %xmm5,48(%rsi)
3114
pxor %xmm5,%xmm8
3115
movups %xmm6,64(%rsi)
3116
pxor %xmm6,%xmm8
3117
movups %xmm7,80(%rsi)
3118
pxor %xmm7,%xmm8
3119
leaq 96(%rsi),%rsi
3120
subq $6,%rdx
3121
jnc .Locb_dec_grandloop
3122
3123
.Locb_dec_short:
3124
addq $6,%rdx
3125
jz .Locb_dec_done
3126
3127
movdqu 0(%rdi),%xmm2
3128
cmpq $2,%rdx
3129
jb .Locb_dec_one
3130
movdqu 16(%rdi),%xmm3
3131
je .Locb_dec_two
3132
3133
movdqu 32(%rdi),%xmm4
3134
cmpq $4,%rdx
3135
jb .Locb_dec_three
3136
movdqu 48(%rdi),%xmm5
3137
je .Locb_dec_four
3138
3139
movdqu 64(%rdi),%xmm6
3140
pxor %xmm7,%xmm7
3141
3142
call __ocb_decrypt6
3143
3144
movdqa %xmm14,%xmm15
3145
movups %xmm2,0(%rsi)
3146
pxor %xmm2,%xmm8
3147
movups %xmm3,16(%rsi)
3148
pxor %xmm3,%xmm8
3149
movups %xmm4,32(%rsi)
3150
pxor %xmm4,%xmm8
3151
movups %xmm5,48(%rsi)
3152
pxor %xmm5,%xmm8
3153
movups %xmm6,64(%rsi)
3154
pxor %xmm6,%xmm8
3155
3156
jmp .Locb_dec_done
3157
3158
.align 16
3159
.Locb_dec_one:
3160
movdqa %xmm10,%xmm7
3161
3162
call __ocb_decrypt1
3163
3164
movdqa %xmm7,%xmm15
3165
movups %xmm2,0(%rsi)
3166
xorps %xmm2,%xmm8
3167
jmp .Locb_dec_done
3168
3169
.align 16
3170
.Locb_dec_two:
3171
pxor %xmm4,%xmm4
3172
pxor %xmm5,%xmm5
3173
3174
call __ocb_decrypt4
3175
3176
movdqa %xmm11,%xmm15
3177
movups %xmm2,0(%rsi)
3178
xorps %xmm2,%xmm8
3179
movups %xmm3,16(%rsi)
3180
xorps %xmm3,%xmm8
3181
3182
jmp .Locb_dec_done
3183
3184
.align 16
3185
.Locb_dec_three:
3186
pxor %xmm5,%xmm5
3187
3188
call __ocb_decrypt4
3189
3190
movdqa %xmm12,%xmm15
3191
movups %xmm2,0(%rsi)
3192
xorps %xmm2,%xmm8
3193
movups %xmm3,16(%rsi)
3194
xorps %xmm3,%xmm8
3195
movups %xmm4,32(%rsi)
3196
xorps %xmm4,%xmm8
3197
3198
jmp .Locb_dec_done
3199
3200
.align 16
3201
.Locb_dec_four:
3202
call __ocb_decrypt4
3203
3204
movdqa %xmm13,%xmm15
3205
movups %xmm2,0(%rsi)
3206
pxor %xmm2,%xmm8
3207
movups %xmm3,16(%rsi)
3208
pxor %xmm3,%xmm8
3209
movups %xmm4,32(%rsi)
3210
pxor %xmm4,%xmm8
3211
movups %xmm5,48(%rsi)
3212
pxor %xmm5,%xmm8
3213
3214
.Locb_dec_done:
3215
pxor %xmm0,%xmm15
3216
movdqu %xmm8,(%rbp)
3217
movdqu %xmm15,(%r9)
3218
3219
xorps %xmm0,%xmm0
3220
pxor %xmm1,%xmm1
3221
pxor %xmm2,%xmm2
3222
pxor %xmm3,%xmm3
3223
pxor %xmm4,%xmm4
3224
pxor %xmm5,%xmm5
3225
pxor %xmm6,%xmm6
3226
pxor %xmm7,%xmm7
3227
pxor %xmm8,%xmm8
3228
pxor %xmm9,%xmm9
3229
pxor %xmm10,%xmm10
3230
pxor %xmm11,%xmm11
3231
pxor %xmm12,%xmm12
3232
pxor %xmm13,%xmm13
3233
pxor %xmm14,%xmm14
3234
pxor %xmm15,%xmm15
3235
leaq 40(%rsp),%rax
3236
.cfi_def_cfa %rax,8
3237
movq -40(%rax),%r14
3238
.cfi_restore %r14
3239
movq -32(%rax),%r13
3240
.cfi_restore %r13
3241
movq -24(%rax),%r12
3242
.cfi_restore %r12
3243
movq -16(%rax),%rbp
3244
.cfi_restore %rbp
3245
movq -8(%rax),%rbx
3246
.cfi_restore %rbx
3247
leaq (%rax),%rsp
3248
.cfi_def_cfa_register %rsp
3249
.Locb_dec_epilogue:
3250
.byte 0xf3,0xc3
3251
.cfi_endproc
3252
.size aesni_ocb_decrypt,.-aesni_ocb_decrypt
3253
3254
.type __ocb_decrypt6,@function
3255
.align 32
3256
__ocb_decrypt6:
3257
.cfi_startproc
3258
pxor %xmm9,%xmm15
3259
movdqu (%rbx,%r12,1),%xmm11
3260
movdqa %xmm10,%xmm12
3261
movdqu (%rbx,%r13,1),%xmm13
3262
movdqa %xmm10,%xmm14
3263
pxor %xmm15,%xmm10
3264
movdqu (%rbx,%r14,1),%xmm15
3265
pxor %xmm10,%xmm11
3266
pxor %xmm10,%xmm2
3267
pxor %xmm11,%xmm12
3268
pxor %xmm11,%xmm3
3269
pxor %xmm12,%xmm13
3270
pxor %xmm12,%xmm4
3271
pxor %xmm13,%xmm14
3272
pxor %xmm13,%xmm5
3273
pxor %xmm14,%xmm15
3274
pxor %xmm14,%xmm6
3275
pxor %xmm15,%xmm7
3276
movups 32(%r11),%xmm0
3277
3278
leaq 1(%r8),%r12
3279
leaq 3(%r8),%r13
3280
leaq 5(%r8),%r14
3281
addq $6,%r8
3282
pxor %xmm9,%xmm10
3283
bsfq %r12,%r12
3284
bsfq %r13,%r13
3285
bsfq %r14,%r14
3286
3287
.byte 102,15,56,222,209
3288
.byte 102,15,56,222,217
3289
.byte 102,15,56,222,225
3290
.byte 102,15,56,222,233
3291
pxor %xmm9,%xmm11
3292
pxor %xmm9,%xmm12
3293
.byte 102,15,56,222,241
3294
pxor %xmm9,%xmm13
3295
pxor %xmm9,%xmm14
3296
.byte 102,15,56,222,249
3297
movups 48(%r11),%xmm1
3298
pxor %xmm9,%xmm15
3299
3300
.byte 102,15,56,222,208
3301
.byte 102,15,56,222,216
3302
.byte 102,15,56,222,224
3303
.byte 102,15,56,222,232
3304
.byte 102,15,56,222,240
3305
.byte 102,15,56,222,248
3306
movups 64(%r11),%xmm0
3307
shlq $4,%r12
3308
shlq $4,%r13
3309
jmp .Locb_dec_loop6
3310
3311
.align 32
3312
.Locb_dec_loop6:
3313
.byte 102,15,56,222,209
3314
.byte 102,15,56,222,217
3315
.byte 102,15,56,222,225
3316
.byte 102,15,56,222,233
3317
.byte 102,15,56,222,241
3318
.byte 102,15,56,222,249
3319
movups (%rcx,%rax,1),%xmm1
3320
addq $32,%rax
3321
3322
.byte 102,15,56,222,208
3323
.byte 102,15,56,222,216
3324
.byte 102,15,56,222,224
3325
.byte 102,15,56,222,232
3326
.byte 102,15,56,222,240
3327
.byte 102,15,56,222,248
3328
movups -16(%rcx,%rax,1),%xmm0
3329
jnz .Locb_dec_loop6
3330
3331
.byte 102,15,56,222,209
3332
.byte 102,15,56,222,217
3333
.byte 102,15,56,222,225
3334
.byte 102,15,56,222,233
3335
.byte 102,15,56,222,241
3336
.byte 102,15,56,222,249
3337
movups 16(%r11),%xmm1
3338
shlq $4,%r14
3339
3340
.byte 102,65,15,56,223,210
3341
movdqu (%rbx),%xmm10
3342
movq %r10,%rax
3343
.byte 102,65,15,56,223,219
3344
.byte 102,65,15,56,223,228
3345
.byte 102,65,15,56,223,237
3346
.byte 102,65,15,56,223,246
3347
.byte 102,65,15,56,223,255
3348
.byte 0xf3,0xc3
3349
.cfi_endproc
3350
.size __ocb_decrypt6,.-__ocb_decrypt6
3351
3352
.type __ocb_decrypt4,@function
3353
.align 32
3354
__ocb_decrypt4:
3355
.cfi_startproc
3356
pxor %xmm9,%xmm15
3357
movdqu (%rbx,%r12,1),%xmm11
3358
movdqa %xmm10,%xmm12
3359
movdqu (%rbx,%r13,1),%xmm13
3360
pxor %xmm15,%xmm10
3361
pxor %xmm10,%xmm11
3362
pxor %xmm10,%xmm2
3363
pxor %xmm11,%xmm12
3364
pxor %xmm11,%xmm3
3365
pxor %xmm12,%xmm13
3366
pxor %xmm12,%xmm4
3367
pxor %xmm13,%xmm5
3368
movups 32(%r11),%xmm0
3369
3370
pxor %xmm9,%xmm10
3371
pxor %xmm9,%xmm11
3372
pxor %xmm9,%xmm12
3373
pxor %xmm9,%xmm13
3374
3375
.byte 102,15,56,222,209
3376
.byte 102,15,56,222,217
3377
.byte 102,15,56,222,225
3378
.byte 102,15,56,222,233
3379
movups 48(%r11),%xmm1
3380
3381
.byte 102,15,56,222,208
3382
.byte 102,15,56,222,216
3383
.byte 102,15,56,222,224
3384
.byte 102,15,56,222,232
3385
movups 64(%r11),%xmm0
3386
jmp .Locb_dec_loop4
3387
3388
.align 32
3389
.Locb_dec_loop4:
3390
.byte 102,15,56,222,209
3391
.byte 102,15,56,222,217
3392
.byte 102,15,56,222,225
3393
.byte 102,15,56,222,233
3394
movups (%rcx,%rax,1),%xmm1
3395
addq $32,%rax
3396
3397
.byte 102,15,56,222,208
3398
.byte 102,15,56,222,216
3399
.byte 102,15,56,222,224
3400
.byte 102,15,56,222,232
3401
movups -16(%rcx,%rax,1),%xmm0
3402
jnz .Locb_dec_loop4
3403
3404
.byte 102,15,56,222,209
3405
.byte 102,15,56,222,217
3406
.byte 102,15,56,222,225
3407
.byte 102,15,56,222,233
3408
movups 16(%r11),%xmm1
3409
movq %r10,%rax
3410
3411
.byte 102,65,15,56,223,210
3412
.byte 102,65,15,56,223,219
3413
.byte 102,65,15,56,223,228
3414
.byte 102,65,15,56,223,237
3415
.byte 0xf3,0xc3
3416
.cfi_endproc
3417
.size __ocb_decrypt4,.-__ocb_decrypt4
3418
3419
.type __ocb_decrypt1,@function
3420
.align 32
3421
__ocb_decrypt1:
3422
.cfi_startproc
3423
pxor %xmm15,%xmm7
3424
pxor %xmm9,%xmm7
3425
pxor %xmm7,%xmm2
3426
movups 32(%r11),%xmm0
3427
3428
.byte 102,15,56,222,209
3429
movups 48(%r11),%xmm1
3430
pxor %xmm9,%xmm7
3431
3432
.byte 102,15,56,222,208
3433
movups 64(%r11),%xmm0
3434
jmp .Locb_dec_loop1
3435
3436
.align 32
3437
.Locb_dec_loop1:
3438
.byte 102,15,56,222,209
3439
movups (%rcx,%rax,1),%xmm1
3440
addq $32,%rax
3441
3442
.byte 102,15,56,222,208
3443
movups -16(%rcx,%rax,1),%xmm0
3444
jnz .Locb_dec_loop1
3445
3446
.byte 102,15,56,222,209
3447
movups 16(%r11),%xmm1
3448
movq %r10,%rax
3449
3450
.byte 102,15,56,223,215
3451
.byte 0xf3,0xc3
3452
.cfi_endproc
3453
.size __ocb_decrypt1,.-__ocb_decrypt1
3454
.globl aesni_cbc_encrypt
3455
.type aesni_cbc_encrypt,@function
3456
.align 16
3457
aesni_cbc_encrypt:
3458
.cfi_startproc
3459
.byte 243,15,30,250
3460
testq %rdx,%rdx
3461
jz .Lcbc_ret
3462
3463
movl 240(%rcx),%r10d
3464
movq %rcx,%r11
3465
testl %r9d,%r9d
3466
jz .Lcbc_decrypt
3467
3468
movups (%r8),%xmm2
3469
movl %r10d,%eax
3470
cmpq $16,%rdx
3471
jb .Lcbc_enc_tail
3472
subq $16,%rdx
3473
jmp .Lcbc_enc_loop
3474
.align 16
3475
.Lcbc_enc_loop:
3476
movups (%rdi),%xmm3
3477
leaq 16(%rdi),%rdi
3478
3479
movups (%rcx),%xmm0
3480
movups 16(%rcx),%xmm1
3481
xorps %xmm0,%xmm3
3482
leaq 32(%rcx),%rcx
3483
xorps %xmm3,%xmm2
3484
.Loop_enc1_15:
3485
.byte 102,15,56,220,209
3486
decl %eax
3487
movups (%rcx),%xmm1
3488
leaq 16(%rcx),%rcx
3489
jnz .Loop_enc1_15
3490
.byte 102,15,56,221,209
3491
movl %r10d,%eax
3492
movq %r11,%rcx
3493
movups %xmm2,0(%rsi)
3494
leaq 16(%rsi),%rsi
3495
subq $16,%rdx
3496
jnc .Lcbc_enc_loop
3497
addq $16,%rdx
3498
jnz .Lcbc_enc_tail
3499
pxor %xmm0,%xmm0
3500
pxor %xmm1,%xmm1
3501
movups %xmm2,(%r8)
3502
pxor %xmm2,%xmm2
3503
pxor %xmm3,%xmm3
3504
jmp .Lcbc_ret
3505
3506
.Lcbc_enc_tail:
3507
movq %rdx,%rcx
3508
xchgq %rdi,%rsi
3509
.long 0x9066A4F3
3510
movl $16,%ecx
3511
subq %rdx,%rcx
3512
xorl %eax,%eax
3513
.long 0x9066AAF3
3514
leaq -16(%rdi),%rdi
3515
movl %r10d,%eax
3516
movq %rdi,%rsi
3517
movq %r11,%rcx
3518
xorq %rdx,%rdx
3519
jmp .Lcbc_enc_loop
3520
3521
.align 16
3522
.Lcbc_decrypt:
3523
cmpq $16,%rdx
3524
jne .Lcbc_decrypt_bulk
3525
3526
3527
3528
movdqu (%rdi),%xmm2
3529
movdqu (%r8),%xmm3
3530
movdqa %xmm2,%xmm4
3531
movups (%rcx),%xmm0
3532
movups 16(%rcx),%xmm1
3533
leaq 32(%rcx),%rcx
3534
xorps %xmm0,%xmm2
3535
.Loop_dec1_16:
3536
.byte 102,15,56,222,209
3537
decl %r10d
3538
movups (%rcx),%xmm1
3539
leaq 16(%rcx),%rcx
3540
jnz .Loop_dec1_16
3541
.byte 102,15,56,223,209
3542
pxor %xmm0,%xmm0
3543
pxor %xmm1,%xmm1
3544
movdqu %xmm4,(%r8)
3545
xorps %xmm3,%xmm2
3546
pxor %xmm3,%xmm3
3547
movups %xmm2,(%rsi)
3548
pxor %xmm2,%xmm2
3549
jmp .Lcbc_ret
3550
.align 16
3551
.Lcbc_decrypt_bulk:
3552
leaq (%rsp),%r11
3553
.cfi_def_cfa_register %r11
3554
pushq %rbp
3555
.cfi_offset %rbp,-16
3556
subq $16,%rsp
3557
andq $-16,%rsp
3558
movq %rcx,%rbp
3559
movups (%r8),%xmm10
3560
movl %r10d,%eax
3561
cmpq $0x50,%rdx
3562
jbe .Lcbc_dec_tail
3563
3564
movups (%rcx),%xmm0
3565
movdqu 0(%rdi),%xmm2
3566
movdqu 16(%rdi),%xmm3
3567
movdqa %xmm2,%xmm11
3568
movdqu 32(%rdi),%xmm4
3569
movdqa %xmm3,%xmm12
3570
movdqu 48(%rdi),%xmm5
3571
movdqa %xmm4,%xmm13
3572
movdqu 64(%rdi),%xmm6
3573
movdqa %xmm5,%xmm14
3574
movdqu 80(%rdi),%xmm7
3575
movdqa %xmm6,%xmm15
3576
movl OPENSSL_ia32cap_P+4(%rip),%r9d
3577
cmpq $0x70,%rdx
3578
jbe .Lcbc_dec_six_or_seven
3579
3580
andl $71303168,%r9d
3581
subq $0x50,%rdx
3582
cmpl $4194304,%r9d
3583
je .Lcbc_dec_loop6_enter
3584
subq $0x20,%rdx
3585
leaq 112(%rcx),%rcx
3586
jmp .Lcbc_dec_loop8_enter
3587
.align 16
3588
.Lcbc_dec_loop8:
3589
movups %xmm9,(%rsi)
3590
leaq 16(%rsi),%rsi
3591
.Lcbc_dec_loop8_enter:
3592
movdqu 96(%rdi),%xmm8
3593
pxor %xmm0,%xmm2
3594
movdqu 112(%rdi),%xmm9
3595
pxor %xmm0,%xmm3
3596
movups 16-112(%rcx),%xmm1
3597
pxor %xmm0,%xmm4
3598
movq $-1,%rbp
3599
cmpq $0x70,%rdx
3600
pxor %xmm0,%xmm5
3601
pxor %xmm0,%xmm6
3602
pxor %xmm0,%xmm7
3603
pxor %xmm0,%xmm8
3604
3605
.byte 102,15,56,222,209
3606
pxor %xmm0,%xmm9
3607
movups 32-112(%rcx),%xmm0
3608
.byte 102,15,56,222,217
3609
.byte 102,15,56,222,225
3610
.byte 102,15,56,222,233
3611
.byte 102,15,56,222,241
3612
.byte 102,15,56,222,249
3613
.byte 102,68,15,56,222,193
3614
adcq $0,%rbp
3615
andq $128,%rbp
3616
.byte 102,68,15,56,222,201
3617
addq %rdi,%rbp
3618
movups 48-112(%rcx),%xmm1
3619
.byte 102,15,56,222,208
3620
.byte 102,15,56,222,216
3621
.byte 102,15,56,222,224
3622
.byte 102,15,56,222,232
3623
.byte 102,15,56,222,240
3624
.byte 102,15,56,222,248
3625
.byte 102,68,15,56,222,192
3626
.byte 102,68,15,56,222,200
3627
movups 64-112(%rcx),%xmm0
3628
nop
3629
.byte 102,15,56,222,209
3630
.byte 102,15,56,222,217
3631
.byte 102,15,56,222,225
3632
.byte 102,15,56,222,233
3633
.byte 102,15,56,222,241
3634
.byte 102,15,56,222,249
3635
.byte 102,68,15,56,222,193
3636
.byte 102,68,15,56,222,201
3637
movups 80-112(%rcx),%xmm1
3638
nop
3639
.byte 102,15,56,222,208
3640
.byte 102,15,56,222,216
3641
.byte 102,15,56,222,224
3642
.byte 102,15,56,222,232
3643
.byte 102,15,56,222,240
3644
.byte 102,15,56,222,248
3645
.byte 102,68,15,56,222,192
3646
.byte 102,68,15,56,222,200
3647
movups 96-112(%rcx),%xmm0
3648
nop
3649
.byte 102,15,56,222,209
3650
.byte 102,15,56,222,217
3651
.byte 102,15,56,222,225
3652
.byte 102,15,56,222,233
3653
.byte 102,15,56,222,241
3654
.byte 102,15,56,222,249
3655
.byte 102,68,15,56,222,193
3656
.byte 102,68,15,56,222,201
3657
movups 112-112(%rcx),%xmm1
3658
nop
3659
.byte 102,15,56,222,208
3660
.byte 102,15,56,222,216
3661
.byte 102,15,56,222,224
3662
.byte 102,15,56,222,232
3663
.byte 102,15,56,222,240
3664
.byte 102,15,56,222,248
3665
.byte 102,68,15,56,222,192
3666
.byte 102,68,15,56,222,200
3667
movups 128-112(%rcx),%xmm0
3668
nop
3669
.byte 102,15,56,222,209
3670
.byte 102,15,56,222,217
3671
.byte 102,15,56,222,225
3672
.byte 102,15,56,222,233
3673
.byte 102,15,56,222,241
3674
.byte 102,15,56,222,249
3675
.byte 102,68,15,56,222,193
3676
.byte 102,68,15,56,222,201
3677
movups 144-112(%rcx),%xmm1
3678
cmpl $11,%eax
3679
.byte 102,15,56,222,208
3680
.byte 102,15,56,222,216
3681
.byte 102,15,56,222,224
3682
.byte 102,15,56,222,232
3683
.byte 102,15,56,222,240
3684
.byte 102,15,56,222,248
3685
.byte 102,68,15,56,222,192
3686
.byte 102,68,15,56,222,200
3687
movups 160-112(%rcx),%xmm0
3688
jb .Lcbc_dec_done
3689
.byte 102,15,56,222,209
3690
.byte 102,15,56,222,217
3691
.byte 102,15,56,222,225
3692
.byte 102,15,56,222,233
3693
.byte 102,15,56,222,241
3694
.byte 102,15,56,222,249
3695
.byte 102,68,15,56,222,193
3696
.byte 102,68,15,56,222,201
3697
movups 176-112(%rcx),%xmm1
3698
nop
3699
.byte 102,15,56,222,208
3700
.byte 102,15,56,222,216
3701
.byte 102,15,56,222,224
3702
.byte 102,15,56,222,232
3703
.byte 102,15,56,222,240
3704
.byte 102,15,56,222,248
3705
.byte 102,68,15,56,222,192
3706
.byte 102,68,15,56,222,200
3707
movups 192-112(%rcx),%xmm0
3708
je .Lcbc_dec_done
3709
.byte 102,15,56,222,209
3710
.byte 102,15,56,222,217
3711
.byte 102,15,56,222,225
3712
.byte 102,15,56,222,233
3713
.byte 102,15,56,222,241
3714
.byte 102,15,56,222,249
3715
.byte 102,68,15,56,222,193
3716
.byte 102,68,15,56,222,201
3717
movups 208-112(%rcx),%xmm1
3718
nop
3719
.byte 102,15,56,222,208
3720
.byte 102,15,56,222,216
3721
.byte 102,15,56,222,224
3722
.byte 102,15,56,222,232
3723
.byte 102,15,56,222,240
3724
.byte 102,15,56,222,248
3725
.byte 102,68,15,56,222,192
3726
.byte 102,68,15,56,222,200
3727
movups 224-112(%rcx),%xmm0
3728
jmp .Lcbc_dec_done
3729
.align 16
3730
.Lcbc_dec_done:
3731
.byte 102,15,56,222,209
3732
.byte 102,15,56,222,217
3733
pxor %xmm0,%xmm10
3734
pxor %xmm0,%xmm11
3735
.byte 102,15,56,222,225
3736
.byte 102,15,56,222,233
3737
pxor %xmm0,%xmm12
3738
pxor %xmm0,%xmm13
3739
.byte 102,15,56,222,241
3740
.byte 102,15,56,222,249
3741
pxor %xmm0,%xmm14
3742
pxor %xmm0,%xmm15
3743
.byte 102,68,15,56,222,193
3744
.byte 102,68,15,56,222,201
3745
movdqu 80(%rdi),%xmm1
3746
3747
.byte 102,65,15,56,223,210
3748
movdqu 96(%rdi),%xmm10
3749
pxor %xmm0,%xmm1
3750
.byte 102,65,15,56,223,219
3751
pxor %xmm0,%xmm10
3752
movdqu 112(%rdi),%xmm0
3753
.byte 102,65,15,56,223,228
3754
leaq 128(%rdi),%rdi
3755
movdqu 0(%rbp),%xmm11
3756
.byte 102,65,15,56,223,237
3757
.byte 102,65,15,56,223,246
3758
movdqu 16(%rbp),%xmm12
3759
movdqu 32(%rbp),%xmm13
3760
.byte 102,65,15,56,223,255
3761
.byte 102,68,15,56,223,193
3762
movdqu 48(%rbp),%xmm14
3763
movdqu 64(%rbp),%xmm15
3764
.byte 102,69,15,56,223,202
3765
movdqa %xmm0,%xmm10
3766
movdqu 80(%rbp),%xmm1
3767
movups -112(%rcx),%xmm0
3768
3769
movups %xmm2,(%rsi)
3770
movdqa %xmm11,%xmm2
3771
movups %xmm3,16(%rsi)
3772
movdqa %xmm12,%xmm3
3773
movups %xmm4,32(%rsi)
3774
movdqa %xmm13,%xmm4
3775
movups %xmm5,48(%rsi)
3776
movdqa %xmm14,%xmm5
3777
movups %xmm6,64(%rsi)
3778
movdqa %xmm15,%xmm6
3779
movups %xmm7,80(%rsi)
3780
movdqa %xmm1,%xmm7
3781
movups %xmm8,96(%rsi)
3782
leaq 112(%rsi),%rsi
3783
3784
subq $0x80,%rdx
3785
ja .Lcbc_dec_loop8
3786
3787
movaps %xmm9,%xmm2
3788
leaq -112(%rcx),%rcx
3789
addq $0x70,%rdx
3790
jle .Lcbc_dec_clear_tail_collected
3791
movups %xmm9,(%rsi)
3792
leaq 16(%rsi),%rsi
3793
cmpq $0x50,%rdx
3794
jbe .Lcbc_dec_tail
3795
3796
movaps %xmm11,%xmm2
3797
.Lcbc_dec_six_or_seven:
3798
cmpq $0x60,%rdx
3799
ja .Lcbc_dec_seven
3800
3801
movaps %xmm7,%xmm8
3802
call _aesni_decrypt6
3803
pxor %xmm10,%xmm2
3804
movaps %xmm8,%xmm10
3805
pxor %xmm11,%xmm3
3806
movdqu %xmm2,(%rsi)
3807
pxor %xmm12,%xmm4
3808
movdqu %xmm3,16(%rsi)
3809
pxor %xmm3,%xmm3
3810
pxor %xmm13,%xmm5
3811
movdqu %xmm4,32(%rsi)
3812
pxor %xmm4,%xmm4
3813
pxor %xmm14,%xmm6
3814
movdqu %xmm5,48(%rsi)
3815
pxor %xmm5,%xmm5
3816
pxor %xmm15,%xmm7
3817
movdqu %xmm6,64(%rsi)
3818
pxor %xmm6,%xmm6
3819
leaq 80(%rsi),%rsi
3820
movdqa %xmm7,%xmm2
3821
pxor %xmm7,%xmm7
3822
jmp .Lcbc_dec_tail_collected
3823
3824
.align 16
3825
.Lcbc_dec_seven:
3826
movups 96(%rdi),%xmm8
3827
xorps %xmm9,%xmm9
3828
call _aesni_decrypt8
3829
movups 80(%rdi),%xmm9
3830
pxor %xmm10,%xmm2
3831
movups 96(%rdi),%xmm10
3832
pxor %xmm11,%xmm3
3833
movdqu %xmm2,(%rsi)
3834
pxor %xmm12,%xmm4
3835
movdqu %xmm3,16(%rsi)
3836
pxor %xmm3,%xmm3
3837
pxor %xmm13,%xmm5
3838
movdqu %xmm4,32(%rsi)
3839
pxor %xmm4,%xmm4
3840
pxor %xmm14,%xmm6
3841
movdqu %xmm5,48(%rsi)
3842
pxor %xmm5,%xmm5
3843
pxor %xmm15,%xmm7
3844
movdqu %xmm6,64(%rsi)
3845
pxor %xmm6,%xmm6
3846
pxor %xmm9,%xmm8
3847
movdqu %xmm7,80(%rsi)
3848
pxor %xmm7,%xmm7
3849
leaq 96(%rsi),%rsi
3850
movdqa %xmm8,%xmm2
3851
pxor %xmm8,%xmm8
3852
pxor %xmm9,%xmm9
3853
jmp .Lcbc_dec_tail_collected
3854
3855
.align 16
3856
.Lcbc_dec_loop6:
3857
movups %xmm7,(%rsi)
3858
leaq 16(%rsi),%rsi
3859
movdqu 0(%rdi),%xmm2
3860
movdqu 16(%rdi),%xmm3
3861
movdqa %xmm2,%xmm11
3862
movdqu 32(%rdi),%xmm4
3863
movdqa %xmm3,%xmm12
3864
movdqu 48(%rdi),%xmm5
3865
movdqa %xmm4,%xmm13
3866
movdqu 64(%rdi),%xmm6
3867
movdqa %xmm5,%xmm14
3868
movdqu 80(%rdi),%xmm7
3869
movdqa %xmm6,%xmm15
3870
.Lcbc_dec_loop6_enter:
3871
leaq 96(%rdi),%rdi
3872
movdqa %xmm7,%xmm8
3873
3874
call _aesni_decrypt6
3875
3876
pxor %xmm10,%xmm2
3877
movdqa %xmm8,%xmm10
3878
pxor %xmm11,%xmm3
3879
movdqu %xmm2,(%rsi)
3880
pxor %xmm12,%xmm4
3881
movdqu %xmm3,16(%rsi)
3882
pxor %xmm13,%xmm5
3883
movdqu %xmm4,32(%rsi)
3884
pxor %xmm14,%xmm6
3885
movq %rbp,%rcx
3886
movdqu %xmm5,48(%rsi)
3887
pxor %xmm15,%xmm7
3888
movl %r10d,%eax
3889
movdqu %xmm6,64(%rsi)
3890
leaq 80(%rsi),%rsi
3891
subq $0x60,%rdx
3892
ja .Lcbc_dec_loop6
3893
3894
movdqa %xmm7,%xmm2
3895
addq $0x50,%rdx
3896
jle .Lcbc_dec_clear_tail_collected
3897
movups %xmm7,(%rsi)
3898
leaq 16(%rsi),%rsi
3899
3900
.Lcbc_dec_tail:
3901
movups (%rdi),%xmm2
3902
subq $0x10,%rdx
3903
jbe .Lcbc_dec_one
3904
3905
movups 16(%rdi),%xmm3
3906
movaps %xmm2,%xmm11
3907
subq $0x10,%rdx
3908
jbe .Lcbc_dec_two
3909
3910
movups 32(%rdi),%xmm4
3911
movaps %xmm3,%xmm12
3912
subq $0x10,%rdx
3913
jbe .Lcbc_dec_three
3914
3915
movups 48(%rdi),%xmm5
3916
movaps %xmm4,%xmm13
3917
subq $0x10,%rdx
3918
jbe .Lcbc_dec_four
3919
3920
movups 64(%rdi),%xmm6
3921
movaps %xmm5,%xmm14
3922
movaps %xmm6,%xmm15
3923
xorps %xmm7,%xmm7
3924
call _aesni_decrypt6
3925
pxor %xmm10,%xmm2
3926
movaps %xmm15,%xmm10
3927
pxor %xmm11,%xmm3
3928
movdqu %xmm2,(%rsi)
3929
pxor %xmm12,%xmm4
3930
movdqu %xmm3,16(%rsi)
3931
pxor %xmm3,%xmm3
3932
pxor %xmm13,%xmm5
3933
movdqu %xmm4,32(%rsi)
3934
pxor %xmm4,%xmm4
3935
pxor %xmm14,%xmm6
3936
movdqu %xmm5,48(%rsi)
3937
pxor %xmm5,%xmm5
3938
leaq 64(%rsi),%rsi
3939
movdqa %xmm6,%xmm2
3940
pxor %xmm6,%xmm6
3941
pxor %xmm7,%xmm7
3942
subq $0x10,%rdx
3943
jmp .Lcbc_dec_tail_collected
3944
3945
.align 16
3946
.Lcbc_dec_one:
3947
movaps %xmm2,%xmm11
3948
movups (%rcx),%xmm0
3949
movups 16(%rcx),%xmm1
3950
leaq 32(%rcx),%rcx
3951
xorps %xmm0,%xmm2
3952
.Loop_dec1_17:
3953
.byte 102,15,56,222,209
3954
decl %eax
3955
movups (%rcx),%xmm1
3956
leaq 16(%rcx),%rcx
3957
jnz .Loop_dec1_17
3958
.byte 102,15,56,223,209
3959
xorps %xmm10,%xmm2
3960
movaps %xmm11,%xmm10
3961
jmp .Lcbc_dec_tail_collected
3962
.align 16
3963
.Lcbc_dec_two:
3964
movaps %xmm3,%xmm12
3965
call _aesni_decrypt2
3966
pxor %xmm10,%xmm2
3967
movaps %xmm12,%xmm10
3968
pxor %xmm11,%xmm3
3969
movdqu %xmm2,(%rsi)
3970
movdqa %xmm3,%xmm2
3971
pxor %xmm3,%xmm3
3972
leaq 16(%rsi),%rsi
3973
jmp .Lcbc_dec_tail_collected
3974
.align 16
3975
.Lcbc_dec_three:
3976
movaps %xmm4,%xmm13
3977
call _aesni_decrypt3
3978
pxor %xmm10,%xmm2
3979
movaps %xmm13,%xmm10
3980
pxor %xmm11,%xmm3
3981
movdqu %xmm2,(%rsi)
3982
pxor %xmm12,%xmm4
3983
movdqu %xmm3,16(%rsi)
3984
pxor %xmm3,%xmm3
3985
movdqa %xmm4,%xmm2
3986
pxor %xmm4,%xmm4
3987
leaq 32(%rsi),%rsi
3988
jmp .Lcbc_dec_tail_collected
3989
.align 16
3990
.Lcbc_dec_four:
3991
movaps %xmm5,%xmm14
3992
call _aesni_decrypt4
3993
pxor %xmm10,%xmm2
3994
movaps %xmm14,%xmm10
3995
pxor %xmm11,%xmm3
3996
movdqu %xmm2,(%rsi)
3997
pxor %xmm12,%xmm4
3998
movdqu %xmm3,16(%rsi)
3999
pxor %xmm3,%xmm3
4000
pxor %xmm13,%xmm5
4001
movdqu %xmm4,32(%rsi)
4002
pxor %xmm4,%xmm4
4003
movdqa %xmm5,%xmm2
4004
pxor %xmm5,%xmm5
4005
leaq 48(%rsi),%rsi
4006
jmp .Lcbc_dec_tail_collected
4007
4008
.align 16
4009
.Lcbc_dec_clear_tail_collected:
4010
pxor %xmm3,%xmm3
4011
pxor %xmm4,%xmm4
4012
pxor %xmm5,%xmm5
4013
pxor %xmm6,%xmm6
4014
pxor %xmm7,%xmm7
4015
pxor %xmm8,%xmm8
4016
pxor %xmm9,%xmm9
4017
.Lcbc_dec_tail_collected:
4018
movups %xmm10,(%r8)
4019
andq $15,%rdx
4020
jnz .Lcbc_dec_tail_partial
4021
movups %xmm2,(%rsi)
4022
pxor %xmm2,%xmm2
4023
jmp .Lcbc_dec_ret
4024
.align 16
4025
.Lcbc_dec_tail_partial:
4026
movaps %xmm2,(%rsp)
4027
pxor %xmm2,%xmm2
4028
movq $16,%rcx
4029
movq %rsi,%rdi
4030
subq %rdx,%rcx
4031
leaq (%rsp),%rsi
4032
.long 0x9066A4F3
4033
movdqa %xmm2,(%rsp)
4034
4035
.Lcbc_dec_ret:
4036
xorps %xmm0,%xmm0
4037
pxor %xmm1,%xmm1
4038
movq -8(%r11),%rbp
4039
.cfi_restore %rbp
4040
leaq (%r11),%rsp
4041
.cfi_def_cfa_register %rsp
4042
.Lcbc_ret:
4043
.byte 0xf3,0xc3
4044
.cfi_endproc
4045
.size aesni_cbc_encrypt,.-aesni_cbc_encrypt
4046
.globl aesni_set_decrypt_key
4047
.type aesni_set_decrypt_key,@function
4048
.align 16
4049
aesni_set_decrypt_key:
4050
.cfi_startproc
4051
.byte 0x48,0x83,0xEC,0x08
4052
.cfi_adjust_cfa_offset 8
4053
call __aesni_set_encrypt_key
4054
shll $4,%esi
4055
testl %eax,%eax
4056
jnz .Ldec_key_ret
4057
leaq 16(%rdx,%rsi,1),%rdi
4058
4059
movups (%rdx),%xmm0
4060
movups (%rdi),%xmm1
4061
movups %xmm0,(%rdi)
4062
movups %xmm1,(%rdx)
4063
leaq 16(%rdx),%rdx
4064
leaq -16(%rdi),%rdi
4065
4066
.Ldec_key_inverse:
4067
movups (%rdx),%xmm0
4068
movups (%rdi),%xmm1
4069
.byte 102,15,56,219,192
4070
.byte 102,15,56,219,201
4071
leaq 16(%rdx),%rdx
4072
leaq -16(%rdi),%rdi
4073
movups %xmm0,16(%rdi)
4074
movups %xmm1,-16(%rdx)
4075
cmpq %rdx,%rdi
4076
ja .Ldec_key_inverse
4077
4078
movups (%rdx),%xmm0
4079
.byte 102,15,56,219,192
4080
pxor %xmm1,%xmm1
4081
movups %xmm0,(%rdi)
4082
pxor %xmm0,%xmm0
4083
.Ldec_key_ret:
4084
addq $8,%rsp
4085
.cfi_adjust_cfa_offset -8
4086
.byte 0xf3,0xc3
4087
.cfi_endproc
4088
.LSEH_end_set_decrypt_key:
4089
.size aesni_set_decrypt_key,.-aesni_set_decrypt_key
4090
.globl aesni_set_encrypt_key
4091
.type aesni_set_encrypt_key,@function
4092
.align 16
4093
aesni_set_encrypt_key:
4094
__aesni_set_encrypt_key:
4095
.cfi_startproc
4096
.byte 0x48,0x83,0xEC,0x08
4097
.cfi_adjust_cfa_offset 8
4098
movq $-1,%rax
4099
testq %rdi,%rdi
4100
jz .Lenc_key_ret
4101
testq %rdx,%rdx
4102
jz .Lenc_key_ret
4103
4104
movl $268437504,%r10d
4105
movups (%rdi),%xmm0
4106
xorps %xmm4,%xmm4
4107
andl OPENSSL_ia32cap_P+4(%rip),%r10d
4108
leaq 16(%rdx),%rax
4109
cmpl $256,%esi
4110
je .L14rounds
4111
cmpl $192,%esi
4112
je .L12rounds
4113
cmpl $128,%esi
4114
jne .Lbad_keybits
4115
4116
.L10rounds:
4117
movl $9,%esi
4118
cmpl $268435456,%r10d
4119
je .L10rounds_alt
4120
4121
movups %xmm0,(%rdx)
4122
.byte 102,15,58,223,200,1
4123
call .Lkey_expansion_128_cold
4124
.byte 102,15,58,223,200,2
4125
call .Lkey_expansion_128
4126
.byte 102,15,58,223,200,4
4127
call .Lkey_expansion_128
4128
.byte 102,15,58,223,200,8
4129
call .Lkey_expansion_128
4130
.byte 102,15,58,223,200,16
4131
call .Lkey_expansion_128
4132
.byte 102,15,58,223,200,32
4133
call .Lkey_expansion_128
4134
.byte 102,15,58,223,200,64
4135
call .Lkey_expansion_128
4136
.byte 102,15,58,223,200,128
4137
call .Lkey_expansion_128
4138
.byte 102,15,58,223,200,27
4139
call .Lkey_expansion_128
4140
.byte 102,15,58,223,200,54
4141
call .Lkey_expansion_128
4142
movups %xmm0,(%rax)
4143
movl %esi,80(%rax)
4144
xorl %eax,%eax
4145
jmp .Lenc_key_ret
4146
4147
.align 16
4148
.L10rounds_alt:
4149
movdqa .Lkey_rotate(%rip),%xmm5
4150
movl $8,%r10d
4151
movdqa .Lkey_rcon1(%rip),%xmm4
4152
movdqa %xmm0,%xmm2
4153
movdqu %xmm0,(%rdx)
4154
jmp .Loop_key128
4155
4156
.align 16
4157
.Loop_key128:
4158
.byte 102,15,56,0,197
4159
.byte 102,15,56,221,196
4160
pslld $1,%xmm4
4161
leaq 16(%rax),%rax
4162
4163
movdqa %xmm2,%xmm3
4164
pslldq $4,%xmm2
4165
pxor %xmm2,%xmm3
4166
pslldq $4,%xmm2
4167
pxor %xmm2,%xmm3
4168
pslldq $4,%xmm2
4169
pxor %xmm3,%xmm2
4170
4171
pxor %xmm2,%xmm0
4172
movdqu %xmm0,-16(%rax)
4173
movdqa %xmm0,%xmm2
4174
4175
decl %r10d
4176
jnz .Loop_key128
4177
4178
movdqa .Lkey_rcon1b(%rip),%xmm4
4179
4180
.byte 102,15,56,0,197
4181
.byte 102,15,56,221,196
4182
pslld $1,%xmm4
4183
4184
movdqa %xmm2,%xmm3
4185
pslldq $4,%xmm2
4186
pxor %xmm2,%xmm3
4187
pslldq $4,%xmm2
4188
pxor %xmm2,%xmm3
4189
pslldq $4,%xmm2
4190
pxor %xmm3,%xmm2
4191
4192
pxor %xmm2,%xmm0
4193
movdqu %xmm0,(%rax)
4194
4195
movdqa %xmm0,%xmm2
4196
.byte 102,15,56,0,197
4197
.byte 102,15,56,221,196
4198
4199
movdqa %xmm2,%xmm3
4200
pslldq $4,%xmm2
4201
pxor %xmm2,%xmm3
4202
pslldq $4,%xmm2
4203
pxor %xmm2,%xmm3
4204
pslldq $4,%xmm2
4205
pxor %xmm3,%xmm2
4206
4207
pxor %xmm2,%xmm0
4208
movdqu %xmm0,16(%rax)
4209
4210
movl %esi,96(%rax)
4211
xorl %eax,%eax
4212
jmp .Lenc_key_ret
4213
4214
.align 16
4215
.L12rounds:
4216
movq 16(%rdi),%xmm2
4217
movl $11,%esi
4218
cmpl $268435456,%r10d
4219
je .L12rounds_alt
4220
4221
movups %xmm0,(%rdx)
4222
.byte 102,15,58,223,202,1
4223
call .Lkey_expansion_192a_cold
4224
.byte 102,15,58,223,202,2
4225
call .Lkey_expansion_192b
4226
.byte 102,15,58,223,202,4
4227
call .Lkey_expansion_192a
4228
.byte 102,15,58,223,202,8
4229
call .Lkey_expansion_192b
4230
.byte 102,15,58,223,202,16
4231
call .Lkey_expansion_192a
4232
.byte 102,15,58,223,202,32
4233
call .Lkey_expansion_192b
4234
.byte 102,15,58,223,202,64
4235
call .Lkey_expansion_192a
4236
.byte 102,15,58,223,202,128
4237
call .Lkey_expansion_192b
4238
movups %xmm0,(%rax)
4239
movl %esi,48(%rax)
4240
xorq %rax,%rax
4241
jmp .Lenc_key_ret
4242
4243
.align 16
4244
.L12rounds_alt:
4245
movdqa .Lkey_rotate192(%rip),%xmm5
4246
movdqa .Lkey_rcon1(%rip),%xmm4
4247
movl $8,%r10d
4248
movdqu %xmm0,(%rdx)
4249
jmp .Loop_key192
4250
4251
.align 16
4252
.Loop_key192:
4253
movq %xmm2,0(%rax)
4254
movdqa %xmm2,%xmm1
4255
.byte 102,15,56,0,213
4256
.byte 102,15,56,221,212
4257
pslld $1,%xmm4
4258
leaq 24(%rax),%rax
4259
4260
movdqa %xmm0,%xmm3
4261
pslldq $4,%xmm0
4262
pxor %xmm0,%xmm3
4263
pslldq $4,%xmm0
4264
pxor %xmm0,%xmm3
4265
pslldq $4,%xmm0
4266
pxor %xmm3,%xmm0
4267
4268
pshufd $0xff,%xmm0,%xmm3
4269
pxor %xmm1,%xmm3
4270
pslldq $4,%xmm1
4271
pxor %xmm1,%xmm3
4272
4273
pxor %xmm2,%xmm0
4274
pxor %xmm3,%xmm2
4275
movdqu %xmm0,-16(%rax)
4276
4277
decl %r10d
4278
jnz .Loop_key192
4279
4280
movl %esi,32(%rax)
4281
xorl %eax,%eax
4282
jmp .Lenc_key_ret
4283
4284
.align 16
4285
.L14rounds:
4286
movups 16(%rdi),%xmm2
4287
movl $13,%esi
4288
leaq 16(%rax),%rax
4289
cmpl $268435456,%r10d
4290
je .L14rounds_alt
4291
4292
movups %xmm0,(%rdx)
4293
movups %xmm2,16(%rdx)
4294
.byte 102,15,58,223,202,1
4295
call .Lkey_expansion_256a_cold
4296
.byte 102,15,58,223,200,1
4297
call .Lkey_expansion_256b
4298
.byte 102,15,58,223,202,2
4299
call .Lkey_expansion_256a
4300
.byte 102,15,58,223,200,2
4301
call .Lkey_expansion_256b
4302
.byte 102,15,58,223,202,4
4303
call .Lkey_expansion_256a
4304
.byte 102,15,58,223,200,4
4305
call .Lkey_expansion_256b
4306
.byte 102,15,58,223,202,8
4307
call .Lkey_expansion_256a
4308
.byte 102,15,58,223,200,8
4309
call .Lkey_expansion_256b
4310
.byte 102,15,58,223,202,16
4311
call .Lkey_expansion_256a
4312
.byte 102,15,58,223,200,16
4313
call .Lkey_expansion_256b
4314
.byte 102,15,58,223,202,32
4315
call .Lkey_expansion_256a
4316
.byte 102,15,58,223,200,32
4317
call .Lkey_expansion_256b
4318
.byte 102,15,58,223,202,64
4319
call .Lkey_expansion_256a
4320
movups %xmm0,(%rax)
4321
movl %esi,16(%rax)
4322
xorq %rax,%rax
4323
jmp .Lenc_key_ret
4324
4325
.align 16
4326
.L14rounds_alt:
4327
movdqa .Lkey_rotate(%rip),%xmm5
4328
movdqa .Lkey_rcon1(%rip),%xmm4
4329
movl $7,%r10d
4330
movdqu %xmm0,0(%rdx)
4331
movdqa %xmm2,%xmm1
4332
movdqu %xmm2,16(%rdx)
4333
jmp .Loop_key256
4334
4335
.align 16
4336
.Loop_key256:
4337
.byte 102,15,56,0,213
4338
.byte 102,15,56,221,212
4339
4340
movdqa %xmm0,%xmm3
4341
pslldq $4,%xmm0
4342
pxor %xmm0,%xmm3
4343
pslldq $4,%xmm0
4344
pxor %xmm0,%xmm3
4345
pslldq $4,%xmm0
4346
pxor %xmm3,%xmm0
4347
pslld $1,%xmm4
4348
4349
pxor %xmm2,%xmm0
4350
movdqu %xmm0,(%rax)
4351
4352
decl %r10d
4353
jz .Ldone_key256
4354
4355
pshufd $0xff,%xmm0,%xmm2
4356
pxor %xmm3,%xmm3
4357
.byte 102,15,56,221,211
4358
4359
movdqa %xmm1,%xmm3
4360
pslldq $4,%xmm1
4361
pxor %xmm1,%xmm3
4362
pslldq $4,%xmm1
4363
pxor %xmm1,%xmm3
4364
pslldq $4,%xmm1
4365
pxor %xmm3,%xmm1
4366
4367
pxor %xmm1,%xmm2
4368
movdqu %xmm2,16(%rax)
4369
leaq 32(%rax),%rax
4370
movdqa %xmm2,%xmm1
4371
4372
jmp .Loop_key256
4373
4374
.Ldone_key256:
4375
movl %esi,16(%rax)
4376
xorl %eax,%eax
4377
jmp .Lenc_key_ret
4378
4379
.align 16
4380
.Lbad_keybits:
4381
movq $-2,%rax
4382
.Lenc_key_ret:
4383
pxor %xmm0,%xmm0
4384
pxor %xmm1,%xmm1
4385
pxor %xmm2,%xmm2
4386
pxor %xmm3,%xmm3
4387
pxor %xmm4,%xmm4
4388
pxor %xmm5,%xmm5
4389
addq $8,%rsp
4390
.cfi_adjust_cfa_offset -8
4391
.byte 0xf3,0xc3
4392
.LSEH_end_set_encrypt_key:
4393
4394
.align 16
4395
.Lkey_expansion_128:
4396
movups %xmm0,(%rax)
4397
leaq 16(%rax),%rax
4398
.Lkey_expansion_128_cold:
4399
shufps $16,%xmm0,%xmm4
4400
xorps %xmm4,%xmm0
4401
shufps $140,%xmm0,%xmm4
4402
xorps %xmm4,%xmm0
4403
shufps $255,%xmm1,%xmm1
4404
xorps %xmm1,%xmm0
4405
.byte 0xf3,0xc3
4406
4407
.align 16
4408
.Lkey_expansion_192a:
4409
movups %xmm0,(%rax)
4410
leaq 16(%rax),%rax
4411
.Lkey_expansion_192a_cold:
4412
movaps %xmm2,%xmm5
4413
.Lkey_expansion_192b_warm:
4414
shufps $16,%xmm0,%xmm4
4415
movdqa %xmm2,%xmm3
4416
xorps %xmm4,%xmm0
4417
shufps $140,%xmm0,%xmm4
4418
pslldq $4,%xmm3
4419
xorps %xmm4,%xmm0
4420
pshufd $85,%xmm1,%xmm1
4421
pxor %xmm3,%xmm2
4422
pxor %xmm1,%xmm0
4423
pshufd $255,%xmm0,%xmm3
4424
pxor %xmm3,%xmm2
4425
.byte 0xf3,0xc3
4426
4427
.align 16
4428
.Lkey_expansion_192b:
4429
movaps %xmm0,%xmm3
4430
shufps $68,%xmm0,%xmm5
4431
movups %xmm5,(%rax)
4432
shufps $78,%xmm2,%xmm3
4433
movups %xmm3,16(%rax)
4434
leaq 32(%rax),%rax
4435
jmp .Lkey_expansion_192b_warm
4436
4437
.align 16
4438
.Lkey_expansion_256a:
4439
movups %xmm2,(%rax)
4440
leaq 16(%rax),%rax
4441
.Lkey_expansion_256a_cold:
4442
shufps $16,%xmm0,%xmm4
4443
xorps %xmm4,%xmm0
4444
shufps $140,%xmm0,%xmm4
4445
xorps %xmm4,%xmm0
4446
shufps $255,%xmm1,%xmm1
4447
xorps %xmm1,%xmm0
4448
.byte 0xf3,0xc3
4449
4450
.align 16
4451
.Lkey_expansion_256b:
4452
movups %xmm0,(%rax)
4453
leaq 16(%rax),%rax
4454
4455
shufps $16,%xmm2,%xmm4
4456
xorps %xmm4,%xmm2
4457
shufps $140,%xmm2,%xmm4
4458
xorps %xmm4,%xmm2
4459
shufps $170,%xmm1,%xmm1
4460
xorps %xmm1,%xmm2
4461
.byte 0xf3,0xc3
4462
.cfi_endproc
4463
.size aesni_set_encrypt_key,.-aesni_set_encrypt_key
4464
.size __aesni_set_encrypt_key,.-__aesni_set_encrypt_key
4465
.section .rodata
4466
.align 64
4467
.Lbswap_mask:
4468
.byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
4469
.Lincrement32:
4470
.long 6,6,6,0
4471
.Lincrement64:
4472
.long 1,0,0,0
4473
.Lxts_magic:
4474
.long 0x87,0,1,0
4475
.Lincrement1:
4476
.byte 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
4477
.Lkey_rotate:
4478
.long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d
4479
.Lkey_rotate192:
4480
.long 0x04070605,0x04070605,0x04070605,0x04070605
4481
.Lkey_rcon1:
4482
.long 1,1,1,1
4483
.Lkey_rcon1b:
4484
.long 0x1b,0x1b,0x1b,0x1b
4485
4486
.byte 65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69,83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
4487
.align 64
4488
.previous
4489
.section ".note.gnu.property", "a"
4490
.p2align 3
4491
.long 1f - 0f
4492
.long 4f - 1f
4493
.long 5
4494
0:
4495
# "GNU" encoded with .byte, since .asciz isn't supported
4496
# on Solaris.
4497
.byte 0x47
4498
.byte 0x4e
4499
.byte 0x55
4500
.byte 0
4501
1:
4502
.p2align 3
4503
.long 0xc0000002
4504
.long 3f - 2f
4505
2:
4506
.long 3
4507
3:
4508
.p2align 3
4509
4:
4510
4511