1/*
2 * Copyright (C) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "testb3.h"
28
29#include <wtf/UniqueArray.h>
30
31#if ENABLE(B3_JIT)
32
33template<typename T>
34void testAtomicWeakCAS()
35{
36 constexpr Type type = NativeTraits<T>::type;
37 constexpr Width width = NativeTraits<T>::width;
38
39 auto checkMyDisassembly = [&] (Compilation& compilation, bool fenced) {
40 if (isX86()) {
41 checkUsesInstruction(compilation, "lock");
42 checkUsesInstruction(compilation, "cmpxchg");
43 } else {
44 if (fenced) {
45 checkUsesInstruction(compilation, "ldax");
46 checkUsesInstruction(compilation, "stlx");
47 } else {
48 checkUsesInstruction(compilation, "ldx");
49 checkUsesInstruction(compilation, "stx");
50 }
51 }
52 };
53
54 {
55 Procedure proc;
56 BasicBlock* root = proc.addBlock();
57 BasicBlock* reloop = proc.addBlock();
58 BasicBlock* done = proc.addBlock();
59
60 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
61 root->appendNew<Value>(proc, Jump, Origin());
62 root->setSuccessors(reloop);
63
64 reloop->appendNew<Value>(
65 proc, Branch, Origin(),
66 reloop->appendNew<AtomicValue>(
67 proc, AtomicWeakCAS, Origin(), width,
68 reloop->appendIntConstant(proc, Origin(), type, 42),
69 reloop->appendIntConstant(proc, Origin(), type, 0xbeef),
70 ptr));
71 reloop->setSuccessors(done, reloop);
72
73 done->appendNew<Value>(proc, Return, Origin());
74
75 auto code = compileProc(proc);
76 T value[2];
77 value[0] = 42;
78 value[1] = 13;
79 invoke<void>(*code, value);
80 CHECK_EQ(value[0], static_cast<T>(0xbeef));
81 CHECK_EQ(value[1], 13);
82 checkMyDisassembly(*code, true);
83 }
84
85 {
86 Procedure proc;
87 BasicBlock* root = proc.addBlock();
88 BasicBlock* reloop = proc.addBlock();
89 BasicBlock* done = proc.addBlock();
90
91 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
92 root->appendNew<Value>(proc, Jump, Origin());
93 root->setSuccessors(reloop);
94
95 reloop->appendNew<Value>(
96 proc, Branch, Origin(),
97 reloop->appendNew<AtomicValue>(
98 proc, AtomicWeakCAS, Origin(), width,
99 reloop->appendIntConstant(proc, Origin(), type, 42),
100 reloop->appendIntConstant(proc, Origin(), type, 0xbeef),
101 ptr, 0, HeapRange(42), HeapRange()));
102 reloop->setSuccessors(done, reloop);
103
104 done->appendNew<Value>(proc, Return, Origin());
105
106 auto code = compileProc(proc);
107 T value[2];
108 value[0] = 42;
109 value[1] = 13;
110 invoke<void>(*code, value);
111 CHECK_EQ(value[0], static_cast<T>(0xbeef));
112 CHECK_EQ(value[1], 13);
113 checkMyDisassembly(*code, false);
114 }
115
116 {
117 Procedure proc;
118 BasicBlock* root = proc.addBlock();
119 BasicBlock* succ = proc.addBlock();
120 BasicBlock* fail = proc.addBlock();
121
122 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
123 root->appendNew<Value>(
124 proc, Branch, Origin(),
125 root->appendNew<AtomicValue>(
126 proc, AtomicWeakCAS, Origin(), width,
127 root->appendIntConstant(proc, Origin(), type, 42),
128 root->appendIntConstant(proc, Origin(), type, 0xbeef),
129 ptr));
130 root->setSuccessors(succ, fail);
131
132 succ->appendNew<MemoryValue>(
133 proc, storeOpcode(GP, width), Origin(),
134 succ->appendIntConstant(proc, Origin(), type, 100),
135 ptr);
136 succ->appendNew<Value>(proc, Return, Origin());
137
138 fail->appendNew<Value>(proc, Return, Origin());
139
140 auto code = compileProc(proc);
141 T value[2];
142 value[0] = 42;
143 value[1] = 13;
144 while (value[0] == 42)
145 invoke<void>(*code, value);
146 CHECK_EQ(value[0], static_cast<T>(100));
147 CHECK_EQ(value[1], 13);
148 value[0] = static_cast<T>(300);
149 invoke<void>(*code, value);
150 CHECK_EQ(value[0], static_cast<T>(300));
151 CHECK_EQ(value[1], 13);
152 checkMyDisassembly(*code, true);
153 }
154
155 {
156 Procedure proc;
157 BasicBlock* root = proc.addBlock();
158 BasicBlock* succ = proc.addBlock();
159 BasicBlock* fail = proc.addBlock();
160
161 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
162 root->appendNew<Value>(
163 proc, Branch, Origin(),
164 root->appendNew<Value>(
165 proc, Equal, Origin(),
166 root->appendNew<AtomicValue>(
167 proc, AtomicWeakCAS, Origin(), width,
168 root->appendIntConstant(proc, Origin(), type, 42),
169 root->appendIntConstant(proc, Origin(), type, 0xbeef),
170 ptr),
171 root->appendIntConstant(proc, Origin(), Int32, 0)));
172 root->setSuccessors(fail, succ);
173
174 succ->appendNew<MemoryValue>(
175 proc, storeOpcode(GP, width), Origin(),
176 succ->appendIntConstant(proc, Origin(), type, 100),
177 ptr);
178 succ->appendNew<Value>(proc, Return, Origin());
179
180 fail->appendNew<Value>(proc, Return, Origin());
181
182 auto code = compileProc(proc);
183 T value[2];
184 value[0] = 42;
185 value[1] = 13;
186 while (value[0] == 42)
187 invoke<void>(*code, value);
188 CHECK_EQ(value[0], static_cast<T>(100));
189 CHECK_EQ(value[1], 13);
190 value[0] = static_cast<T>(300);
191 invoke<void>(*code, value);
192 CHECK_EQ(value[0], static_cast<T>(300));
193 CHECK_EQ(value[1], 13);
194 checkMyDisassembly(*code, true);
195 }
196
197 {
198 Procedure proc;
199 BasicBlock* root = proc.addBlock();
200 root->appendNew<Value>(
201 proc, Return, Origin(),
202 root->appendNew<AtomicValue>(
203 proc, AtomicWeakCAS, Origin(), width,
204 root->appendIntConstant(proc, Origin(), type, 42),
205 root->appendIntConstant(proc, Origin(), type, 0xbeef),
206 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
207
208 auto code = compileProc(proc);
209 T value[2];
210 value[0] = 42;
211 value[1] = 13;
212 while (!invoke<bool>(*code, value)) { }
213 CHECK_EQ(value[0], static_cast<T>(0xbeef));
214 CHECK_EQ(value[1], 13);
215
216 value[0] = static_cast<T>(300);
217 CHECK(!invoke<bool>(*code, value));
218 CHECK_EQ(value[0], static_cast<T>(300));
219 CHECK_EQ(value[1], 13);
220 checkMyDisassembly(*code, true);
221 }
222
223 {
224 Procedure proc;
225 BasicBlock* root = proc.addBlock();
226 root->appendNew<Value>(
227 proc, Return, Origin(),
228 root->appendNew<Value>(
229 proc, Equal, Origin(),
230 root->appendNew<AtomicValue>(
231 proc, AtomicWeakCAS, Origin(), width,
232 root->appendIntConstant(proc, Origin(), type, 42),
233 root->appendIntConstant(proc, Origin(), type, 0xbeef),
234 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
235 root->appendNew<Const32Value>(proc, Origin(), 0)));
236
237 auto code = compileProc(proc);
238 T value[2];
239 value[0] = 42;
240 value[1] = 13;
241 while (invoke<bool>(*code, value)) { }
242 CHECK_EQ(value[0], static_cast<T>(0xbeef));
243 CHECK_EQ(value[1], 13);
244
245 value[0] = static_cast<T>(300);
246 CHECK(invoke<bool>(*code, value));
247 CHECK_EQ(value[0], static_cast<T>(300));
248 CHECK_EQ(value[1], 13);
249 checkMyDisassembly(*code, true);
250 }
251
252 {
253 Procedure proc;
254 BasicBlock* root = proc.addBlock();
255 root->appendNew<Value>(
256 proc, Return, Origin(),
257 root->appendNew<AtomicValue>(
258 proc, AtomicWeakCAS, Origin(), width,
259 root->appendIntConstant(proc, Origin(), type, 42),
260 root->appendIntConstant(proc, Origin(), type, 0xbeef),
261 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
262 42));
263
264 auto code = compileProc(proc);
265 T value[2];
266 value[0] = 42;
267 value[1] = 13;
268 while (!invoke<bool>(*code, bitwise_cast<intptr_t>(value) - 42)) { }
269 CHECK_EQ(value[0], static_cast<T>(0xbeef));
270 CHECK_EQ(value[1], 13);
271
272 value[0] = static_cast<T>(300);
273 CHECK(!invoke<bool>(*code, bitwise_cast<intptr_t>(value) - 42));
274 CHECK_EQ(value[0], static_cast<T>(300));
275 CHECK_EQ(value[1], 13);
276 checkMyDisassembly(*code, true);
277 }
278}
279
280template<typename T>
281void testAtomicStrongCAS()
282{
283 constexpr Type type = NativeTraits<T>::type;
284 constexpr Width width = NativeTraits<T>::width;
285
286 auto checkMyDisassembly = [&] (Compilation& compilation, bool fenced) {
287 if (isX86()) {
288 checkUsesInstruction(compilation, "lock");
289 checkUsesInstruction(compilation, "cmpxchg");
290 } else {
291 if (fenced) {
292 checkUsesInstruction(compilation, "ldax");
293 checkUsesInstruction(compilation, "stlx");
294 } else {
295 checkUsesInstruction(compilation, "ldx");
296 checkUsesInstruction(compilation, "stx");
297 }
298 }
299 };
300
301 {
302 Procedure proc;
303 BasicBlock* root = proc.addBlock();
304 BasicBlock* succ = proc.addBlock();
305 BasicBlock* fail = proc.addBlock();
306
307 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
308 root->appendNew<Value>(
309 proc, Branch, Origin(),
310 root->appendNew<Value>(
311 proc, Equal, Origin(),
312 root->appendNew<AtomicValue>(
313 proc, AtomicStrongCAS, Origin(), width,
314 root->appendIntConstant(proc, Origin(), type, 42),
315 root->appendIntConstant(proc, Origin(), type, 0xbeef),
316 ptr),
317 root->appendIntConstant(proc, Origin(), type, 42)));
318 root->setSuccessors(succ, fail);
319
320 succ->appendNew<MemoryValue>(
321 proc, storeOpcode(GP, width), Origin(),
322 succ->appendIntConstant(proc, Origin(), type, 100),
323 ptr);
324 succ->appendNew<Value>(proc, Return, Origin());
325
326 fail->appendNew<Value>(proc, Return, Origin());
327
328 auto code = compileProc(proc);
329 T value[2];
330 value[0] = 42;
331 value[1] = 13;
332 invoke<void>(*code, value);
333 CHECK_EQ(value[0], static_cast<T>(100));
334 CHECK_EQ(value[1], 13);
335 value[0] = static_cast<T>(300);
336 invoke<void>(*code, value);
337 CHECK_EQ(value[0], static_cast<T>(300));
338 CHECK_EQ(value[1], 13);
339 checkMyDisassembly(*code, true);
340 }
341
342 {
343 Procedure proc;
344 BasicBlock* root = proc.addBlock();
345 BasicBlock* succ = proc.addBlock();
346 BasicBlock* fail = proc.addBlock();
347
348 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
349 root->appendNew<Value>(
350 proc, Branch, Origin(),
351 root->appendNew<Value>(
352 proc, Equal, Origin(),
353 root->appendNew<AtomicValue>(
354 proc, AtomicStrongCAS, Origin(), width,
355 root->appendIntConstant(proc, Origin(), type, 42),
356 root->appendIntConstant(proc, Origin(), type, 0xbeef),
357 ptr, 0, HeapRange(42), HeapRange()),
358 root->appendIntConstant(proc, Origin(), type, 42)));
359 root->setSuccessors(succ, fail);
360
361 succ->appendNew<MemoryValue>(
362 proc, storeOpcode(GP, width), Origin(),
363 succ->appendIntConstant(proc, Origin(), type, 100),
364 ptr);
365 succ->appendNew<Value>(proc, Return, Origin());
366
367 fail->appendNew<Value>(proc, Return, Origin());
368
369 auto code = compileProc(proc);
370 T value[2];
371 value[0] = 42;
372 value[1] = 13;
373 invoke<void>(*code, value);
374 CHECK_EQ(value[0], static_cast<T>(100));
375 CHECK_EQ(value[1], 13);
376 value[0] = static_cast<T>(300);
377 invoke<void>(*code, value);
378 CHECK_EQ(value[0], static_cast<T>(300));
379 CHECK_EQ(value[1], 13);
380 checkMyDisassembly(*code, false);
381 }
382
383 {
384 Procedure proc;
385 BasicBlock* root = proc.addBlock();
386 BasicBlock* succ = proc.addBlock();
387 BasicBlock* fail = proc.addBlock();
388
389 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
390 root->appendNew<Value>(
391 proc, Branch, Origin(),
392 root->appendNew<Value>(
393 proc, NotEqual, Origin(),
394 root->appendNew<AtomicValue>(
395 proc, AtomicStrongCAS, Origin(), width,
396 root->appendIntConstant(proc, Origin(), type, 42),
397 root->appendIntConstant(proc, Origin(), type, 0xbeef),
398 ptr),
399 root->appendIntConstant(proc, Origin(), type, 42)));
400 root->setSuccessors(fail, succ);
401
402 succ->appendNew<MemoryValue>(
403 proc, storeOpcode(GP, width), Origin(),
404 succ->appendIntConstant(proc, Origin(), type, 100),
405 ptr);
406 succ->appendNew<Value>(proc, Return, Origin());
407
408 fail->appendNew<Value>(proc, Return, Origin());
409
410 auto code = compileProc(proc);
411 T value[2];
412 value[0] = 42;
413 value[1] = 13;
414 invoke<void>(*code, value);
415 CHECK_EQ(value[0], static_cast<T>(100));
416 CHECK_EQ(value[1], 13);
417 value[0] = static_cast<T>(300);
418 invoke<void>(*code, value);
419 CHECK_EQ(value[0], static_cast<T>(300));
420 CHECK_EQ(value[1], 13);
421 checkMyDisassembly(*code, true);
422 }
423
424 {
425 Procedure proc;
426 BasicBlock* root = proc.addBlock();
427 root->appendNew<Value>(
428 proc, Return, Origin(),
429 root->appendNew<AtomicValue>(
430 proc, AtomicStrongCAS, Origin(), width,
431 root->appendIntConstant(proc, Origin(), type, 42),
432 root->appendIntConstant(proc, Origin(), type, 0xbeef),
433 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
434
435 auto code = compileProc(proc);
436 T value[2];
437 value[0] = 42;
438 value[1] = 13;
439 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), 42);
440 CHECK_EQ(value[0], static_cast<T>(0xbeef));
441 CHECK_EQ(value[1], 13);
442 value[0] = static_cast<T>(300);
443 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), static_cast<typename NativeTraits<T>::CanonicalType>(static_cast<T>(300)));
444 CHECK_EQ(value[0], static_cast<T>(300));
445 CHECK_EQ(value[1], 13);
446 value[0] = static_cast<T>(-1);
447 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), static_cast<typename NativeTraits<T>::CanonicalType>(static_cast<T>(-1)));
448 CHECK_EQ(value[0], static_cast<T>(-1));
449 CHECK_EQ(value[1], 13);
450 checkMyDisassembly(*code, true);
451 }
452
453 {
454 // Test for https://bugs.webkit.org/show_bug.cgi?id=169867.
455
456 Procedure proc;
457 BasicBlock* root = proc.addBlock();
458 root->appendNew<Value>(
459 proc, Return, Origin(),
460 root->appendNew<Value>(
461 proc, BitXor, Origin(),
462 root->appendNew<AtomicValue>(
463 proc, AtomicStrongCAS, Origin(), width,
464 root->appendIntConstant(proc, Origin(), type, 42),
465 root->appendIntConstant(proc, Origin(), type, 0xbeef),
466 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
467 root->appendIntConstant(proc, Origin(), type, 1)));
468
469 typename NativeTraits<T>::CanonicalType one = 1;
470
471 auto code = compileProc(proc);
472 T value[2];
473 value[0] = 42;
474 value[1] = 13;
475 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), 42 ^ one);
476 CHECK_EQ(value[0], static_cast<T>(0xbeef));
477 CHECK_EQ(value[1], 13);
478 value[0] = static_cast<T>(300);
479 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), static_cast<typename NativeTraits<T>::CanonicalType>(static_cast<T>(300)) ^ one);
480 CHECK_EQ(value[0], static_cast<T>(300));
481 CHECK_EQ(value[1], 13);
482 value[0] = static_cast<T>(-1);
483 CHECK_EQ(invoke<typename NativeTraits<T>::CanonicalType>(*code, value), static_cast<typename NativeTraits<T>::CanonicalType>(static_cast<T>(-1)) ^ one);
484 CHECK_EQ(value[0], static_cast<T>(-1));
485 CHECK_EQ(value[1], 13);
486 checkMyDisassembly(*code, true);
487 }
488
489 {
490 Procedure proc;
491 BasicBlock* root = proc.addBlock();
492 root->appendNew<Value>(
493 proc, Return, Origin(),
494 root->appendNew<Value>(
495 proc, Equal, Origin(),
496 root->appendNew<AtomicValue>(
497 proc, AtomicStrongCAS, Origin(), width,
498 root->appendIntConstant(proc, Origin(), type, 42),
499 root->appendIntConstant(proc, Origin(), type, 0xbeef),
500 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
501 root->appendIntConstant(proc, Origin(), type, 42)));
502
503 auto code = compileProc(proc);
504 T value[2];
505 value[0] = 42;
506 value[1] = 13;
507 CHECK(invoke<bool>(*code, value));
508 CHECK_EQ(value[0], static_cast<T>(0xbeef));
509 CHECK_EQ(value[1], 13);
510 value[0] = static_cast<T>(300);
511 CHECK(!invoke<bool>(*code, value));
512 CHECK_EQ(value[0], static_cast<T>(300));
513 CHECK_EQ(value[1], 13);
514 checkMyDisassembly(*code, true);
515 }
516
517 {
518 Procedure proc;
519 BasicBlock* root = proc.addBlock();
520 root->appendNew<Value>(
521 proc, Return, Origin(),
522 root->appendNew<Value>(
523 proc, Equal, Origin(),
524 root->appendNew<Value>(
525 proc, NotEqual, Origin(),
526 root->appendNew<AtomicValue>(
527 proc, AtomicStrongCAS, Origin(), width,
528 root->appendIntConstant(proc, Origin(), type, 42),
529 root->appendIntConstant(proc, Origin(), type, 0xbeef),
530 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
531 root->appendIntConstant(proc, Origin(), type, 42)),
532 root->appendNew<Const32Value>(proc, Origin(), 0)));
533
534 auto code = compileProc(proc);
535 T value[2];
536 value[0] = 42;
537 value[1] = 13;
538 CHECK(invoke<bool>(*code, value));
539 CHECK_EQ(value[0], static_cast<T>(0xbeef));
540 CHECK_EQ(value[1], 13);
541 value[0] = static_cast<T>(300);
542 CHECK(!invoke<bool>(*code, &value));
543 CHECK_EQ(value[0], static_cast<T>(300));
544 CHECK_EQ(value[1], 13);
545 checkMyDisassembly(*code, true);
546 }
547}
548
549template<typename T>
550void testAtomicXchg(B3::Opcode opcode)
551{
552 constexpr Type type = NativeTraits<T>::type;
553 constexpr Width width = NativeTraits<T>::width;
554
555 auto doTheMath = [&] (T& memory, T operand) -> T {
556 T oldValue = memory;
557 switch (opcode) {
558 case AtomicXchgAdd:
559 memory += operand;
560 break;
561 case AtomicXchgAnd:
562 memory &= operand;
563 break;
564 case AtomicXchgOr:
565 memory |= operand;
566 break;
567 case AtomicXchgSub:
568 memory -= operand;
569 break;
570 case AtomicXchgXor:
571 memory ^= operand;
572 break;
573 case AtomicXchg:
574 memory = operand;
575 break;
576 default:
577 RELEASE_ASSERT_NOT_REACHED();
578 }
579 return oldValue;
580 };
581
582 auto oldValue = [&] (T memory, T operand) -> T {
583 return doTheMath(memory, operand);
584 };
585
586 auto newValue = [&] (T memory, T operand) -> T {
587 doTheMath(memory, operand);
588 return memory;
589 };
590
591 auto checkMyDisassembly = [&] (Compilation& compilation, bool fenced) {
592 if (isX86())
593 checkUsesInstruction(compilation, "lock");
594 else {
595 if (fenced) {
596 checkUsesInstruction(compilation, "ldax");
597 checkUsesInstruction(compilation, "stlx");
598 } else {
599 checkUsesInstruction(compilation, "ldx");
600 checkUsesInstruction(compilation, "stx");
601 }
602 }
603 };
604
605 {
606 Procedure proc;
607 BasicBlock* root = proc.addBlock();
608 root->appendNew<Value>(
609 proc, Return, Origin(),
610 root->appendNew<AtomicValue>(
611 proc, opcode, Origin(), width,
612 root->appendIntConstant(proc, Origin(), type, 1),
613 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
614
615 auto code = compileProc(proc);
616 T value[2];
617 value[0] = 5;
618 value[1] = 100;
619 CHECK_EQ(invoke<T>(*code, value), oldValue(5, 1));
620 CHECK_EQ(value[0], newValue(5, 1));
621 CHECK_EQ(value[1], 100);
622 checkMyDisassembly(*code, true);
623 }
624
625 {
626 Procedure proc;
627 BasicBlock* root = proc.addBlock();
628 root->appendNew<Value>(
629 proc, Return, Origin(),
630 root->appendNew<AtomicValue>(
631 proc, opcode, Origin(), width,
632 root->appendIntConstant(proc, Origin(), type, 42),
633 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
634
635 auto code = compileProc(proc);
636 T value[2];
637 value[0] = 5;
638 value[1] = 100;
639 CHECK_EQ(invoke<T>(*code, value), oldValue(5, 42));
640 CHECK_EQ(value[0], newValue(5, 42));
641 CHECK_EQ(value[1], 100);
642 checkMyDisassembly(*code, true);
643 }
644
645 {
646 Procedure proc;
647 BasicBlock* root = proc.addBlock();
648 root->appendNew<AtomicValue>(
649 proc, opcode, Origin(), width,
650 root->appendIntConstant(proc, Origin(), type, 42),
651 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
652 root->appendNew<Value>(proc, Return, Origin());
653
654 auto code = compileProc(proc);
655 T value[2];
656 value[0] = 5;
657 value[1] = 100;
658 invoke<T>(*code, value);
659 CHECK_EQ(value[0], newValue(5, 42));
660 CHECK_EQ(value[1], 100);
661 checkMyDisassembly(*code, true);
662 }
663
664 {
665 Procedure proc;
666 BasicBlock* root = proc.addBlock();
667 root->appendNew<AtomicValue>(
668 proc, opcode, Origin(), width,
669 root->appendIntConstant(proc, Origin(), type, 42),
670 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
671 0, HeapRange(42), HeapRange());
672 root->appendNew<Value>(proc, Return, Origin());
673
674 auto code = compileProc(proc);
675 T value[2];
676 value[0] = 5;
677 value[1] = 100;
678 invoke<T>(*code, value);
679 CHECK_EQ(value[0], newValue(5, 42));
680 CHECK_EQ(value[1], 100);
681 checkMyDisassembly(*code, false);
682 }
683}
684
685void addAtomicTests(const char* filter, Deque<RefPtr<SharedTask<void()>>>& tasks)
686{
687 RUN(testAtomicWeakCAS<int8_t>());
688 RUN(testAtomicWeakCAS<int16_t>());
689 RUN(testAtomicWeakCAS<int32_t>());
690 RUN(testAtomicWeakCAS<int64_t>());
691 RUN(testAtomicStrongCAS<int8_t>());
692 RUN(testAtomicStrongCAS<int16_t>());
693 RUN(testAtomicStrongCAS<int32_t>());
694 RUN(testAtomicStrongCAS<int64_t>());
695 RUN(testAtomicXchg<int8_t>(AtomicXchgAdd));
696 RUN(testAtomicXchg<int16_t>(AtomicXchgAdd));
697 RUN(testAtomicXchg<int32_t>(AtomicXchgAdd));
698 RUN(testAtomicXchg<int64_t>(AtomicXchgAdd));
699 RUN(testAtomicXchg<int8_t>(AtomicXchgAnd));
700 RUN(testAtomicXchg<int16_t>(AtomicXchgAnd));
701 RUN(testAtomicXchg<int32_t>(AtomicXchgAnd));
702 RUN(testAtomicXchg<int64_t>(AtomicXchgAnd));
703 RUN(testAtomicXchg<int8_t>(AtomicXchgOr));
704 RUN(testAtomicXchg<int16_t>(AtomicXchgOr));
705 RUN(testAtomicXchg<int32_t>(AtomicXchgOr));
706 RUN(testAtomicXchg<int64_t>(AtomicXchgOr));
707 RUN(testAtomicXchg<int8_t>(AtomicXchgSub));
708 RUN(testAtomicXchg<int16_t>(AtomicXchgSub));
709 RUN(testAtomicXchg<int32_t>(AtomicXchgSub));
710 RUN(testAtomicXchg<int64_t>(AtomicXchgSub));
711 RUN(testAtomicXchg<int8_t>(AtomicXchgXor));
712 RUN(testAtomicXchg<int16_t>(AtomicXchgXor));
713 RUN(testAtomicXchg<int32_t>(AtomicXchgXor));
714 RUN(testAtomicXchg<int64_t>(AtomicXchgXor));
715 RUN(testAtomicXchg<int8_t>(AtomicXchg));
716 RUN(testAtomicXchg<int16_t>(AtomicXchg));
717 RUN(testAtomicXchg<int32_t>(AtomicXchg));
718 RUN(testAtomicXchg<int64_t>(AtomicXchg));
719}
720
721template<typename CType, typename InputType>
722void testLoad(B3::Type type, B3::Opcode opcode, InputType value)
723{
724 // Simple load from an absolute address.
725 {
726 Procedure proc;
727 BasicBlock* root = proc.addBlock();
728
729 root->appendNewControlValue(
730 proc, Return, Origin(),
731 root->appendNew<MemoryValue>(
732 proc, opcode, type, Origin(),
733 root->appendNew<ConstPtrValue>(proc, Origin(), &value)));
734
735 CHECK(isIdentical(compileAndRun<CType>(proc), modelLoad<CType>(value)));
736 }
737
738 // Simple load from an address in a register.
739 {
740 Procedure proc;
741 BasicBlock* root = proc.addBlock();
742
743 root->appendNewControlValue(
744 proc, Return, Origin(),
745 root->appendNew<MemoryValue>(
746 proc, opcode, type, Origin(),
747 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
748
749 CHECK(isIdentical(compileAndRun<CType>(proc, &value), modelLoad<CType>(value)));
750 }
751
752 // Simple load from an address in a register, at an offset.
753 {
754 Procedure proc;
755 BasicBlock* root = proc.addBlock();
756
757 root->appendNewControlValue(
758 proc, Return, Origin(),
759 root->appendNew<MemoryValue>(
760 proc, opcode, type, Origin(),
761 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
762 static_cast<int32_t>(sizeof(InputType))));
763
764 CHECK(isIdentical(compileAndRun<CType>(proc, &value - 1), modelLoad<CType>(value)));
765 }
766
767 // Load from a simple base-index with various scales.
768 for (unsigned logScale = 0; logScale <= 3; ++logScale) {
769 Procedure proc;
770 BasicBlock* root = proc.addBlock();
771
772 root->appendNewControlValue(
773 proc, Return, Origin(),
774 root->appendNew<MemoryValue>(
775 proc, opcode, type, Origin(),
776 root->appendNew<Value>(
777 proc, Add, Origin(),
778 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
779 root->appendNew<Value>(
780 proc, Shl, Origin(),
781 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
782 root->appendNew<Const32Value>(proc, Origin(), logScale)))));
783
784 CHECK(isIdentical(compileAndRun<CType>(proc, &value - 2, (sizeof(InputType) * 2) >> logScale), modelLoad<CType>(value)));
785 }
786
787 // Load from a simple base-index with various scales, but commuted.
788 for (unsigned logScale = 0; logScale <= 3; ++logScale) {
789 Procedure proc;
790 BasicBlock* root = proc.addBlock();
791
792 root->appendNewControlValue(
793 proc, Return, Origin(),
794 root->appendNew<MemoryValue>(
795 proc, opcode, type, Origin(),
796 root->appendNew<Value>(
797 proc, Add, Origin(),
798 root->appendNew<Value>(
799 proc, Shl, Origin(),
800 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
801 root->appendNew<Const32Value>(proc, Origin(), logScale)),
802 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0))));
803
804 CHECK(isIdentical(compileAndRun<CType>(proc, &value - 2, (sizeof(InputType) * 2) >> logScale), modelLoad<CType>(value)));
805 }
806}
807
808template<typename T>
809void testLoad(B3::Opcode opcode, int32_t value)
810{
811 return testLoad<T>(B3::Int32, opcode, value);
812}
813
814template<typename T>
815void testLoad(B3::Type type, T value)
816{
817 return testLoad<T>(type, Load, value);
818}
819
820void addLoadTests(const char* filter, Deque<RefPtr<SharedTask<void()>>>& tasks)
821{
822 RUN(testLoad(Int32, 60));
823 RUN(testLoad(Int32, -60));
824 RUN(testLoad(Int32, 1000));
825 RUN(testLoad(Int32, -1000));
826 RUN(testLoad(Int32, 1000000));
827 RUN(testLoad(Int32, -1000000));
828 RUN(testLoad(Int32, 1000000000));
829 RUN(testLoad(Int32, -1000000000));
830 RUN_BINARY(testLoad, { MAKE_OPERAND(Int64) }, int64Operands());
831 RUN_BINARY(testLoad, { MAKE_OPERAND(Float) }, floatingPointOperands<float>());
832 RUN_BINARY(testLoad, { MAKE_OPERAND(Double) }, floatingPointOperands<double>());
833
834 RUN(testLoad<int8_t>(Load8S, 60));
835 RUN(testLoad<int8_t>(Load8S, -60));
836 RUN(testLoad<int8_t>(Load8S, 1000));
837 RUN(testLoad<int8_t>(Load8S, -1000));
838 RUN(testLoad<int8_t>(Load8S, 1000000));
839 RUN(testLoad<int8_t>(Load8S, -1000000));
840 RUN(testLoad<int8_t>(Load8S, 1000000000));
841 RUN(testLoad<int8_t>(Load8S, -1000000000));
842
843 RUN(testLoad<uint8_t>(Load8Z, 60));
844 RUN(testLoad<uint8_t>(Load8Z, -60));
845 RUN(testLoad<uint8_t>(Load8Z, 1000));
846 RUN(testLoad<uint8_t>(Load8Z, -1000));
847 RUN(testLoad<uint8_t>(Load8Z, 1000000));
848 RUN(testLoad<uint8_t>(Load8Z, -1000000));
849 RUN(testLoad<uint8_t>(Load8Z, 1000000000));
850 RUN(testLoad<uint8_t>(Load8Z, -1000000000));
851
852 RUN(testLoad<int16_t>(Load16S, 60));
853 RUN(testLoad<int16_t>(Load16S, -60));
854 RUN(testLoad<int16_t>(Load16S, 1000));
855 RUN(testLoad<int16_t>(Load16S, -1000));
856 RUN(testLoad<int16_t>(Load16S, 1000000));
857 RUN(testLoad<int16_t>(Load16S, -1000000));
858 RUN(testLoad<int16_t>(Load16S, 1000000000));
859 RUN(testLoad<int16_t>(Load16S, -1000000000));
860
861 RUN(testLoad<uint16_t>(Load16Z, 60));
862 RUN(testLoad<uint16_t>(Load16Z, -60));
863 RUN(testLoad<uint16_t>(Load16Z, 1000));
864 RUN(testLoad<uint16_t>(Load16Z, -1000));
865 RUN(testLoad<uint16_t>(Load16Z, 1000000));
866 RUN(testLoad<uint16_t>(Load16Z, -1000000));
867 RUN(testLoad<uint16_t>(Load16Z, 1000000000));
868 RUN(testLoad<uint16_t>(Load16Z, -1000000000));
869}
870
871void testFastForwardCopy32()
872{
873#if CPU(X86_64)
874 for (const bool aligned : { true, false }) {
875 for (const bool overlap : { false, true }) {
876 for (size_t arrsize : { 1, 4, 5, 6, 8, 10, 12, 16, 20, 40, 100, 1000}) {
877 size_t overlapAmount = 5;
878
879 UniqueArray<uint32_t> array1, array2;
880 uint32_t* arr1, *arr2;
881
882 if (overlap) {
883 array1 = makeUniqueArray<uint32_t>(arrsize * 2);
884 arr1 = &array1[0];
885 arr2 = arr1 + (arrsize - overlapAmount);
886 } else {
887 array1 = makeUniqueArray<uint32_t>(arrsize);
888 array2 = makeUniqueArray<uint32_t>(arrsize);
889 arr1 = &array1[0];
890 arr2 = &array2[0];
891 }
892
893 if (!aligned && arrsize < 3)
894 continue;
895 if (overlap && arrsize <= overlapAmount + 3)
896 continue;
897
898 if (!aligned) {
899 ++arr1;
900 ++arr2;
901 arrsize -= 1;
902 overlapAmount -= 1;
903 }
904
905 for (size_t i = 0; i < arrsize; ++i)
906 arr1[i] = i;
907
908 fastForwardCopy32(arr2, arr1, arrsize);
909
910 if (overlap) {
911 for (size_t i = 0; i < arrsize - overlapAmount; ++i)
912 CHECK(arr2[i] == i);
913 for (size_t i = arrsize - overlapAmount; i < arrsize; ++i)
914 CHECK(arr2[i] == i - (arrsize - overlapAmount));
915 } else {
916 for (size_t i = 0; i < arrsize; ++i)
917 CHECK(arr2[i] == i);
918 }
919
920 if (!aligned) {
921 --arr1;
922 --arr2;
923 }
924 }
925 }
926 }
927#endif
928}
929
930void testByteCopyLoop()
931{
932 Procedure proc;
933 BasicBlock* root = proc.addBlock();
934 BasicBlock* head = proc.addBlock();
935 BasicBlock* update = proc.addBlock();
936 BasicBlock* continuation = proc.addBlock();
937
938 auto* arraySrc = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
939 auto* arrayDst = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
940 auto* arraySize = root->appendNew<Value>(proc, Trunc, Origin(), root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2));
941 auto* one = root->appendNew<Const32Value>(proc, Origin(), 1);
942 auto* two = root->appendNew<Const32Value>(proc, Origin(), 2);
943 UpsilonValue* startingIndex = root->appendNew<UpsilonValue>(proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 0));
944 root->appendNew<Value>(proc, Jump, Origin());
945 root->setSuccessors(FrequentedBlock(head));
946
947 auto* index = head->appendNew<Value>(proc, Phi, Int32, Origin());
948 startingIndex->setPhi(index);
949 auto* loadIndex = head->appendNew<Value>(proc, Add, Origin(), arraySrc,
950 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
951 auto* storeIndex = head->appendNew<Value>(proc, Add, Origin(), arrayDst,
952 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
953 head->appendNew<MemoryValue>(proc, Store, Origin(), head->appendNew<MemoryValue>(proc, Load, Int32, Origin(), loadIndex), storeIndex);
954 auto* newIndex = head->appendNew<Value>(proc, Add, Origin(), index, one);
955 auto* cmpValue = head->appendNew<Value>(proc, GreaterThan, Origin(), newIndex, arraySize);
956 head->appendNew<Value>(proc, Branch, Origin(), cmpValue);
957 head->setSuccessors(FrequentedBlock(continuation), FrequentedBlock(update));
958
959 UpsilonValue* updateIndex = update->appendNew<UpsilonValue>(proc, Origin(), newIndex);
960 updateIndex->setPhi(index);
961 update->appendNew<Value>(proc, Jump, Origin());
962 update->setSuccessors(FrequentedBlock(head));
963
964 continuation->appendNewControlValue(proc, Return, Origin());
965
966 int* arr1 = new int[3];
967 int* arr2 = new int[3];
968
969 arr1[0] = 0;
970 arr1[1] = 0;
971 arr1[2] = 0;
972 arr2[0] = 1;
973 arr2[1] = 2;
974 arr2[2] = 3;
975
976 compileAndRun<void>(proc, arr2, arr1, 3);
977
978 CHECK_EQ(arr1[0], 1);
979 CHECK_EQ(arr1[1], 2);
980 CHECK_EQ(arr1[2], 3);
981
982 delete[] arr1;
983 delete [] arr2;
984}
985
986void testByteCopyLoopStartIsLoopDependent()
987{
988 Procedure proc;
989 BasicBlock* root = proc.addBlock();
990 BasicBlock* head = proc.addBlock();
991 BasicBlock* update = proc.addBlock();
992 BasicBlock* continuation = proc.addBlock();
993
994 auto* arraySrc = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
995 auto* arrayDst = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
996 auto* arraySize = root->appendNew<Value>(proc, Trunc, Origin(), root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2));
997 auto* one = root->appendNew<Const32Value>(proc, Origin(), 1);
998 auto* two = root->appendNew<Const32Value>(proc, Origin(), 2);
999 root->appendNew<Value>(proc, Jump, Origin());
1000 root->setSuccessors(FrequentedBlock(head));
1001
1002 UpsilonValue* startingIndex = head->appendNew<UpsilonValue>(proc, Origin(), head->appendNew<Const32Value>(proc, Origin(), 0));
1003 auto* index = head->appendNew<Value>(proc, Phi, Int32, Origin());
1004 startingIndex->setPhi(index);
1005 auto* loadIndex = head->appendNew<Value>(proc, Add, Origin(), arraySrc,
1006 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
1007 auto* storeIndex = head->appendNew<Value>(proc, Add, Origin(), arrayDst,
1008 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
1009 head->appendNew<MemoryValue>(proc, Store, Origin(), head->appendNew<MemoryValue>(proc, Load, Int32, Origin(), loadIndex), storeIndex);
1010 auto* newIndex = head->appendNew<Value>(proc, Add, Origin(), index, one);
1011 auto* cmpValue = head->appendNew<Value>(proc, GreaterThan, Origin(), newIndex, arraySize);
1012 head->appendNew<Value>(proc, Branch, Origin(), cmpValue);
1013 head->setSuccessors(FrequentedBlock(continuation), FrequentedBlock(update));
1014
1015 UpsilonValue* updateIndex = update->appendNew<UpsilonValue>(proc, Origin(), newIndex);
1016 updateIndex->setPhi(index);
1017 update->appendNew<Value>(proc, Jump, Origin());
1018 update->setSuccessors(FrequentedBlock(head));
1019
1020 continuation->appendNewControlValue(proc, Return, Origin());
1021
1022 int* arr1 = new int[3];
1023 int* arr2 = new int[3];
1024
1025 arr1[0] = 0;
1026 arr1[1] = 0;
1027 arr1[2] = 0;
1028 arr2[0] = 1;
1029 arr2[1] = 2;
1030 arr2[2] = 3;
1031
1032 compileAndRun<void>(proc, arr2, arr1, 0);
1033
1034 CHECK_EQ(arr1[0], 1);
1035 CHECK_EQ(arr1[1], 0);
1036 CHECK_EQ(arr1[2], 0);
1037
1038 delete[] arr1;
1039 delete [] arr2;
1040}
1041
1042void testByteCopyLoopBoundIsLoopDependent()
1043{
1044 Procedure proc;
1045 BasicBlock* root = proc.addBlock();
1046 BasicBlock* head = proc.addBlock();
1047 BasicBlock* update = proc.addBlock();
1048 BasicBlock* continuation = proc.addBlock();
1049
1050 auto* arraySrc = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1051 auto* arrayDst = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
1052 auto* one = root->appendNew<Const32Value>(proc, Origin(), 1);
1053 auto* two = root->appendNew<Const32Value>(proc, Origin(), 2);
1054 UpsilonValue* startingIndex = root->appendNew<UpsilonValue>(proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 0));
1055 root->appendNew<Value>(proc, Jump, Origin());
1056 root->setSuccessors(FrequentedBlock(head));
1057
1058 auto* index = head->appendNew<Value>(proc, Phi, Int32, Origin());
1059 startingIndex->setPhi(index);
1060 auto* loadIndex = head->appendNew<Value>(proc, Add, Origin(), arraySrc,
1061 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
1062 auto* storeIndex = head->appendNew<Value>(proc, Add, Origin(), arrayDst,
1063 head->appendNew<Value>(proc, ZExt32, Origin(), head->appendNew<Value>(proc, Shl, Origin(), index, two)));
1064 head->appendNew<MemoryValue>(proc, Store, Origin(), head->appendNew<MemoryValue>(proc, Load, Int32, Origin(), loadIndex), storeIndex);
1065 auto* newIndex = head->appendNew<Value>(proc, Add, Origin(), index, one);
1066 auto* cmpValue = head->appendNew<Value>(proc, GreaterThan, Origin(), newIndex, index);
1067 head->appendNew<Value>(proc, Branch, Origin(), cmpValue);
1068 head->setSuccessors(FrequentedBlock(continuation), FrequentedBlock(update));
1069
1070 UpsilonValue* updateIndex = update->appendNew<UpsilonValue>(proc, Origin(), newIndex);
1071 updateIndex->setPhi(index);
1072 update->appendNew<Value>(proc, Jump, Origin());
1073 update->setSuccessors(FrequentedBlock(head));
1074
1075 continuation->appendNewControlValue(proc, Return, Origin());
1076
1077 int* arr1 = new int[3];
1078 int* arr2 = new int[3];
1079
1080 arr1[0] = 0;
1081 arr1[1] = 0;
1082 arr1[2] = 0;
1083 arr2[0] = 1;
1084 arr2[1] = 2;
1085 arr2[2] = 3;
1086
1087 compileAndRun<void>(proc, arr2, arr1, 3);
1088
1089 CHECK_EQ(arr1[0], 1);
1090 CHECK_EQ(arr1[1], 0);
1091 CHECK_EQ(arr1[2], 0);
1092
1093 delete[] arr1;
1094 delete [] arr2;
1095}
1096
1097void addCopyTests(const char* filter, Deque<RefPtr<SharedTask<void()>>>& tasks)
1098{
1099 RUN(testFastForwardCopy32());
1100 RUN(testByteCopyLoop());
1101 RUN(testByteCopyLoopStartIsLoopDependent());
1102 RUN(testByteCopyLoopBoundIsLoopDependent());
1103}
1104
1105#endif // ENABLE(B3_JIT)
1106