1/*
2 * Copyright (C) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "testb3.h"
28
29#if ENABLE(B3_JIT)
30
31void testPinRegisters()
32{
33 auto go = [&] (bool pin) {
34 Procedure proc;
35 RegisterSet csrs;
36 csrs.merge(RegisterSet::calleeSaveRegisters());
37 csrs.exclude(RegisterSet::stackRegisters());
38 if (pin) {
39 csrs.forEach(
40 [&] (Reg reg) {
41 proc.pinRegister(reg);
42 });
43 }
44 BasicBlock* root = proc.addBlock();
45 Value* a = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
46 Value* b = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
47 Value* c = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2);
48 Value* d = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::regCS0);
49 root->appendNew<CCallValue>(
50 proc, Void, Origin(),
51 root->appendNew<ConstPtrValue>(proc, Origin(), static_cast<intptr_t>(0x1234)));
52 root->appendNew<CCallValue>(
53 proc, Void, Origin(),
54 root->appendNew<ConstPtrValue>(proc, Origin(), static_cast<intptr_t>(0x1235)),
55 a, b, c);
56 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
57 patchpoint->appendSomeRegister(d);
58 unsigned optLevel = proc.optLevel();
59 patchpoint->setGenerator(
60 [&] (CCallHelpers&, const StackmapGenerationParams& params) {
61 if (optLevel > 1)
62 CHECK_EQ(params[0].gpr(), GPRInfo::regCS0);
63 });
64 root->appendNew<Value>(proc, Return, Origin());
65 auto code = compileProc(proc);
66 bool usesCSRs = false;
67 for (Air::BasicBlock* block : proc.code()) {
68 for (Air::Inst& inst : *block) {
69 if (inst.kind.opcode == Air::Patch && inst.origin == patchpoint)
70 continue;
71 inst.forEachTmpFast(
72 [&] (Air::Tmp tmp) {
73 if (tmp.isReg())
74 usesCSRs |= csrs.get(tmp.reg());
75 });
76 }
77 }
78 if (proc.optLevel() < 2) {
79 // Our less good register allocators may use the
80 // pinned CSRs in a move.
81 usesCSRs = false;
82 }
83 for (const RegisterAtOffset& regAtOffset : proc.calleeSaveRegisterAtOffsetList())
84 usesCSRs |= csrs.get(regAtOffset.reg());
85 CHECK_EQ(usesCSRs, !pin);
86 };
87
88 go(true);
89 go(false);
90}
91
92void testX86LeaAddAddShlLeft()
93{
94 // Add(Add(Shl(@x, $c), @y), $d)
95 Procedure proc;
96 BasicBlock* root = proc.addBlock();
97 Value* result = root->appendNew<Value>(
98 proc, Add, Origin(),
99 root->appendNew<Value>(
100 proc, Add, Origin(),
101 root->appendNew<Value>(
102 proc, Shl, Origin(),
103 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
104 root->appendNew<Const32Value>(proc, Origin(), 2)),
105 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
106 root->appendNew<ConstPtrValue>(proc, Origin(), 100));
107 root->appendNew<Value>(proc, Return, Origin(), result);
108
109 auto code = compileProc(proc);
110 if (proc.optLevel() > 1)
111 checkUsesInstruction(*code, "lea 0x64(%rdi,%rsi,4), %rax");
112 else
113 checkUsesInstruction(*code, "lea");
114 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), (1 + (2 << 2)) + 100);
115}
116
117void testX86LeaAddAddShlRight()
118{
119 // Add(Add(@x, Shl(@y, $c)), $d)
120 Procedure proc;
121 BasicBlock* root = proc.addBlock();
122 Value* result = root->appendNew<Value>(
123 proc, Add, Origin(),
124 root->appendNew<Value>(
125 proc, Add, Origin(),
126 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
127 root->appendNew<Value>(
128 proc, Shl, Origin(),
129 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
130 root->appendNew<Const32Value>(proc, Origin(), 2))),
131 root->appendNew<ConstPtrValue>(proc, Origin(), 100));
132 root->appendNew<Value>(proc, Return, Origin(), result);
133
134 auto code = compileProc(proc);
135 if (proc.optLevel() > 1)
136 checkUsesInstruction(*code, "lea 0x64(%rdi,%rsi,4), %rax");
137 else
138 checkUsesInstruction(*code, "lea");
139 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), (1 + (2 << 2)) + 100);
140}
141
142void testX86LeaAddAdd()
143{
144 // Add(Add(@x, @y), $c)
145 Procedure proc;
146 BasicBlock* root = proc.addBlock();
147 Value* result = root->appendNew<Value>(
148 proc, Add, Origin(),
149 root->appendNew<Value>(
150 proc, Add, Origin(),
151 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
152 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
153 root->appendNew<ConstPtrValue>(proc, Origin(), 100));
154 root->appendNew<Value>(proc, Return, Origin(), result);
155
156 auto code = compileProc(proc);
157 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), (1 + 2) + 100);
158 if (proc.optLevel() > 1) {
159 checkDisassembly(
160 *code,
161 [&] (const char* disassembly) -> bool {
162 return strstr(disassembly, "lea 0x64(%rdi,%rsi), %rax")
163 || strstr(disassembly, "lea 0x64(%rsi,%rdi), %rax");
164 },
165 "Expected to find something like lea 0x64(%rdi,%rsi), %rax but didn't!");
166 }
167}
168
169void testX86LeaAddShlRight()
170{
171 // Add(Shl(@x, $c), @y)
172 Procedure proc;
173 BasicBlock* root = proc.addBlock();
174 Value* result = root->appendNew<Value>(
175 proc, Add, Origin(),
176 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
177 root->appendNew<Value>(
178 proc, Shl, Origin(),
179 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
180 root->appendNew<Const32Value>(proc, Origin(), 2)));
181 root->appendNew<Value>(proc, Return, Origin(), result);
182
183 auto code = compileProc(proc);
184 if (proc.optLevel() > 1)
185 checkUsesInstruction(*code, "lea (%rdi,%rsi,4), %rax");
186 else
187 checkUsesInstruction(*code, "lea");
188 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + (2 << 2));
189}
190
191void testX86LeaAddShlLeftScale1()
192{
193 // Add(Shl(@x, $c), @y)
194 Procedure proc;
195 BasicBlock* root = proc.addBlock();
196 Value* result = root->appendNew<Value>(
197 proc, Add, Origin(),
198 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
199 root->appendNew<Value>(
200 proc, Shl, Origin(),
201 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
202 root->appendNew<Const32Value>(proc, Origin(), 0)));
203 root->appendNew<Value>(proc, Return, Origin(), result);
204
205 auto code = compileProc(proc);
206 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + 2);
207 if (proc.optLevel() > 1) {
208 checkDisassembly(
209 *code,
210 [&] (const char* disassembly) -> bool {
211 return strstr(disassembly, "lea (%rdi,%rsi), %rax")
212 || strstr(disassembly, "lea (%rsi,%rdi), %rax");
213 },
214 "Expected to find something like lea (%rdi,%rsi), %rax but didn't!");
215 }
216}
217
218void testX86LeaAddShlLeftScale2()
219{
220 // Add(Shl(@x, $c), @y)
221 Procedure proc;
222 BasicBlock* root = proc.addBlock();
223 Value* result = root->appendNew<Value>(
224 proc, Add, Origin(),
225 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
226 root->appendNew<Value>(
227 proc, Shl, Origin(),
228 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
229 root->appendNew<Const32Value>(proc, Origin(), 1)));
230 root->appendNew<Value>(proc, Return, Origin(), result);
231
232 auto code = compileProc(proc);
233 if (proc.optLevel() > 1)
234 checkUsesInstruction(*code, "lea (%rdi,%rsi,2), %rax");
235 else
236 checkUsesInstruction(*code, "lea");
237 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + (2 << 1));
238}
239
240void testX86LeaAddShlLeftScale4()
241{
242 // Add(Shl(@x, $c), @y)
243 Procedure proc;
244 BasicBlock* root = proc.addBlock();
245 Value* result = root->appendNew<Value>(
246 proc, Add, Origin(),
247 root->appendNew<Value>(
248 proc, Shl, Origin(),
249 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
250 root->appendNew<Const32Value>(proc, Origin(), 2)),
251 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
252 root->appendNew<Value>(proc, Return, Origin(), result);
253
254 auto code = compileProc(proc);
255 if (proc.optLevel() > 1)
256 checkUsesInstruction(*code, "lea (%rdi,%rsi,4), %rax");
257 else
258 checkUsesInstruction(*code, "lea");
259 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + (2 << 2));
260}
261
262void testX86LeaAddShlLeftScale8()
263{
264 // Add(Shl(@x, $c), @y)
265 Procedure proc;
266 BasicBlock* root = proc.addBlock();
267 Value* result = root->appendNew<Value>(
268 proc, Add, Origin(),
269 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
270 root->appendNew<Value>(
271 proc, Shl, Origin(),
272 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
273 root->appendNew<Const32Value>(proc, Origin(), 3)));
274 root->appendNew<Value>(proc, Return, Origin(), result);
275
276 auto code = compileProc(proc);
277 if (proc.optLevel() > 1)
278 checkUsesInstruction(*code, "lea (%rdi,%rsi,8), %rax");
279 else
280 checkUsesInstruction(*code, "lea");
281 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + (2 << 3));
282}
283
284void testAddShl32()
285{
286 // Add(Shl(@x, $c), @y)
287 Procedure proc;
288 BasicBlock* root = proc.addBlock();
289 Value* result = root->appendNew<Value>(
290 proc, Add, Origin(),
291 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
292 root->appendNew<Value>(
293 proc, Shl, Origin(),
294 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
295 root->appendNew<Const32Value>(proc, Origin(), 32)));
296 root->appendNew<Value>(proc, Return, Origin(), result);
297
298 auto code = compileProc(proc);
299 CHECK_EQ(invoke<int64_t>(*code, 1, 2), 1 + (static_cast<int64_t>(2) << static_cast<int64_t>(32)));
300}
301
302void testAddShl64()
303{
304 // Add(Shl(@x, $c), @y)
305 Procedure proc;
306 BasicBlock* root = proc.addBlock();
307 Value* result = root->appendNew<Value>(
308 proc, Add, Origin(),
309 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
310 root->appendNew<Value>(
311 proc, Shl, Origin(),
312 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
313 root->appendNew<Const32Value>(proc, Origin(), 64)));
314 root->appendNew<Value>(proc, Return, Origin(), result);
315
316 auto code = compileProc(proc);
317 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + 2);
318}
319
320void testAddShl65()
321{
322 // Add(Shl(@x, $c), @y)
323 Procedure proc;
324 BasicBlock* root = proc.addBlock();
325 Value* result = root->appendNew<Value>(
326 proc, Add, Origin(),
327 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
328 root->appendNew<Value>(
329 proc, Shl, Origin(),
330 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
331 root->appendNew<Const32Value>(proc, Origin(), 65)));
332 root->appendNew<Value>(proc, Return, Origin(), result);
333
334 auto code = compileProc(proc);
335 CHECK_EQ(invoke<intptr_t>(*code, 1, 2), 1 + (2 << 1));
336}
337
338void testReduceStrengthReassociation(bool flip)
339{
340 // Add(Add(@x, $c), @y) -> Add(Add(@x, @y), $c)
341 // and
342 // Add(@y, Add(@x, $c)) -> Add(Add(@x, @y), $c)
343 Procedure proc;
344 BasicBlock* root = proc.addBlock();
345 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
346 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
347
348 Value* innerAdd = root->appendNew<Value>(
349 proc, Add, Origin(), arg1,
350 root->appendNew<ConstPtrValue>(proc, Origin(), 42));
351
352 Value* outerAdd;
353 if (flip)
354 outerAdd = root->appendNew<Value>(proc, Add, Origin(), arg2, innerAdd);
355 else
356 outerAdd = root->appendNew<Value>(proc, Add, Origin(), innerAdd, arg2);
357
358 root->appendNew<Value>(proc, Return, Origin(), outerAdd);
359
360 proc.resetReachability();
361
362 if (shouldBeVerbose()) {
363 dataLog("IR before reduceStrength:\n");
364 dataLog(proc);
365 }
366
367 reduceStrength(proc);
368
369 if (shouldBeVerbose()) {
370 dataLog("IR after reduceStrength:\n");
371 dataLog(proc);
372 }
373
374 CHECK_EQ(root->last()->opcode(), Return);
375 CHECK_EQ(root->last()->child(0)->opcode(), Add);
376 CHECK(root->last()->child(0)->child(1)->isIntPtr(42));
377 CHECK_EQ(root->last()->child(0)->child(0)->opcode(), Add);
378 CHECK(
379 (root->last()->child(0)->child(0)->child(0) == arg1 && root->last()->child(0)->child(0)->child(1) == arg2) ||
380 (root->last()->child(0)->child(0)->child(0) == arg2 && root->last()->child(0)->child(0)->child(1) == arg1));
381}
382
383void testLoadBaseIndexShift2()
384{
385 Procedure proc;
386 BasicBlock* root = proc.addBlock();
387 root->appendNew<Value>(
388 proc, Return, Origin(),
389 root->appendNew<MemoryValue>(
390 proc, Load, Int32, Origin(),
391 root->appendNew<Value>(
392 proc, Add, Origin(),
393 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
394 root->appendNew<Value>(
395 proc, Shl, Origin(),
396 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
397 root->appendNew<Const32Value>(proc, Origin(), 2)))));
398 auto code = compileProc(proc);
399 if (isX86() && proc.optLevel() > 1)
400 checkUsesInstruction(*code, "(%rdi,%rsi,4)");
401 int32_t value = 12341234;
402 char* ptr = bitwise_cast<char*>(&value);
403 for (unsigned i = 0; i < 10; ++i)
404 CHECK_EQ(invoke<int32_t>(*code, ptr - (static_cast<intptr_t>(1) << static_cast<intptr_t>(2)) * i, i), 12341234);
405}
406
407void testLoadBaseIndexShift32()
408{
409#if CPU(ADDRESS64)
410 Procedure proc;
411 BasicBlock* root = proc.addBlock();
412 root->appendNew<Value>(
413 proc, Return, Origin(),
414 root->appendNew<MemoryValue>(
415 proc, Load, Int32, Origin(),
416 root->appendNew<Value>(
417 proc, Add, Origin(),
418 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
419 root->appendNew<Value>(
420 proc, Shl, Origin(),
421 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
422 root->appendNew<Const32Value>(proc, Origin(), 32)))));
423 auto code = compileProc(proc);
424 int32_t value = 12341234;
425 char* ptr = bitwise_cast<char*>(&value);
426 for (unsigned i = 0; i < 10; ++i)
427 CHECK_EQ(invoke<int32_t>(*code, ptr - (static_cast<intptr_t>(1) << static_cast<intptr_t>(32)) * i, i), 12341234);
428#endif
429}
430
431void testOptimizeMaterialization()
432{
433 Procedure proc;
434 if (proc.optLevel() < 2)
435 return;
436
437 BasicBlock* root = proc.addBlock();
438 root->appendNew<CCallValue>(
439 proc, Void, Origin(),
440 root->appendNew<ConstPtrValue>(proc, Origin(), 0x123423453456llu),
441 root->appendNew<ConstPtrValue>(proc, Origin(), 0x123423453456llu + 35));
442 root->appendNew<Value>(proc, Return, Origin());
443
444 auto code = compileProc(proc);
445 bool found = false;
446 for (Air::BasicBlock* block : proc.code()) {
447 for (Air::Inst& inst : *block) {
448 if (inst.kind.opcode != Air::Add64)
449 continue;
450 if (inst.args[0] != Air::Arg::imm(35))
451 continue;
452 found = true;
453 }
454 }
455 CHECK(found);
456}
457
458template<typename Func>
459void generateLoop(Procedure& proc, const Func& func)
460{
461 BasicBlock* root = proc.addBlock();
462 BasicBlock* loop = proc.addBlock();
463 BasicBlock* end = proc.addBlock();
464
465 UpsilonValue* initialIndex = root->appendNew<UpsilonValue>(
466 proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 0));
467 root->appendNew<Value>(proc, Jump, Origin());
468 root->setSuccessors(loop);
469
470 Value* index = loop->appendNew<Value>(proc, Phi, Int32, Origin());
471 initialIndex->setPhi(index);
472
473 Value* one = func(loop, index);
474
475 Value* nextIndex = loop->appendNew<Value>(proc, Add, Origin(), index, one);
476 UpsilonValue* loopIndex = loop->appendNew<UpsilonValue>(proc, Origin(), nextIndex);
477 loopIndex->setPhi(index);
478 loop->appendNew<Value>(
479 proc, Branch, Origin(),
480 loop->appendNew<Value>(
481 proc, LessThan, Origin(), nextIndex,
482 loop->appendNew<Const32Value>(proc, Origin(), 100)));
483 loop->setSuccessors(loop, end);
484
485 end->appendNew<Value>(proc, Return, Origin());
486}
487
488static std::array<int, 100> makeArrayForLoops()
489{
490 std::array<int, 100> result;
491 for (unsigned i = 0; i < result.size(); ++i)
492 result[i] = i & 1;
493 return result;
494}
495
496template<typename Func>
497void generateLoopNotBackwardsDominant(Procedure& proc, std::array<int, 100>& array, const Func& func)
498{
499 BasicBlock* root = proc.addBlock();
500 BasicBlock* loopHeader = proc.addBlock();
501 BasicBlock* loopCall = proc.addBlock();
502 BasicBlock* loopFooter = proc.addBlock();
503 BasicBlock* end = proc.addBlock();
504
505 UpsilonValue* initialIndex = root->appendNew<UpsilonValue>(
506 proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 0));
507 // If you look carefully, you'll notice that this is an extremely sneaky use of Upsilon that demonstrates
508 // the extent to which our SSA is different from normal-person SSA.
509 UpsilonValue* defaultOne = root->appendNew<UpsilonValue>(
510 proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 1));
511 root->appendNew<Value>(proc, Jump, Origin());
512 root->setSuccessors(loopHeader);
513
514 Value* index = loopHeader->appendNew<Value>(proc, Phi, Int32, Origin());
515 initialIndex->setPhi(index);
516
517 // if (array[index])
518 loopHeader->appendNew<Value>(
519 proc, Branch, Origin(),
520 loopHeader->appendNew<MemoryValue>(
521 proc, Load, Int32, Origin(),
522 loopHeader->appendNew<Value>(
523 proc, Add, Origin(),
524 loopHeader->appendNew<ConstPtrValue>(proc, Origin(), &array),
525 loopHeader->appendNew<Value>(
526 proc, Mul, Origin(),
527 loopHeader->appendNew<Value>(proc, ZExt32, Origin(), index),
528 loopHeader->appendNew<ConstPtrValue>(proc, Origin(), sizeof(int))))));
529 loopHeader->setSuccessors(loopCall, loopFooter);
530
531 Value* functionCall = func(loopCall, index);
532 UpsilonValue* oneFromFunction = loopCall->appendNew<UpsilonValue>(proc, Origin(), functionCall);
533 loopCall->appendNew<Value>(proc, Jump, Origin());
534 loopCall->setSuccessors(loopFooter);
535
536 Value* one = loopFooter->appendNew<Value>(proc, Phi, Int32, Origin());
537 defaultOne->setPhi(one);
538 oneFromFunction->setPhi(one);
539 Value* nextIndex = loopFooter->appendNew<Value>(proc, Add, Origin(), index, one);
540 UpsilonValue* loopIndex = loopFooter->appendNew<UpsilonValue>(proc, Origin(), nextIndex);
541 loopIndex->setPhi(index);
542 loopFooter->appendNew<Value>(
543 proc, Branch, Origin(),
544 loopFooter->appendNew<Value>(
545 proc, LessThan, Origin(), nextIndex,
546 loopFooter->appendNew<Const32Value>(proc, Origin(), 100)));
547 loopFooter->setSuccessors(loopHeader, end);
548
549 end->appendNew<Value>(proc, Return, Origin());
550}
551
552static int oneFunction(int* callCount)
553{
554 (*callCount)++;
555 return 1;
556}
557
558static void noOpFunction()
559{
560}
561
562void testLICMPure()
563{
564 Procedure proc;
565
566 if (proc.optLevel() < 2)
567 return;
568
569 generateLoop(
570 proc,
571 [&] (BasicBlock* loop, Value*) -> Value* {
572 return loop->appendNew<CCallValue>(
573 proc, Int32, Origin(), Effects::none(),
574 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
575 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
576 });
577
578 unsigned callCount = 0;
579 compileAndRun<void>(proc, &callCount);
580 CHECK_EQ(callCount, 1u);
581}
582
583void testLICMPureSideExits()
584{
585 Procedure proc;
586 if (proc.optLevel() < 2)
587 return;
588 generateLoop(
589 proc,
590 [&] (BasicBlock* loop, Value*) -> Value* {
591 Effects effects = Effects::none();
592 effects.exitsSideways = true;
593 loop->appendNew<CCallValue>(
594 proc, Void, Origin(), effects,
595 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
596
597 return loop->appendNew<CCallValue>(
598 proc, Int32, Origin(), Effects::none(),
599 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
600 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
601 });
602
603 unsigned callCount = 0;
604 compileAndRun<void>(proc, &callCount);
605 CHECK_EQ(callCount, 1u);
606}
607
608void testLICMPureWritesPinned()
609{
610 Procedure proc;
611 if (proc.optLevel() < 2)
612 return;
613 generateLoop(
614 proc,
615 [&] (BasicBlock* loop, Value*) -> Value* {
616 Effects effects = Effects::none();
617 effects.writesPinned = true;
618 loop->appendNew<CCallValue>(
619 proc, Void, Origin(), effects,
620 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
621
622 return loop->appendNew<CCallValue>(
623 proc, Int32, Origin(), Effects::none(),
624 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
625 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
626 });
627
628 unsigned callCount = 0;
629 compileAndRun<void>(proc, &callCount);
630 CHECK_EQ(callCount, 1u);
631}
632
633void testLICMPureWrites()
634{
635 Procedure proc;
636 if (proc.optLevel() < 2)
637 return;
638 generateLoop(
639 proc,
640 [&] (BasicBlock* loop, Value*) -> Value* {
641 Effects effects = Effects::none();
642 effects.writes = HeapRange(63479);
643 loop->appendNew<CCallValue>(
644 proc, Void, Origin(), effects,
645 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
646
647 return loop->appendNew<CCallValue>(
648 proc, Int32, Origin(), Effects::none(),
649 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
650 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
651 });
652
653 unsigned callCount = 0;
654 compileAndRun<void>(proc, &callCount);
655 CHECK_EQ(callCount, 1u);
656}
657
658void testLICMReadsLocalState()
659{
660 Procedure proc;
661 generateLoop(
662 proc,
663 [&] (BasicBlock* loop, Value*) -> Value* {
664 Effects effects = Effects::none();
665 effects.readsLocalState = true;
666 return loop->appendNew<CCallValue>(
667 proc, Int32, Origin(), effects,
668 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
669 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
670 });
671
672 unsigned callCount = 0;
673 compileAndRun<void>(proc, &callCount);
674 CHECK_EQ(callCount, 100u); // We'll fail to hoist because the loop has Upsilons.
675}
676
677void testLICMReadsPinned()
678{
679 Procedure proc;
680 if (proc.optLevel() < 2)
681 return;
682 generateLoop(
683 proc,
684 [&] (BasicBlock* loop, Value*) -> Value* {
685 Effects effects = Effects::none();
686 effects.readsPinned = true;
687 return loop->appendNew<CCallValue>(
688 proc, Int32, Origin(), effects,
689 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
690 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
691 });
692
693 unsigned callCount = 0;
694 compileAndRun<void>(proc, &callCount);
695 CHECK_EQ(callCount, 1u);
696}
697
698void testLICMReads()
699{
700 Procedure proc;
701 if (proc.optLevel() < 2)
702 return;
703 generateLoop(
704 proc,
705 [&] (BasicBlock* loop, Value*) -> Value* {
706 Effects effects = Effects::none();
707 effects.reads = HeapRange::top();
708 return loop->appendNew<CCallValue>(
709 proc, Int32, Origin(), effects,
710 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
711 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
712 });
713
714 unsigned callCount = 0;
715 compileAndRun<void>(proc, &callCount);
716 CHECK_EQ(callCount, 1u);
717}
718
719void testLICMPureNotBackwardsDominant()
720{
721 Procedure proc;
722 if (proc.optLevel() < 2)
723 return;
724 auto array = makeArrayForLoops();
725 generateLoopNotBackwardsDominant(
726 proc, array,
727 [&] (BasicBlock* loop, Value*) -> Value* {
728 return loop->appendNew<CCallValue>(
729 proc, Int32, Origin(), Effects::none(),
730 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
731 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
732 });
733
734 unsigned callCount = 0;
735 compileAndRun<void>(proc, &callCount);
736 CHECK_EQ(callCount, 1u);
737}
738
739void testLICMPureFoiledByChild()
740{
741 Procedure proc;
742 generateLoop(
743 proc,
744 [&] (BasicBlock* loop, Value* index) -> Value* {
745 return loop->appendNew<CCallValue>(
746 proc, Int32, Origin(), Effects::none(),
747 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
748 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
749 index);
750 });
751
752 unsigned callCount = 0;
753 compileAndRun<void>(proc, &callCount);
754 CHECK_EQ(callCount, 100u);
755}
756
757void testLICMPureNotBackwardsDominantFoiledByChild()
758{
759 Procedure proc;
760 if (proc.optLevel() < 2)
761 return;
762 auto array = makeArrayForLoops();
763 generateLoopNotBackwardsDominant(
764 proc, array,
765 [&] (BasicBlock* loop, Value* index) -> Value* {
766 return loop->appendNew<CCallValue>(
767 proc, Int32, Origin(), Effects::none(),
768 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
769 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
770 index);
771 });
772
773 unsigned callCount = 0;
774 compileAndRun<void>(proc, &callCount);
775 CHECK_EQ(callCount, 50u);
776}
777
778void testLICMExitsSideways()
779{
780 Procedure proc;
781 generateLoop(
782 proc,
783 [&] (BasicBlock* loop, Value*) -> Value* {
784 Effects effects = Effects::none();
785 effects.exitsSideways = true;
786 return loop->appendNew<CCallValue>(
787 proc, Int32, Origin(), effects,
788 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
789 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
790 });
791
792 unsigned callCount = 0;
793 compileAndRun<void>(proc, &callCount);
794 CHECK_EQ(callCount, 100u);
795}
796
797void testLICMWritesLocalState()
798{
799 Procedure proc;
800 generateLoop(
801 proc,
802 [&] (BasicBlock* loop, Value*) -> Value* {
803 Effects effects = Effects::none();
804 effects.writesLocalState = true;
805 return loop->appendNew<CCallValue>(
806 proc, Int32, Origin(), effects,
807 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
808 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
809 });
810
811 unsigned callCount = 0;
812 compileAndRun<void>(proc, &callCount);
813 CHECK_EQ(callCount, 100u);
814}
815
816void testLICMWrites()
817{
818 Procedure proc;
819 generateLoop(
820 proc,
821 [&] (BasicBlock* loop, Value*) -> Value* {
822 Effects effects = Effects::none();
823 effects.writes = HeapRange(666);
824 return loop->appendNew<CCallValue>(
825 proc, Int32, Origin(), effects,
826 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
827 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
828 });
829
830 unsigned callCount = 0;
831 compileAndRun<void>(proc, &callCount);
832 CHECK_EQ(callCount, 100u);
833}
834
835void testLICMFence()
836{
837 Procedure proc;
838 generateLoop(
839 proc,
840 [&] (BasicBlock* loop, Value*) -> Value* {
841 Effects effects = Effects::none();
842 effects.fence = true;
843 return loop->appendNew<CCallValue>(
844 proc, Int32, Origin(), effects,
845 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
846 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
847 });
848
849 unsigned callCount = 0;
850 compileAndRun<void>(proc, &callCount);
851 CHECK_EQ(callCount, 100u);
852}
853
854void testLICMWritesPinned()
855{
856 Procedure proc;
857 generateLoop(
858 proc,
859 [&] (BasicBlock* loop, Value*) -> Value* {
860 Effects effects = Effects::none();
861 effects.writesPinned = true;
862 return loop->appendNew<CCallValue>(
863 proc, Int32, Origin(), effects,
864 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
865 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
866 });
867
868 unsigned callCount = 0;
869 compileAndRun<void>(proc, &callCount);
870 CHECK_EQ(callCount, 100u);
871}
872
873void testLICMControlDependent()
874{
875 Procedure proc;
876 if (proc.optLevel() < 2)
877 return;
878 generateLoop(
879 proc,
880 [&] (BasicBlock* loop, Value*) -> Value* {
881 Effects effects = Effects::none();
882 effects.controlDependent = true;
883 return loop->appendNew<CCallValue>(
884 proc, Int32, Origin(), effects,
885 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
886 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
887 });
888
889 unsigned callCount = 0;
890 compileAndRun<void>(proc, &callCount);
891 CHECK_EQ(callCount, 1u);
892}
893
894void testLICMControlDependentNotBackwardsDominant()
895{
896 Procedure proc;
897 if (proc.optLevel() < 2)
898 return;
899 auto array = makeArrayForLoops();
900 generateLoopNotBackwardsDominant(
901 proc, array,
902 [&] (BasicBlock* loop, Value*) -> Value* {
903 Effects effects = Effects::none();
904 effects.controlDependent = true;
905 return loop->appendNew<CCallValue>(
906 proc, Int32, Origin(), effects,
907 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
908 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
909 });
910
911 unsigned callCount = 0;
912 compileAndRun<void>(proc, &callCount);
913 CHECK_EQ(callCount, 50u);
914}
915
916void testLICMControlDependentSideExits()
917{
918 Procedure proc;
919 generateLoop(
920 proc,
921 [&] (BasicBlock* loop, Value*) -> Value* {
922 Effects effects = Effects::none();
923 effects.exitsSideways = true;
924 loop->appendNew<CCallValue>(
925 proc, Void, Origin(), effects,
926 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
927
928 effects = Effects::none();
929 effects.controlDependent = true;
930 return loop->appendNew<CCallValue>(
931 proc, Int32, Origin(), effects,
932 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
933 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
934 });
935
936 unsigned callCount = 0;
937 compileAndRun<void>(proc, &callCount);
938 CHECK_EQ(callCount, 100u);
939}
940
941void testLICMReadsPinnedWritesPinned()
942{
943 Procedure proc;
944 generateLoop(
945 proc,
946 [&] (BasicBlock* loop, Value*) -> Value* {
947 Effects effects = Effects::none();
948 effects.writesPinned = true;
949 loop->appendNew<CCallValue>(
950 proc, Void, Origin(), effects,
951 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
952
953 effects = Effects::none();
954 effects.readsPinned = true;
955 return loop->appendNew<CCallValue>(
956 proc, Int32, Origin(), effects,
957 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
958 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
959 });
960
961 unsigned callCount = 0;
962 compileAndRun<void>(proc, &callCount);
963 CHECK_EQ(callCount, 100u);
964}
965
966void testLICMReadsWritesDifferentHeaps()
967{
968 Procedure proc;
969 if (proc.optLevel() < 2)
970 return;
971 generateLoop(
972 proc,
973 [&] (BasicBlock* loop, Value*) -> Value* {
974 Effects effects = Effects::none();
975 effects.writes = HeapRange(6436);
976 loop->appendNew<CCallValue>(
977 proc, Void, Origin(), effects,
978 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
979
980 effects = Effects::none();
981 effects.reads = HeapRange(4886);
982 return loop->appendNew<CCallValue>(
983 proc, Int32, Origin(), effects,
984 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
985 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
986 });
987
988 unsigned callCount = 0;
989 compileAndRun<void>(proc, &callCount);
990 CHECK_EQ(callCount, 1u);
991}
992
993void testLICMReadsWritesOverlappingHeaps()
994{
995 Procedure proc;
996 generateLoop(
997 proc,
998 [&] (BasicBlock* loop, Value*) -> Value* {
999 Effects effects = Effects::none();
1000 effects.writes = HeapRange(6436, 74458);
1001 loop->appendNew<CCallValue>(
1002 proc, Void, Origin(), effects,
1003 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(noOpFunction, B3CCallPtrTag)));
1004
1005 effects = Effects::none();
1006 effects.reads = HeapRange(48864, 78239);
1007 return loop->appendNew<CCallValue>(
1008 proc, Int32, Origin(), effects,
1009 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
1010 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1011 });
1012
1013 unsigned callCount = 0;
1014 compileAndRun<void>(proc, &callCount);
1015 CHECK_EQ(callCount, 100u);
1016}
1017
1018void testLICMDefaultCall()
1019{
1020 Procedure proc;
1021 generateLoop(
1022 proc,
1023 [&] (BasicBlock* loop, Value*) -> Value* {
1024 return loop->appendNew<CCallValue>(
1025 proc, Int32, Origin(),
1026 loop->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(oneFunction, B3CCallPtrTag)),
1027 loop->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1028 });
1029
1030 unsigned callCount = 0;
1031 compileAndRun<void>(proc, &callCount);
1032 CHECK_EQ(callCount, 100u);
1033}
1034
1035void testDepend32()
1036{
1037 Procedure proc;
1038 BasicBlock* root = proc.addBlock();
1039 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1040 Value* first = root->appendNew<MemoryValue>(proc, Load, Int32, Origin(), ptr, 0);
1041 Value* second = root->appendNew<MemoryValue>(
1042 proc, Load, Int32, Origin(),
1043 root->appendNew<Value>(
1044 proc, Add, Origin(), ptr,
1045 root->appendNew<Value>(
1046 proc, ZExt32, Origin(),
1047 root->appendNew<Value>(proc, Depend, Origin(), first))),
1048 4);
1049 root->appendNew<Value>(
1050 proc, Return, Origin(),
1051 root->appendNew<Value>(proc, Add, Origin(), first, second));
1052
1053 int32_t values[2];
1054 values[0] = 42;
1055 values[1] = 0xbeef;
1056
1057 auto code = compileProc(proc);
1058 if (isARM64())
1059 checkUsesInstruction(*code, "eor");
1060 else if (isX86()) {
1061 checkDoesNotUseInstruction(*code, "mfence");
1062 checkDoesNotUseInstruction(*code, "lock");
1063 }
1064 CHECK_EQ(invoke<int32_t>(*code, values), 42 + 0xbeef);
1065}
1066
1067void testDepend64()
1068{
1069 Procedure proc;
1070 BasicBlock* root = proc.addBlock();
1071 Value* ptr = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1072 Value* first = root->appendNew<MemoryValue>(proc, Load, Int64, Origin(), ptr, 0);
1073 Value* second = root->appendNew<MemoryValue>(
1074 proc, Load, Int64, Origin(),
1075 root->appendNew<Value>(
1076 proc, Add, Origin(), ptr,
1077 root->appendNew<Value>(proc, Depend, Origin(), first)),
1078 8);
1079 root->appendNew<Value>(
1080 proc, Return, Origin(),
1081 root->appendNew<Value>(proc, Add, Origin(), first, second));
1082
1083 int64_t values[2];
1084 values[0] = 42;
1085 values[1] = 0xbeef;
1086
1087 auto code = compileProc(proc);
1088 if (isARM64())
1089 checkUsesInstruction(*code, "eor");
1090 else if (isX86()) {
1091 checkDoesNotUseInstruction(*code, "mfence");
1092 checkDoesNotUseInstruction(*code, "lock");
1093 }
1094 CHECK_EQ(invoke<int64_t>(*code, values), 42 + 0xbeef);
1095}
1096
1097void testWasmBoundsCheck(unsigned offset)
1098{
1099 Procedure proc;
1100 if (proc.optLevel() < 1)
1101 return;
1102 GPRReg pinned = GPRInfo::argumentGPR1;
1103 proc.pinRegister(pinned);
1104
1105 proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
1106 CHECK_EQ(pinnedGPR, pinned);
1107
1108 // This should always work because a function this simple should never have callee
1109 // saves.
1110 jit.move(CCallHelpers::TrustedImm32(42), GPRInfo::returnValueGPR);
1111 jit.emitFunctionEpilogue();
1112 jit.ret();
1113 });
1114
1115 BasicBlock* root = proc.addBlock();
1116 Value* left = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1117 if (pointerType() != Int32)
1118 left = root->appendNew<Value>(proc, Trunc, Origin(), left);
1119 root->appendNew<WasmBoundsCheckValue>(proc, Origin(), pinned, left, offset);
1120 Value* result = root->appendNew<Const32Value>(proc, Origin(), 0x42);
1121 root->appendNewControlValue(proc, Return, Origin(), result);
1122
1123 auto code = compileProc(proc);
1124 uint32_t bound = 2 + offset;
1125 auto computeResult = [&] (uint32_t input) {
1126 return input + offset < bound ? 0x42 : 42;
1127 };
1128
1129 CHECK_EQ(invoke<int32_t>(*code, 1, bound), computeResult(1));
1130 CHECK_EQ(invoke<int32_t>(*code, 3, bound), computeResult(3));
1131 CHECK_EQ(invoke<int32_t>(*code, 2, bound), computeResult(2));
1132}
1133
1134void testWasmAddress()
1135{
1136 Procedure proc;
1137 GPRReg pinnedGPR = GPRInfo::argumentGPR2;
1138 proc.pinRegister(pinnedGPR);
1139
1140 unsigned loopCount = 100;
1141 Vector<unsigned> values(loopCount);
1142 unsigned numToStore = 42;
1143
1144 BasicBlock* root = proc.addBlock();
1145 BasicBlock* header = proc.addBlock();
1146 BasicBlock* body = proc.addBlock();
1147 BasicBlock* continuation = proc.addBlock();
1148
1149 // Root
1150 Value* loopCountValue = root->appendNew<Value>(proc, Trunc, Origin(), root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1151 Value* valueToStore = root->appendNew<Value>(proc, Trunc, Origin(), root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
1152 UpsilonValue* beginUpsilon = root->appendNew<UpsilonValue>(proc, Origin(), root->appendNew<Const32Value>(proc, Origin(), 0));
1153 root->appendNewControlValue(proc, Jump, Origin(), header);
1154
1155 // Header
1156 Value* indexPhi = header->appendNew<Value>(proc, Phi, Int32, Origin());
1157 header->appendNewControlValue(proc, Branch, Origin(),
1158 header->appendNew<Value>(proc, Below, Origin(), indexPhi, loopCountValue),
1159 body, continuation);
1160
1161 // Body
1162 Value* pointer = body->appendNew<Value>(proc, Mul, Origin(), indexPhi,
1163 body->appendNew<Const32Value>(proc, Origin(), sizeof(unsigned)));
1164 pointer = body->appendNew<Value>(proc, ZExt32, Origin(), pointer);
1165 body->appendNew<MemoryValue>(proc, Store, Origin(), valueToStore,
1166 body->appendNew<WasmAddressValue>(proc, Origin(), pointer, pinnedGPR), 0);
1167 UpsilonValue* incUpsilon = body->appendNew<UpsilonValue>(proc, Origin(),
1168 body->appendNew<Value>(proc, Add, Origin(), indexPhi,
1169 body->appendNew<Const32Value>(proc, Origin(), 1)));
1170 body->appendNewControlValue(proc, Jump, Origin(), header);
1171
1172 // Continuation
1173 continuation->appendNewControlValue(proc, Return, Origin());
1174
1175 beginUpsilon->setPhi(indexPhi);
1176 incUpsilon->setPhi(indexPhi);
1177
1178
1179 auto code = compileProc(proc);
1180 invoke<void>(*code, loopCount, numToStore, values.data());
1181 for (unsigned value : values)
1182 CHECK_EQ(numToStore, value);
1183}
1184
1185void testFastTLSLoad()
1186{
1187#if ENABLE(FAST_TLS_JIT)
1188 _pthread_setspecific_direct(WTF_TESTING_KEY, bitwise_cast<void*>(static_cast<uintptr_t>(0xbeef)));
1189
1190 Procedure proc;
1191 BasicBlock* root = proc.addBlock();
1192
1193 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, pointerType(), Origin());
1194 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1195 patchpoint->setGenerator(
1196 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1197 AllowMacroScratchRegisterUsage allowScratch(jit);
1198 jit.loadFromTLSPtr(fastTLSOffsetForKey(WTF_TESTING_KEY), params[0].gpr());
1199 });
1200
1201 root->appendNew<Value>(proc, Return, Origin(), patchpoint);
1202
1203 CHECK_EQ(compileAndRun<uintptr_t>(proc), static_cast<uintptr_t>(0xbeef));
1204#endif
1205}
1206
1207void testFastTLSStore()
1208{
1209#if ENABLE(FAST_TLS_JIT)
1210 Procedure proc;
1211 BasicBlock* root = proc.addBlock();
1212
1213 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
1214 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1215 patchpoint->numGPScratchRegisters = 1;
1216 patchpoint->setGenerator(
1217 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1218 AllowMacroScratchRegisterUsage allowScratch(jit);
1219 GPRReg scratch = params.gpScratch(0);
1220 jit.move(CCallHelpers::TrustedImm32(0xdead), scratch);
1221 jit.storeToTLSPtr(scratch, fastTLSOffsetForKey(WTF_TESTING_KEY));
1222 });
1223
1224 root->appendNewControlValue(proc, Return, Origin());
1225
1226 compileAndRun<void>(proc);
1227 CHECK_EQ(bitwise_cast<uintptr_t>(_pthread_getspecific_direct(WTF_TESTING_KEY)), static_cast<uintptr_t>(0xdead));
1228#endif
1229}
1230
1231static NEVER_INLINE bool doubleEq(double a, double b) { return a == b; }
1232static NEVER_INLINE bool doubleNeq(double a, double b) { return a != b; }
1233static NEVER_INLINE bool doubleGt(double a, double b) { return a > b; }
1234static NEVER_INLINE bool doubleGte(double a, double b) { return a >= b; }
1235static NEVER_INLINE bool doubleLt(double a, double b) { return a < b; }
1236static NEVER_INLINE bool doubleLte(double a, double b) { return a <= b; }
1237
1238void testDoubleLiteralComparison(double a, double b)
1239{
1240 using Test = std::tuple<B3::Opcode, bool (*)(double, double)>;
1241 StdList<Test> tests = {
1242 Test { NotEqual, doubleNeq },
1243 Test { Equal, doubleEq },
1244 Test { EqualOrUnordered, doubleEq },
1245 Test { GreaterThan, doubleGt },
1246 Test { GreaterEqual, doubleGte },
1247 Test { LessThan, doubleLt },
1248 Test { LessEqual, doubleLte },
1249 };
1250
1251 for (const Test& test : tests) {
1252 Procedure proc;
1253 BasicBlock* root = proc.addBlock();
1254 Value* valueA = root->appendNew<ConstDoubleValue>(proc, Origin(), a);
1255 Value* valueB = root->appendNew<ConstDoubleValue>(proc, Origin(), b);
1256
1257 // This is here just to make reduceDoubleToFloat do things.
1258 Value* valueC = root->appendNew<ConstDoubleValue>(proc, Origin(), 0.0);
1259 Value* valueAsFloat = root->appendNew<Value>(proc, DoubleToFloat, Origin(), valueC);
1260
1261 root->appendNewControlValue(
1262 proc, Return, Origin(),
1263 root->appendNew<Value>(proc, BitAnd, Origin(),
1264 root->appendNew<Value>(proc, std::get<0>(test), Origin(), valueA, valueB),
1265 root->appendNew<Value>(proc, Equal, Origin(), valueAsFloat, valueAsFloat)));
1266
1267 CHECK(!!compileAndRun<int32_t>(proc) == std::get<1>(test)(a, b));
1268 }
1269}
1270
1271void testFloatEqualOrUnorderedFolding()
1272{
1273 for (auto& first : floatingPointOperands<float>()) {
1274 for (auto& second : floatingPointOperands<float>()) {
1275 float a = first.value;
1276 float b = second.value;
1277 bool expectedResult = (a == b) || std::isunordered(a, b);
1278 Procedure proc;
1279 BasicBlock* root = proc.addBlock();
1280 Value* constA = root->appendNew<ConstFloatValue>(proc, Origin(), a);
1281 Value* constB = root->appendNew<ConstFloatValue>(proc, Origin(), b);
1282
1283 root->appendNewControlValue(proc, Return, Origin(),
1284 root->appendNew<Value>(
1285 proc, EqualOrUnordered, Origin(),
1286 constA,
1287 constB));
1288 CHECK(!!compileAndRun<int32_t>(proc) == expectedResult);
1289 }
1290 }
1291}
1292
1293void testFloatEqualOrUnorderedFoldingNaN()
1294{
1295 StdList<float> nans = {
1296 bitwise_cast<float>(0xfffffffd),
1297 bitwise_cast<float>(0xfffffffe),
1298 bitwise_cast<float>(0xfffffff0),
1299 static_cast<float>(PNaN),
1300 };
1301
1302 unsigned i = 0;
1303 for (float nan : nans) {
1304 RELEASE_ASSERT(std::isnan(nan));
1305 Procedure proc;
1306 BasicBlock* root = proc.addBlock();
1307 Value* a = root->appendNew<ConstFloatValue>(proc, Origin(), nan);
1308 Value* b = root->appendNew<Value>(proc, DoubleToFloat, Origin(),
1309 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0));
1310
1311 if (i % 2)
1312 std::swap(a, b);
1313 ++i;
1314 root->appendNewControlValue(proc, Return, Origin(),
1315 root->appendNew<Value>(proc, EqualOrUnordered, Origin(), a, b));
1316 CHECK(!!compileAndRun<int32_t>(proc, static_cast<double>(1.0)));
1317 }
1318}
1319
1320void testFloatEqualOrUnorderedDontFold()
1321{
1322 for (auto& first : floatingPointOperands<float>()) {
1323 float a = first.value;
1324 Procedure proc;
1325 BasicBlock* root = proc.addBlock();
1326 Value* constA = root->appendNew<ConstFloatValue>(proc, Origin(), a);
1327 Value* b = root->appendNew<Value>(proc, DoubleToFloat, Origin(),
1328 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0));
1329 root->appendNewControlValue(proc, Return, Origin(),
1330 root->appendNew<Value>(
1331 proc, EqualOrUnordered, Origin(), constA, b));
1332
1333 auto code = compileProc(proc);
1334
1335 for (auto& second : floatingPointOperands<float>()) {
1336 float b = second.value;
1337 bool expectedResult = (a == b) || std::isunordered(a, b);
1338 CHECK(!!invoke<int32_t>(*code, static_cast<double>(b)) == expectedResult);
1339 }
1340 }
1341}
1342
1343static void functionNineArgs(int32_t, void*, void*, void*, void*, void*, void*, void*, void*) { }
1344
1345void testShuffleDoesntTrashCalleeSaves()
1346{
1347 Procedure proc;
1348
1349 BasicBlock* root = proc.addBlock();
1350 BasicBlock* likely = proc.addBlock();
1351 BasicBlock* unlikely = proc.addBlock();
1352
1353 RegisterSet regs = RegisterSet::allGPRs();
1354 regs.exclude(RegisterSet::stackRegisters());
1355 regs.exclude(RegisterSet::reservedHardwareRegisters());
1356 regs.exclude(RegisterSet::calleeSaveRegisters());
1357 regs.exclude(RegisterSet::argumentGPRS());
1358
1359 unsigned i = 0;
1360 Vector<Value*> patches;
1361 for (Reg reg : regs) {
1362 ++i;
1363 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
1364 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1365 RELEASE_ASSERT(reg.isGPR());
1366 patchpoint->resultConstraints = { ValueRep::reg(reg.gpr()) };
1367 patchpoint->setGenerator(
1368 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1369 AllowMacroScratchRegisterUsage allowScratch(jit);
1370 jit.move(CCallHelpers::TrustedImm32(i), params[0].gpr());
1371 });
1372 patches.append(patchpoint);
1373 }
1374
1375 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(0 % GPRInfo::numberOfArgumentRegisters));
1376 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(1 % GPRInfo::numberOfArgumentRegisters));
1377 Value* arg3 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(2 % GPRInfo::numberOfArgumentRegisters));
1378 Value* arg4 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(3 % GPRInfo::numberOfArgumentRegisters));
1379 Value* arg5 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(4 % GPRInfo::numberOfArgumentRegisters));
1380 Value* arg6 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(5 % GPRInfo::numberOfArgumentRegisters));
1381 Value* arg7 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(6 % GPRInfo::numberOfArgumentRegisters));
1382 Value* arg8 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::toArgumentRegister(7 % GPRInfo::numberOfArgumentRegisters));
1383
1384 PatchpointValue* ptr = root->appendNew<PatchpointValue>(proc, Int64, Origin());
1385 ptr->clobber(RegisterSet::macroScratchRegisters());
1386 ptr->resultConstraints = { ValueRep::reg(GPRInfo::regCS0) };
1387 ptr->appendSomeRegister(arg1);
1388 ptr->setGenerator(
1389 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1390 AllowMacroScratchRegisterUsage allowScratch(jit);
1391 jit.move(params[1].gpr(), params[0].gpr());
1392 });
1393
1394 Value* condition = root->appendNew<Value>(
1395 proc, Equal, Origin(),
1396 ptr,
1397 root->appendNew<Const64Value>(proc, Origin(), 0));
1398
1399 root->appendNewControlValue(
1400 proc, Branch, Origin(),
1401 condition,
1402 FrequentedBlock(likely, FrequencyClass::Normal), FrequentedBlock(unlikely, FrequencyClass::Rare));
1403
1404 // Never executes.
1405 Value* const42 = likely->appendNew<Const32Value>(proc, Origin(), 42);
1406 likely->appendNewControlValue(proc, Return, Origin(), const42);
1407
1408 // Always executes.
1409 Value* constNumber = unlikely->appendNew<Const32Value>(proc, Origin(), 0x1);
1410
1411 unlikely->appendNew<CCallValue>(
1412 proc, Void, Origin(),
1413 unlikely->appendNew<ConstPtrValue>(proc, Origin(), tagCFunctionPtr<void*>(functionNineArgs, B3CCallPtrTag)),
1414 constNumber, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
1415
1416 PatchpointValue* voidPatch = unlikely->appendNew<PatchpointValue>(proc, Void, Origin());
1417 voidPatch->clobber(RegisterSet::macroScratchRegisters());
1418 for (Value* v : patches)
1419 voidPatch->appendSomeRegister(v);
1420 voidPatch->appendSomeRegister(arg1);
1421 voidPatch->appendSomeRegister(arg2);
1422 voidPatch->appendSomeRegister(arg3);
1423 voidPatch->appendSomeRegister(arg4);
1424 voidPatch->appendSomeRegister(arg5);
1425 voidPatch->appendSomeRegister(arg6);
1426 voidPatch->setGenerator([=] (CCallHelpers&, const StackmapGenerationParams&) { });
1427
1428 unlikely->appendNewControlValue(proc, Return, Origin(),
1429 unlikely->appendNew<MemoryValue>(proc, Load, Int32, Origin(), ptr));
1430
1431 int32_t* inputPtr = static_cast<int32_t*>(fastMalloc(sizeof(int32_t)));
1432 *inputPtr = 48;
1433 CHECK(compileAndRun<int32_t>(proc, inputPtr) == 48);
1434 fastFree(inputPtr);
1435}
1436
1437void testDemotePatchpointTerminal()
1438{
1439 Procedure proc;
1440
1441 BasicBlock* root = proc.addBlock();
1442 BasicBlock* done = proc.addBlock();
1443
1444 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
1445 patchpoint->effects.terminal = true;
1446 root->setSuccessors(done);
1447
1448 done->appendNew<Value>(proc, Return, Origin(), patchpoint);
1449
1450 proc.resetReachability();
1451 breakCriticalEdges(proc);
1452 IndexSet<Value*> valuesToDemote;
1453 valuesToDemote.add(patchpoint);
1454 demoteValues(proc, valuesToDemote);
1455 validate(proc);
1456}
1457
1458void testReportUsedRegistersLateUseFollowedByEarlyDefDoesNotMarkUseAsDead()
1459{
1460 Procedure proc;
1461 if (proc.optLevel() < 2)
1462 return;
1463 BasicBlock* root = proc.addBlock();
1464
1465 RegisterSet allRegs = RegisterSet::allGPRs();
1466 allRegs.exclude(RegisterSet::stackRegisters());
1467 allRegs.exclude(RegisterSet::reservedHardwareRegisters());
1468
1469 {
1470 // Make every reg 42 (just needs to be a value other than 10).
1471 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
1472 Value* const42 = root->appendNew<Const32Value>(proc, Origin(), 42);
1473 for (Reg reg : allRegs)
1474 patchpoint->append(const42, ValueRep::reg(reg));
1475 patchpoint->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
1476 }
1477
1478 {
1479 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
1480 Value* const10 = root->appendNew<Const32Value>(proc, Origin(), 10);
1481 for (Reg reg : allRegs)
1482 patchpoint->append(const10, ValueRep::lateReg(reg));
1483 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams&) {
1484 for (Reg reg : allRegs) {
1485 auto done = jit.branch32(CCallHelpers::Equal, reg.gpr(), CCallHelpers::TrustedImm32(10));
1486 jit.breakpoint();
1487 done.link(&jit);
1488 }
1489 });
1490 }
1491
1492 {
1493 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
1494 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
1495 patchpoint->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams& params) {
1496 RELEASE_ASSERT(allRegs.contains(params[0].gpr()));
1497 });
1498 }
1499
1500 root->appendNewControlValue(proc, Return, Origin());
1501
1502 compileAndRun<void>(proc);
1503}
1504
1505void testInfiniteLoopDoesntCauseBadHoisting()
1506{
1507 Procedure proc;
1508 if (proc.optLevel() < 2)
1509 return;
1510 BasicBlock* root = proc.addBlock();
1511 BasicBlock* header = proc.addBlock();
1512 BasicBlock* loadBlock = proc.addBlock();
1513 BasicBlock* postLoadBlock = proc.addBlock();
1514
1515 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1516 root->appendNewControlValue(proc, Jump, Origin(), header);
1517
1518 header->appendNewControlValue(
1519 proc, Branch, Origin(),
1520 header->appendNew<Value>(proc, Equal, Origin(),
1521 arg,
1522 header->appendNew<Const64Value>(proc, Origin(), 10)), header, loadBlock);
1523
1524 PatchpointValue* patchpoint = loadBlock->appendNew<PatchpointValue>(proc, Void, Origin());
1525 patchpoint->effects = Effects::none();
1526 patchpoint->effects.writesLocalState = true; // Don't DCE this.
1527 patchpoint->setGenerator(
1528 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
1529 // This works because we don't have callee saves.
1530 jit.emitFunctionEpilogue();
1531 jit.ret();
1532 });
1533
1534 Value* badLoad = loadBlock->appendNew<MemoryValue>(proc, Load, Int64, Origin(), arg, 0);
1535
1536 loadBlock->appendNewControlValue(
1537 proc, Branch, Origin(),
1538 loadBlock->appendNew<Value>(proc, Equal, Origin(),
1539 badLoad,
1540 loadBlock->appendNew<Const64Value>(proc, Origin(), 45)), header, postLoadBlock);
1541
1542 postLoadBlock->appendNewControlValue(proc, Return, Origin(), badLoad);
1543
1544 // The patchpoint early ret() works because we don't have callee saves.
1545 auto code = compileProc(proc);
1546 RELEASE_ASSERT(!proc.calleeSaveRegisterAtOffsetList().size());
1547 invoke<void>(*code, static_cast<uint64_t>(55)); // Shouldn't crash dereferncing 55.
1548}
1549
1550static void testSimpleTuplePair(unsigned first, int64_t second)
1551{
1552 Procedure proc;
1553 BasicBlock* root = proc.addBlock();
1554
1555 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, proc.addTuple({ Int32, Int64 }), Origin());
1556 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1557 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::SomeRegister };
1558 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1559 AllowMacroScratchRegisterUsage allowScratch(jit);
1560 jit.move(CCallHelpers::TrustedImm32(first), params[0].gpr());
1561 jit.move(CCallHelpers::TrustedImm64(second), params[1].gpr());
1562 });
1563 Value* i32 = root->appendNew<Value>(proc, ZExt32, Origin(),
1564 root->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 0));
1565 Value* i64 = root->appendNew<ExtractValue>(proc, Origin(), Int64, patchpoint, 1);
1566 root->appendNew<Value>(proc, Return, Origin(), root->appendNew<Value>(proc, Add, Origin(), i32, i64));
1567
1568 CHECK_EQ(compileAndRun<int64_t>(proc), first + second);
1569}
1570
1571static void testSimpleTuplePairUnused(unsigned first, int64_t second)
1572{
1573 Procedure proc;
1574 BasicBlock* root = proc.addBlock();
1575
1576 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, proc.addTuple({ Int32, Int64, Double }), Origin());
1577 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1578 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::SomeRegister, ValueRep::SomeRegister };
1579 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1580 AllowMacroScratchRegisterUsage allowScratch(jit);
1581 jit.move(CCallHelpers::TrustedImm32(first), params[0].gpr());
1582 jit.move(CCallHelpers::TrustedImm64(second), params[1].gpr());
1583 jit.moveDouble(CCallHelpers::Imm64(bitwise_cast<uint64_t>(0.0)), params[2].fpr());
1584 });
1585 Value* i32 = root->appendNew<Value>(proc, ZExt32, Origin(),
1586 root->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 0));
1587 Value* i64 = root->appendNew<ExtractValue>(proc, Origin(), Int64, patchpoint, 1);
1588 root->appendNew<Value>(proc, Return, Origin(), root->appendNew<Value>(proc, Add, Origin(), i32, i64));
1589
1590 CHECK_EQ(compileAndRun<int64_t>(proc), first + second);
1591}
1592
1593static void testSimpleTuplePairStack(unsigned first, int64_t second)
1594{
1595 Procedure proc;
1596 BasicBlock* root = proc.addBlock();
1597
1598 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, proc.addTuple({ Int32, Int64 }), Origin());
1599 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1600 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::stackArgument(0) };
1601 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1602 AllowMacroScratchRegisterUsage allowScratch(jit);
1603 jit.move(CCallHelpers::TrustedImm32(first), params[0].gpr());
1604 jit.store64(CCallHelpers::TrustedImm64(second), CCallHelpers::Address(CCallHelpers::framePointerRegister, params[1].offsetFromFP()));
1605 });
1606 Value* i32 = root->appendNew<Value>(proc, ZExt32, Origin(),
1607 root->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 0));
1608 Value* i64 = root->appendNew<ExtractValue>(proc, Origin(), Int64, patchpoint, 1);
1609 root->appendNew<Value>(proc, Return, Origin(), root->appendNew<Value>(proc, Add, Origin(), i32, i64));
1610
1611 CHECK_EQ(compileAndRun<int64_t>(proc), first + second);
1612}
1613
1614template<bool shouldFixSSA>
1615static void tailDupedTuplePair(unsigned first, double second)
1616{
1617 Procedure proc;
1618 BasicBlock* root = proc.addBlock();
1619 BasicBlock* truthy = proc.addBlock();
1620 BasicBlock* falsey = proc.addBlock();
1621
1622 Type tupleType = proc.addTuple({ Int32, Double });
1623 Variable* var = proc.addVariable(tupleType);
1624
1625 Value* test = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1626 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, tupleType, Origin());
1627 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1628 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::stackArgument(0) };
1629 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1630 AllowMacroScratchRegisterUsage allowScratch(jit);
1631 jit.move(CCallHelpers::TrustedImm32(first), params[0].gpr());
1632 jit.store64(CCallHelpers::TrustedImm64(bitwise_cast<uint64_t>(second)), CCallHelpers::Address(CCallHelpers::framePointerRegister, params[1].offsetFromFP()));
1633 });
1634 root->appendNew<VariableValue>(proc, Set, Origin(), var, patchpoint);
1635 root->appendNewControlValue(proc, Branch, Origin(), test, FrequentedBlock(truthy), FrequentedBlock(falsey));
1636
1637 auto addDup = [&] (BasicBlock* block) {
1638 Value* tuple = block->appendNew<VariableValue>(proc, B3::Get, Origin(), var);
1639 Value* i32 = block->appendNew<Value>(proc, ZExt32, Origin(),
1640 block->appendNew<ExtractValue>(proc, Origin(), Int32, tuple, 0));
1641 i32 = block->appendNew<Value>(proc, IToD, Origin(), i32);
1642 Value* f64 = block->appendNew<ExtractValue>(proc, Origin(), Double, tuple, 1);
1643 block->appendNew<Value>(proc, Return, Origin(), block->appendNew<Value>(proc, Add, Origin(), i32, f64));
1644 };
1645
1646 addDup(truthy);
1647 addDup(falsey);
1648
1649 proc.resetReachability();
1650 if (shouldFixSSA)
1651 fixSSA(proc);
1652 CHECK_EQ(compileAndRun<double>(proc, first), first + second);
1653}
1654
1655template<bool shouldFixSSA>
1656static void tuplePairVariableLoop(unsigned first, uint64_t second)
1657{
1658 Procedure proc;
1659 BasicBlock* root = proc.addBlock();
1660 BasicBlock* body = proc.addBlock();
1661 BasicBlock* exit = proc.addBlock();
1662
1663 Type tupleType = proc.addTuple({ Int32, Int64 });
1664 Variable* var = proc.addVariable(tupleType);
1665
1666 {
1667 Value* first = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1668 Value* second = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
1669 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, tupleType, Origin());
1670 patchpoint->append({ first, ValueRep::SomeRegister });
1671 patchpoint->append({ second, ValueRep::SomeRegister });
1672 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister, ValueRep::SomeEarlyRegister };
1673 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1674 jit.move(params[2].gpr(), params[0].gpr());
1675 jit.move(params[3].gpr(), params[1].gpr());
1676 });
1677 root->appendNew<VariableValue>(proc, Set, Origin(), var, patchpoint);
1678 root->appendNewControlValue(proc, Jump, Origin(), body);
1679 }
1680
1681 {
1682 Value* tuple = body->appendNew<VariableValue>(proc, B3::Get, Origin(), var);
1683 Value* first = body->appendNew<ExtractValue>(proc, Origin(), Int32, tuple, 0);
1684 Value* second = body->appendNew<ExtractValue>(proc, Origin(), Int64, tuple, 1);
1685 PatchpointValue* patchpoint = body->appendNew<PatchpointValue>(proc, tupleType, Origin());
1686 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1687 patchpoint->append({ first, ValueRep::SomeRegister });
1688 patchpoint->append({ second, ValueRep::SomeRegister });
1689 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister, ValueRep::stackArgument(0) };
1690 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1691 AllowMacroScratchRegisterUsage allowScratch(jit);
1692 CHECK(params[3].gpr() != params[0].gpr());
1693 CHECK(params[2].gpr() != params[0].gpr());
1694 jit.add64(CCallHelpers::TrustedImm32(1), params[3].gpr(), params[0].gpr());
1695 jit.store64(params[0].gpr(), CCallHelpers::Address(CCallHelpers::framePointerRegister, params[1].offsetFromFP()));
1696
1697 jit.move(params[2].gpr(), params[0].gpr());
1698 jit.urshift32(CCallHelpers::TrustedImm32(1), params[0].gpr());
1699 });
1700 body->appendNew<VariableValue>(proc, Set, Origin(), var, patchpoint);
1701 Value* condition = body->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 0);
1702 body->appendNewControlValue(proc, Branch, Origin(), condition, FrequentedBlock(body), FrequentedBlock(exit));
1703 }
1704
1705 {
1706 Value* tuple = exit->appendNew<VariableValue>(proc, B3::Get, Origin(), var);
1707 Value* second = exit->appendNew<ExtractValue>(proc, Origin(), Int64, tuple, 1);
1708 exit->appendNew<Value>(proc, Return, Origin(), second);
1709 }
1710
1711 proc.resetReachability();
1712 validate(proc);
1713 if (shouldFixSSA)
1714 fixSSA(proc);
1715 CHECK_EQ(compileAndRun<uint64_t>(proc, first, second), second + (first ? getMSBSet(first) : first) + 1);
1716}
1717
1718template<bool shouldFixSSA>
1719static void tupleNestedLoop(int32_t first, double second)
1720{
1721 Procedure proc;
1722 BasicBlock* root = proc.addBlock();
1723 BasicBlock* outerLoop = proc.addBlock();
1724 BasicBlock* innerLoop = proc.addBlock();
1725 BasicBlock* outerContinuation = proc.addBlock();
1726
1727 Type tupleType = proc.addTuple({ Int32, Double, Int32 });
1728 Variable* varOuter = proc.addVariable(tupleType);
1729 Variable* varInner = proc.addVariable(tupleType);
1730 Variable* tookInner = proc.addVariable(Int32);
1731
1732 {
1733 Value* first = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1734 Value* second = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
1735 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, tupleType, Origin());
1736 patchpoint->append({ first, ValueRep::SomeRegisterWithClobber });
1737 patchpoint->append({ second, ValueRep::SomeRegisterWithClobber });
1738 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::SomeRegister, ValueRep::SomeEarlyRegister };
1739 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1740 jit.move(params[3].gpr(), params[0].gpr());
1741 jit.move(params[0].gpr(), params[2].gpr());
1742 jit.move(params[4].fpr(), params[1].fpr());
1743 });
1744 root->appendNew<VariableValue>(proc, Set, Origin(), varOuter, patchpoint);
1745 root->appendNew<VariableValue>(proc, Set, Origin(), tookInner, root->appendIntConstant(proc, Origin(), Int32, 0));
1746 root->appendNewControlValue(proc, Jump, Origin(), outerLoop);
1747 }
1748
1749 {
1750 Value* tuple = outerLoop->appendNew<VariableValue>(proc, B3::Get, Origin(), varOuter);
1751 Value* first = outerLoop->appendNew<ExtractValue>(proc, Origin(), Int32, tuple, 0);
1752 Value* second = outerLoop->appendNew<ExtractValue>(proc, Origin(), Double, tuple, 1);
1753 Value* third = outerLoop->appendNew<VariableValue>(proc, B3::Get, Origin(), tookInner);
1754 PatchpointValue* patchpoint = outerLoop->appendNew<PatchpointValue>(proc, tupleType, Origin());
1755 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1756 patchpoint->append({ first, ValueRep::SomeRegisterWithClobber });
1757 patchpoint->append({ second, ValueRep::SomeRegisterWithClobber });
1758 patchpoint->append({ third, ValueRep::SomeRegisterWithClobber });
1759 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::SomeRegister, ValueRep::SomeRegister };
1760 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1761 AllowMacroScratchRegisterUsage allowScratch(jit);
1762 jit.move(params[3].gpr(), params[0].gpr());
1763 jit.moveConditionally32(CCallHelpers::Equal, params[5].gpr(), CCallHelpers::TrustedImm32(0), params[0].gpr(), params[5].gpr(), params[2].gpr());
1764 jit.move(params[4].fpr(), params[1].fpr());
1765 });
1766 outerLoop->appendNew<VariableValue>(proc, Set, Origin(), varOuter, patchpoint);
1767 outerLoop->appendNew<VariableValue>(proc, Set, Origin(), varInner, patchpoint);
1768 Value* condition = outerLoop->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 2);
1769 outerLoop->appendNewControlValue(proc, Branch, Origin(), condition, FrequentedBlock(outerContinuation), FrequentedBlock(innerLoop));
1770 }
1771
1772 {
1773 Value* tuple = innerLoop->appendNew<VariableValue>(proc, B3::Get, Origin(), varInner);
1774 Value* first = innerLoop->appendNew<ExtractValue>(proc, Origin(), Int32, tuple, 0);
1775 Value* second = innerLoop->appendNew<ExtractValue>(proc, Origin(), Double, tuple, 1);
1776 PatchpointValue* patchpoint = innerLoop->appendNew<PatchpointValue>(proc, tupleType, Origin());
1777 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1778 patchpoint->append({ first, ValueRep::SomeRegisterWithClobber });
1779 patchpoint->append({ second, ValueRep::SomeRegisterWithClobber });
1780 patchpoint->resultConstraints = { ValueRep::SomeRegister, ValueRep::SomeRegister, ValueRep::SomeEarlyRegister };
1781 patchpoint->setGenerator([&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1782 AllowMacroScratchRegisterUsage allowScratch(jit);
1783 jit.move(params[3].gpr(), params[0].gpr());
1784 jit.move(CCallHelpers::TrustedImm32(0), params[2].gpr());
1785 jit.move(params[4].fpr(), params[1].fpr());
1786 });
1787 innerLoop->appendNew<VariableValue>(proc, Set, Origin(), varOuter, patchpoint);
1788 innerLoop->appendNew<VariableValue>(proc, Set, Origin(), varInner, patchpoint);
1789 Value* condition = innerLoop->appendNew<ExtractValue>(proc, Origin(), Int32, patchpoint, 2);
1790 innerLoop->appendNew<VariableValue>(proc, Set, Origin(), tookInner, innerLoop->appendIntConstant(proc, Origin(), Int32, 1));
1791 innerLoop->appendNewControlValue(proc, Branch, Origin(), condition, FrequentedBlock(innerLoop), FrequentedBlock(outerLoop));
1792 }
1793
1794 {
1795 Value* tuple = outerContinuation->appendNew<VariableValue>(proc, B3::Get, Origin(), varInner);
1796 Value* first = outerContinuation->appendNew<ExtractValue>(proc, Origin(), Int32, tuple, 0);
1797 Value* second = outerContinuation->appendNew<ExtractValue>(proc, Origin(), Double, tuple, 1);
1798 Value* result = outerContinuation->appendNew<Value>(proc, Add, Origin(), second, outerContinuation->appendNew<Value>(proc, IToD, Origin(), first));
1799 outerContinuation->appendNewControlValue(proc, Return, Origin(), result);
1800 }
1801
1802 proc.resetReachability();
1803 validate(proc);
1804 if (shouldFixSSA)
1805 fixSSA(proc);
1806 CHECK_EQ(compileAndRun<double>(proc, first, second), first + second);
1807}
1808
1809void addTupleTests(const char* filter, Deque<RefPtr<SharedTask<void()>>>& tasks)
1810{
1811 RUN_BINARY(testSimpleTuplePair, int32Operands(), int64Operands());
1812 RUN_BINARY(testSimpleTuplePairUnused, int32Operands(), int64Operands());
1813 RUN_BINARY(testSimpleTuplePairStack, int32Operands(), int64Operands());
1814 // use int64 as second argument because checking for NaN is annoying and doesn't really matter for this test.
1815 RUN_BINARY(tailDupedTuplePair<true>, int32Operands(), int64Operands());
1816 RUN_BINARY(tailDupedTuplePair<false>, int32Operands(), int64Operands());
1817 RUN_BINARY(tuplePairVariableLoop<true>, int32Operands(), int64Operands());
1818 RUN_BINARY(tuplePairVariableLoop<false>, int32Operands(), int64Operands());
1819 RUN_BINARY(tupleNestedLoop<true>, int32Operands(), int64Operands());
1820 RUN_BINARY(tupleNestedLoop<false>, int32Operands(), int64Operands());
1821}
1822
1823#endif // ENABLE(B3_JIT)
1824