1/*
2 * Copyright (C) 2015-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "testb3.h"
28
29#if ENABLE(B3_JIT)
30
31void testBitAndSExt32(int32_t value, int64_t mask)
32{
33 Procedure proc;
34 BasicBlock* root = proc.addBlock();
35 root->appendNewControlValue(
36 proc, Return, Origin(),
37 root->appendNew<Value>(
38 proc, BitAnd, Origin(),
39 root->appendNew<Value>(
40 proc, SExt32, Origin(),
41 root->appendNew<Value>(
42 proc, Trunc, Origin(),
43 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0))),
44 root->appendNew<Const64Value>(proc, Origin(), mask)));
45
46 CHECK(compileAndRun<int64_t>(proc, value) == (static_cast<int64_t>(value) & mask));
47}
48
49void testBasicSelect()
50{
51 Procedure proc;
52 BasicBlock* root = proc.addBlock();
53 root->appendNewControlValue(
54 proc, Return, Origin(),
55 root->appendNew<Value>(
56 proc, Select, Origin(),
57 root->appendNew<Value>(
58 proc, Equal, Origin(),
59 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
60 root->appendNew<ConstPtrValue>(proc, Origin(), 42)),
61 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
62 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
63
64 auto code = compileProc(proc);
65 CHECK(invoke<intptr_t>(*code, 42, 1, 2) == 1);
66 CHECK(invoke<intptr_t>(*code, 42, 642462, 32533) == 642462);
67 CHECK(invoke<intptr_t>(*code, 43, 1, 2) == 2);
68 CHECK(invoke<intptr_t>(*code, 43, 642462, 32533) == 32533);
69}
70
71void testSelectTest()
72{
73 Procedure proc;
74 BasicBlock* root = proc.addBlock();
75 root->appendNewControlValue(
76 proc, Return, Origin(),
77 root->appendNew<Value>(
78 proc, Select, Origin(),
79 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
80 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
81 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
82
83 auto code = compileProc(proc);
84 CHECK(invoke<intptr_t>(*code, 42, 1, 2) == 1);
85 CHECK(invoke<intptr_t>(*code, 42, 642462, 32533) == 642462);
86 CHECK(invoke<intptr_t>(*code, 0, 1, 2) == 2);
87 CHECK(invoke<intptr_t>(*code, 0, 642462, 32533) == 32533);
88}
89
90void testSelectCompareDouble()
91{
92 Procedure proc;
93 BasicBlock* root = proc.addBlock();
94 root->appendNewControlValue(
95 proc, Return, Origin(),
96 root->appendNew<Value>(
97 proc, Select, Origin(),
98 root->appendNew<Value>(
99 proc, LessThan, Origin(),
100 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0),
101 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1)),
102 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
103 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
104
105 auto code = compileProc(proc);
106 CHECK(invoke<intptr_t>(*code, -1.0, 1.0, 1, 2) == 1);
107 CHECK(invoke<intptr_t>(*code, 42.5, 42.51, 642462, 32533) == 642462);
108 CHECK(invoke<intptr_t>(*code, PNaN, 0.0, 1, 2) == 2);
109 CHECK(invoke<intptr_t>(*code, 42.51, 42.5, 642462, 32533) == 32533);
110 CHECK(invoke<intptr_t>(*code, 42.52, 42.52, 524978245, 352) == 352);
111}
112
113template<B3::Opcode opcode>
114void testSelectCompareFloat(float a, float b, bool (*operation)(float, float))
115{
116 Procedure proc;
117 BasicBlock* root = proc.addBlock();
118 Value* argument1int32 = root->appendNew<Value>(proc, Trunc, Origin(),
119 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
120 Value* argument2int32 = root->appendNew<Value>(proc, Trunc, Origin(),
121 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
122 Value* floatValue1 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument1int32);
123 Value* floatValue2 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument2int32);
124
125 root->appendNewControlValue(
126 proc, Return, Origin(),
127 root->appendNew<Value>(
128 proc, Select, Origin(),
129 root->appendNew<Value>(
130 proc, opcode, Origin(),
131 floatValue1,
132 floatValue2),
133 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2),
134 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
135 CHECK(isIdentical(compileAndRun<int32_t>(proc, bitwise_cast<int32_t>(a), bitwise_cast<int32_t>(b), 42, -5), operation(a, b) ? 42 : -5));
136}
137
138void testSelectCompareFloat(float a, float b)
139{
140 testSelectCompareFloat<Equal>(a, b, [](float a, float b) -> bool { return a == b; });
141 testSelectCompareFloat<NotEqual>(a, b, [](float a, float b) -> bool { return a != b; });
142 testSelectCompareFloat<LessThan>(a, b, [](float a, float b) -> bool { return a < b; });
143 testSelectCompareFloat<GreaterThan>(a, b, [](float a, float b) -> bool { return a > b; });
144 testSelectCompareFloat<LessEqual>(a, b, [](float a, float b) -> bool { return a <= b; });
145 testSelectCompareFloat<GreaterEqual>(a, b, [](float a, float b) -> bool { return a >= b; });
146 testSelectCompareFloat<EqualOrUnordered>(a, b, [](float a, float b) -> bool { return a != a || b != b || a == b; });
147}
148
149template<B3::Opcode opcode>
150void testSelectCompareFloatToDouble(float a, float b, bool (*operation)(float, float))
151{
152 Procedure proc;
153 BasicBlock* root = proc.addBlock();
154 Value* argument1int32 = root->appendNew<Value>(proc, Trunc, Origin(),
155 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
156 Value* argument2int32 = root->appendNew<Value>(proc, Trunc, Origin(),
157 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
158 Value* floatValue1 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument1int32);
159 Value* floatValue2 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument2int32);
160 Value* doubleValue1 = root->appendNew<Value>(proc, FloatToDouble, Origin(), floatValue1);
161 Value* doubleValue2 = root->appendNew<Value>(proc, FloatToDouble, Origin(), floatValue2);
162
163 root->appendNewControlValue(
164 proc, Return, Origin(),
165 root->appendNew<Value>(
166 proc, Select, Origin(),
167 root->appendNew<Value>(
168 proc, opcode, Origin(),
169 doubleValue1,
170 doubleValue2),
171 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2),
172 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
173 CHECK(isIdentical(compileAndRun<int32_t>(proc, bitwise_cast<int32_t>(a), bitwise_cast<int32_t>(b), 42, -5), operation(a, b) ? 42 : -5));
174}
175
176void testSelectCompareFloatToDouble(float a, float b)
177{
178 testSelectCompareFloatToDouble<Equal>(a, b, [](float a, float b) -> bool { return a == b; });
179 testSelectCompareFloatToDouble<NotEqual>(a, b, [](float a, float b) -> bool { return a != b; });
180 testSelectCompareFloatToDouble<LessThan>(a, b, [](float a, float b) -> bool { return a < b; });
181 testSelectCompareFloatToDouble<GreaterThan>(a, b, [](float a, float b) -> bool { return a > b; });
182 testSelectCompareFloatToDouble<LessEqual>(a, b, [](float a, float b) -> bool { return a <= b; });
183 testSelectCompareFloatToDouble<GreaterEqual>(a, b, [](float a, float b) -> bool { return a >= b; });
184 testSelectCompareFloatToDouble<EqualOrUnordered>(a, b, [](float a, float b) -> bool { return a != a || b != b || a == b; });
185}
186
187void testSelectDouble()
188{
189 Procedure proc;
190 BasicBlock* root = proc.addBlock();
191 root->appendNewControlValue(
192 proc, Return, Origin(),
193 root->appendNew<Value>(
194 proc, Select, Origin(),
195 root->appendNew<Value>(
196 proc, Equal, Origin(),
197 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
198 root->appendNew<ConstPtrValue>(proc, Origin(), 42)),
199 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0),
200 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1)));
201
202 auto code = compileProc(proc);
203 CHECK(invoke<double>(*code, 42, 1.5, 2.6) == 1.5);
204 CHECK(invoke<double>(*code, 42, 642462.7, 32533.8) == 642462.7);
205 CHECK(invoke<double>(*code, 43, 1.9, 2.0) == 2.0);
206 CHECK(invoke<double>(*code, 43, 642462.1, 32533.2) == 32533.2);
207}
208
209void testSelectDoubleTest()
210{
211 Procedure proc;
212 BasicBlock* root = proc.addBlock();
213 root->appendNewControlValue(
214 proc, Return, Origin(),
215 root->appendNew<Value>(
216 proc, Select, Origin(),
217 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
218 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0),
219 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1)));
220
221 auto code = compileProc(proc);
222 CHECK(invoke<double>(*code, 42, 1.5, 2.6) == 1.5);
223 CHECK(invoke<double>(*code, 42, 642462.7, 32533.8) == 642462.7);
224 CHECK(invoke<double>(*code, 0, 1.9, 2.0) == 2.0);
225 CHECK(invoke<double>(*code, 0, 642462.1, 32533.2) == 32533.2);
226}
227
228void testSelectDoubleCompareDouble()
229{
230 Procedure proc;
231 BasicBlock* root = proc.addBlock();
232 root->appendNewControlValue(
233 proc, Return, Origin(),
234 root->appendNew<Value>(
235 proc, Select, Origin(),
236 root->appendNew<Value>(
237 proc, LessThan, Origin(),
238 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0),
239 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1)),
240 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2),
241 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR3)));
242
243 auto code = compileProc(proc);
244 CHECK(invoke<double>(*code, -1.0, 1.0, 1.1, 2.2) == 1.1);
245 CHECK(invoke<double>(*code, 42.5, 42.51, 642462.3, 32533.4) == 642462.3);
246 CHECK(invoke<double>(*code, PNaN, 0.0, 1.5, 2.6) == 2.6);
247 CHECK(invoke<double>(*code, 42.51, 42.5, 642462.7, 32533.8) == 32533.8);
248 CHECK(invoke<double>(*code, 42.52, 42.52, 524978245.9, 352.0) == 352.0);
249}
250
251void testSelectDoubleCompareFloat(float a, float b)
252{
253 Procedure proc;
254 BasicBlock* root = proc.addBlock();
255 Value* argument1int32 = root->appendNew<Value>(proc, Trunc, Origin(),
256 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
257 Value* argument2int32 = root->appendNew<Value>(proc, Trunc, Origin(),
258 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
259 Value* floatValue1 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument1int32);
260 Value* floatValue2 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument2int32);
261
262 root->appendNewControlValue(
263 proc, Return, Origin(),
264 root->appendNew<Value>(
265 proc, Select, Origin(),
266 root->appendNew<Value>(
267 proc, LessThan, Origin(),
268 floatValue1,
269 floatValue2),
270 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0),
271 root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1)));
272
273 CHECK(isIdentical(compileAndRun<double>(proc, bitwise_cast<int32_t>(a), bitwise_cast<int32_t>(b), 42.1, -M_PI), a < b ? 42.1 : -M_PI));
274}
275
276void testSelectFloatCompareFloat(float a, float b)
277{
278 Procedure proc;
279 BasicBlock* root = proc.addBlock();
280 Value* argument1int32 = root->appendNew<Value>(proc, Trunc, Origin(),
281 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
282 Value* argument2int32 = root->appendNew<Value>(proc, Trunc, Origin(),
283 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
284 Value* argument3int32 = root->appendNew<Value>(proc, Trunc, Origin(),
285 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2));
286 Value* argument4int32 = root->appendNew<Value>(proc, Trunc, Origin(),
287 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3));
288 Value* floatValue1 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument1int32);
289 Value* floatValue2 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument2int32);
290 Value* floatValue3 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument3int32);
291 Value* floatValue4 = root->appendNew<Value>(proc, BitwiseCast, Origin(), argument4int32);
292
293 root->appendNewControlValue(
294 proc, Return, Origin(),
295 root->appendNew<Value>(
296 proc, Select, Origin(),
297 root->appendNew<Value>(
298 proc, LessThan, Origin(),
299 floatValue1,
300 floatValue2),
301 floatValue3,
302 floatValue4));
303
304 CHECK(isIdentical(compileAndRun<float>(proc, bitwise_cast<int32_t>(a), bitwise_cast<int32_t>(b), bitwise_cast<int32_t>(1.1f), bitwise_cast<int32_t>(-42.f)), a < b ? 1.1f : -42.f));
305}
306
307
308template<B3::Opcode opcode>
309void testSelectDoubleCompareDouble(bool (*operation)(double, double))
310{
311 { // Compare arguments and selected arguments are all different.
312 Procedure proc;
313 BasicBlock* root = proc.addBlock();
314 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
315 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
316 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
317 Value* arg3 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR3);
318
319 root->appendNewControlValue(
320 proc, Return, Origin(),
321 root->appendNew<Value>(
322 proc, Select, Origin(),
323 root->appendNew<Value>(
324 proc, opcode, Origin(),
325 arg0,
326 arg1),
327 arg2,
328 arg3));
329 auto code = compileProc(proc);
330
331 for (auto& left : floatingPointOperands<double>()) {
332 for (auto& right : floatingPointOperands<double>()) {
333 double expected = operation(left.value, right.value) ? 42.5 : -66.5;
334 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, -66.5), expected));
335 }
336 }
337 }
338 { // Compare arguments and selected arguments are all different. "thenCase" is live after operation.
339 Procedure proc;
340 BasicBlock* root = proc.addBlock();
341 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
342 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
343 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
344 Value* arg3 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR3);
345
346 Value* result = root->appendNew<Value>(proc, Select, Origin(),
347 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
348 arg2,
349 arg3);
350
351 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
352 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
353 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
354
355 root->appendNewControlValue(proc, Return, Origin(), result);
356 auto code = compileProc(proc);
357
358 for (auto& left : floatingPointOperands<double>()) {
359 for (auto& right : floatingPointOperands<double>()) {
360 double expected = operation(left.value, right.value) ? 42.5 : -66.5;
361 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, -66.5), expected));
362 }
363 }
364 }
365 { // Compare arguments and selected arguments are all different. "elseCase" is live after operation.
366 Procedure proc;
367 BasicBlock* root = proc.addBlock();
368 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
369 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
370 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
371 Value* arg3 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR3);
372
373 Value* result = root->appendNew<Value>(proc, Select, Origin(),
374 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
375 arg2,
376 arg3);
377
378 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
379 keepValuesLive->append(ConstrainedValue(arg3, ValueRep::SomeRegister));
380 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
381
382 root->appendNewControlValue(proc, Return, Origin(), result);
383 auto code = compileProc(proc);
384
385 for (auto& left : floatingPointOperands<double>()) {
386 for (auto& right : floatingPointOperands<double>()) {
387 double expected = operation(left.value, right.value) ? 42.5 : -66.5;
388 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, -66.5), expected));
389 }
390 }
391 }
392 { // Compare arguments and selected arguments are all different. Both cases are live after operation.
393 Procedure proc;
394 BasicBlock* root = proc.addBlock();
395 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
396 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
397 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
398 Value* arg3 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR3);
399
400 Value* result = root->appendNew<Value>(proc, Select, Origin(),
401 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
402 arg2,
403 arg3);
404
405 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
406 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
407 keepValuesLive->append(ConstrainedValue(arg3, ValueRep::SomeRegister));
408 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
409
410 root->appendNewControlValue(proc, Return, Origin(), result);
411 auto code = compileProc(proc);
412
413 for (auto& left : floatingPointOperands<double>()) {
414 for (auto& right : floatingPointOperands<double>()) {
415 double expected = operation(left.value, right.value) ? 42.5 : -66.5;
416 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, -66.5), expected));
417 }
418 }
419 }
420 { // The left argument is the same as the "elseCase" argument.
421 Procedure proc;
422 BasicBlock* root = proc.addBlock();
423 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
424 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
425 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
426
427 root->appendNewControlValue(
428 proc, Return, Origin(),
429 root->appendNew<Value>(
430 proc, Select, Origin(),
431 root->appendNew<Value>(
432 proc, opcode, Origin(),
433 arg0,
434 arg1),
435 arg2,
436 arg0));
437 auto code = compileProc(proc);
438
439 for (auto& left : floatingPointOperands<double>()) {
440 for (auto& right : floatingPointOperands<double>()) {
441 double expected = operation(left.value, right.value) ? 42.5 : left.value;
442 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, left.value), expected));
443 }
444 }
445 }
446 { // The left argument is the same as the "elseCase" argument. "thenCase" is live after operation.
447 Procedure proc;
448 BasicBlock* root = proc.addBlock();
449 Value* arg0 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
450 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR1);
451 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR2);
452
453 Value* result = root->appendNew<Value>(proc, Select, Origin(),
454 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
455 arg2,
456 arg0);
457
458 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
459 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
460 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
461
462 root->appendNewControlValue(proc, Return, Origin(), result);
463 auto code = compileProc(proc);
464
465 for (auto& left : floatingPointOperands<double>()) {
466 for (auto& right : floatingPointOperands<double>()) {
467 double expected = operation(left.value, right.value) ? 42.5 : left.value;
468 CHECK(isIdentical(invoke<double>(*code, left.value, right.value, 42.5, left.value), expected));
469 }
470 }
471 }
472}
473
474void testSelectDoubleCompareDoubleWithAliasing()
475{
476 testSelectDoubleCompareDouble<Equal>([](double a, double b) -> bool { return a == b; });
477 testSelectDoubleCompareDouble<NotEqual>([](double a, double b) -> bool { return a != b; });
478 testSelectDoubleCompareDouble<LessThan>([](double a, double b) -> bool { return a < b; });
479 testSelectDoubleCompareDouble<GreaterThan>([](double a, double b) -> bool { return a > b; });
480 testSelectDoubleCompareDouble<LessEqual>([](double a, double b) -> bool { return a <= b; });
481 testSelectDoubleCompareDouble<GreaterEqual>([](double a, double b) -> bool { return a >= b; });
482 testSelectDoubleCompareDouble<EqualOrUnordered>([](double a, double b) -> bool { return a != a || b != b || a == b; });
483}
484
485template<B3::Opcode opcode>
486void testSelectFloatCompareFloat(bool (*operation)(float, float))
487{
488 { // Compare arguments and selected arguments are all different.
489 Procedure proc;
490 BasicBlock* root = proc.addBlock();
491
492 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
493 root->appendNew<Value>(proc, Trunc, Origin(),
494 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
495 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
496 root->appendNew<Value>(proc, Trunc, Origin(),
497 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
498 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
499 root->appendNew<Value>(proc, Trunc, Origin(),
500 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
501 Value* arg3 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
502 root->appendNew<Value>(proc, Trunc, Origin(),
503 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
504
505 root->appendNewControlValue(
506 proc, Return, Origin(),
507 root->appendNew<Value>(
508 proc, Select, Origin(),
509 root->appendNew<Value>(
510 proc, opcode, Origin(),
511 arg0,
512 arg1),
513 arg2,
514 arg3));
515 auto code = compileProc(proc);
516
517 for (auto& left : floatingPointOperands<float>()) {
518 for (auto& right : floatingPointOperands<float>()) {
519 float expected = operation(left.value, right.value) ? 42.5 : -66.5;
520 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(-66.5f)), expected));
521 }
522 }
523 }
524 { // Compare arguments and selected arguments are all different. "thenCase" is live after operation.
525 Procedure proc;
526 BasicBlock* root = proc.addBlock();
527 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
528 root->appendNew<Value>(proc, Trunc, Origin(),
529 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
530 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
531 root->appendNew<Value>(proc, Trunc, Origin(),
532 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
533 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
534 root->appendNew<Value>(proc, Trunc, Origin(),
535 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
536 Value* arg3 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
537 root->appendNew<Value>(proc, Trunc, Origin(),
538 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
539
540 Value* result = root->appendNew<Value>(proc, Select, Origin(),
541 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
542 arg2,
543 arg3);
544
545 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
546 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
547 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
548
549 root->appendNewControlValue(proc, Return, Origin(), result);
550 auto code = compileProc(proc);
551
552 for (auto& left : floatingPointOperands<float>()) {
553 for (auto& right : floatingPointOperands<float>()) {
554 float expected = operation(left.value, right.value) ? 42.5 : -66.5;
555 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(-66.5f)), expected));
556 }
557 }
558 }
559 { // Compare arguments and selected arguments are all different. "elseCase" is live after operation.
560 Procedure proc;
561 BasicBlock* root = proc.addBlock();
562 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
563 root->appendNew<Value>(proc, Trunc, Origin(),
564 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
565 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
566 root->appendNew<Value>(proc, Trunc, Origin(),
567 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
568 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
569 root->appendNew<Value>(proc, Trunc, Origin(),
570 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
571 Value* arg3 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
572 root->appendNew<Value>(proc, Trunc, Origin(),
573 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
574
575 Value* result = root->appendNew<Value>(proc, Select, Origin(),
576 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
577 arg2,
578 arg3);
579
580 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
581 keepValuesLive->append(ConstrainedValue(arg3, ValueRep::SomeRegister));
582 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
583
584 root->appendNewControlValue(proc, Return, Origin(), result);
585 auto code = compileProc(proc);
586
587 for (auto& left : floatingPointOperands<float>()) {
588 for (auto& right : floatingPointOperands<float>()) {
589 float expected = operation(left.value, right.value) ? 42.5 : -66.5;
590 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(-66.5f)), expected));
591 }
592 }
593 }
594 { // Compare arguments and selected arguments are all different. Both cases are live after operation.
595 Procedure proc;
596 BasicBlock* root = proc.addBlock();
597 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
598 root->appendNew<Value>(proc, Trunc, Origin(),
599 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
600 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
601 root->appendNew<Value>(proc, Trunc, Origin(),
602 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
603 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
604 root->appendNew<Value>(proc, Trunc, Origin(),
605 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
606 Value* arg3 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
607 root->appendNew<Value>(proc, Trunc, Origin(),
608 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3)));
609
610 Value* result = root->appendNew<Value>(proc, Select, Origin(),
611 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
612 arg2,
613 arg3);
614
615 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
616 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
617 keepValuesLive->append(ConstrainedValue(arg3, ValueRep::SomeRegister));
618 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
619
620 root->appendNewControlValue(proc, Return, Origin(), result);
621 auto code = compileProc(proc);
622
623 for (auto& left : floatingPointOperands<float>()) {
624 for (auto& right : floatingPointOperands<float>()) {
625 float expected = operation(left.value, right.value) ? 42.5 : -66.5;
626 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(-66.5f)), expected));
627 }
628 }
629 }
630 { // The left argument is the same as the "elseCase" argument.
631 Procedure proc;
632 BasicBlock* root = proc.addBlock();
633 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
634 root->appendNew<Value>(proc, Trunc, Origin(),
635 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
636 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
637 root->appendNew<Value>(proc, Trunc, Origin(),
638 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
639 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
640 root->appendNew<Value>(proc, Trunc, Origin(),
641 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
642
643 root->appendNewControlValue(
644 proc, Return, Origin(),
645 root->appendNew<Value>(
646 proc, Select, Origin(),
647 root->appendNew<Value>(
648 proc, opcode, Origin(),
649 arg0,
650 arg1),
651 arg2,
652 arg0));
653 auto code = compileProc(proc);
654
655 for (auto& left : floatingPointOperands<float>()) {
656 for (auto& right : floatingPointOperands<float>()) {
657 float expected = operation(left.value, right.value) ? 42.5 : left.value;
658 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(left.value)), expected));
659 }
660 }
661 }
662 { // The left argument is the same as the "elseCase" argument. "thenCase" is live after operation.
663 Procedure proc;
664 BasicBlock* root = proc.addBlock();
665 Value* arg0 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
666 root->appendNew<Value>(proc, Trunc, Origin(),
667 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
668 Value* arg1 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
669 root->appendNew<Value>(proc, Trunc, Origin(),
670 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
671 Value* arg2 = root->appendNew<Value>(proc, BitwiseCast, Origin(),
672 root->appendNew<Value>(proc, Trunc, Origin(),
673 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
674
675 Value* result = root->appendNew<Value>(proc, Select, Origin(),
676 root->appendNew<Value>(proc, opcode, Origin(), arg0, arg1),
677 arg2,
678 arg0);
679
680 PatchpointValue* keepValuesLive = root->appendNew<PatchpointValue>(proc, Void, Origin());
681 keepValuesLive->append(ConstrainedValue(arg2, ValueRep::SomeRegister));
682 keepValuesLive->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
683
684 root->appendNewControlValue(proc, Return, Origin(), result);
685 auto code = compileProc(proc);
686
687 for (auto& left : floatingPointOperands<float>()) {
688 for (auto& right : floatingPointOperands<float>()) {
689 float expected = operation(left.value, right.value) ? 42.5 : left.value;
690 CHECK(isIdentical(invoke<float>(*code, bitwise_cast<int32_t>(left.value), bitwise_cast<int32_t>(right.value), bitwise_cast<int32_t>(42.5f), bitwise_cast<int32_t>(left.value)), expected));
691 }
692 }
693 }
694}
695
696void testSelectFloatCompareFloatWithAliasing()
697{
698 testSelectFloatCompareFloat<Equal>([](float a, float b) -> bool { return a == b; });
699 testSelectFloatCompareFloat<NotEqual>([](float a, float b) -> bool { return a != b; });
700 testSelectFloatCompareFloat<LessThan>([](float a, float b) -> bool { return a < b; });
701 testSelectFloatCompareFloat<GreaterThan>([](float a, float b) -> bool { return a > b; });
702 testSelectFloatCompareFloat<LessEqual>([](float a, float b) -> bool { return a <= b; });
703 testSelectFloatCompareFloat<GreaterEqual>([](float a, float b) -> bool { return a >= b; });
704 testSelectFloatCompareFloat<EqualOrUnordered>([](float a, float b) -> bool { return a != a || b != b || a == b; });
705}
706
707void testSelectFold(intptr_t value)
708{
709 Procedure proc;
710 BasicBlock* root = proc.addBlock();
711 root->appendNewControlValue(
712 proc, Return, Origin(),
713 root->appendNew<Value>(
714 proc, Select, Origin(),
715 root->appendNew<Value>(
716 proc, Equal, Origin(),
717 root->appendNew<ConstPtrValue>(proc, Origin(), value),
718 root->appendNew<ConstPtrValue>(proc, Origin(), 42)),
719 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
720 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
721
722 auto code = compileProc(proc);
723 CHECK(invoke<intptr_t>(*code, 1, 2) == (value == 42 ? 1 : 2));
724 CHECK(invoke<intptr_t>(*code, 642462, 32533) == (value == 42 ? 642462 : 32533));
725}
726
727void testSelectInvert()
728{
729 Procedure proc;
730 BasicBlock* root = proc.addBlock();
731 root->appendNewControlValue(
732 proc, Return, Origin(),
733 root->appendNew<Value>(
734 proc, Select, Origin(),
735 root->appendNew<Value>(
736 proc, Equal, Origin(),
737 root->appendNew<Value>(
738 proc, NotEqual, Origin(),
739 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
740 root->appendNew<ConstPtrValue>(proc, Origin(), 42)),
741 root->appendNew<Const32Value>(proc, Origin(), 0)),
742 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1),
743 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2)));
744
745 auto code = compileProc(proc);
746 CHECK(invoke<intptr_t>(*code, 42, 1, 2) == 1);
747 CHECK(invoke<intptr_t>(*code, 42, 642462, 32533) == 642462);
748 CHECK(invoke<intptr_t>(*code, 43, 1, 2) == 2);
749 CHECK(invoke<intptr_t>(*code, 43, 642462, 32533) == 32533);
750}
751
752void testCheckSelect()
753{
754 Procedure proc;
755 if (proc.optLevel() < 1)
756 return;
757 BasicBlock* root = proc.addBlock();
758
759 CheckValue* check = root->appendNew<CheckValue>(
760 proc, Check, Origin(),
761 root->appendNew<Value>(
762 proc, Add, Origin(),
763 root->appendNew<Value>(
764 proc, Select, Origin(),
765 root->appendNew<Value>(
766 proc, BitAnd, Origin(),
767 root->appendNew<Value>(
768 proc, Trunc, Origin(),
769 root->appendNew<ArgumentRegValue>(
770 proc, Origin(), GPRInfo::argumentGPR0)),
771 root->appendNew<Const32Value>(proc, Origin(), 0xff)),
772 root->appendNew<ConstPtrValue>(proc, Origin(), -42),
773 root->appendNew<ConstPtrValue>(proc, Origin(), 35)),
774 root->appendNew<ConstPtrValue>(proc, Origin(), 42)));
775 unsigned generationCount = 0;
776 check->setGenerator(
777 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
778 AllowMacroScratchRegisterUsage allowScratch(jit);
779
780 generationCount++;
781 jit.move(CCallHelpers::TrustedImm32(666), GPRInfo::returnValueGPR);
782 jit.emitFunctionEpilogue();
783 jit.ret();
784 });
785
786 root->appendNewControlValue(
787 proc, Return, Origin(),
788 root->appendNew<Const32Value>(proc, Origin(), 0));
789
790 auto code = compileProc(proc);
791 CHECK(generationCount == 1);
792 CHECK(invoke<int>(*code, true) == 0);
793 CHECK(invoke<int>(*code, false) == 666);
794}
795
796void testCheckSelectCheckSelect()
797{
798 Procedure proc;
799 if (proc.optLevel() < 1)
800 return;
801 BasicBlock* root = proc.addBlock();
802
803 CheckValue* check = root->appendNew<CheckValue>(
804 proc, Check, Origin(),
805 root->appendNew<Value>(
806 proc, Add, Origin(),
807 root->appendNew<Value>(
808 proc, Select, Origin(),
809 root->appendNew<Value>(
810 proc, BitAnd, Origin(),
811 root->appendNew<Value>(
812 proc, Trunc, Origin(),
813 root->appendNew<ArgumentRegValue>(
814 proc, Origin(), GPRInfo::argumentGPR0)),
815 root->appendNew<Const32Value>(proc, Origin(), 0xff)),
816 root->appendNew<ConstPtrValue>(proc, Origin(), -42),
817 root->appendNew<ConstPtrValue>(proc, Origin(), 35)),
818 root->appendNew<ConstPtrValue>(proc, Origin(), 42)));
819
820 unsigned generationCount = 0;
821 check->setGenerator(
822 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
823 AllowMacroScratchRegisterUsage allowScratch(jit);
824
825 generationCount++;
826 jit.move(CCallHelpers::TrustedImm32(666), GPRInfo::returnValueGPR);
827 jit.emitFunctionEpilogue();
828 jit.ret();
829 });
830
831 CheckValue* check2 = root->appendNew<CheckValue>(
832 proc, Check, Origin(),
833 root->appendNew<Value>(
834 proc, Add, Origin(),
835 root->appendNew<Value>(
836 proc, Select, Origin(),
837 root->appendNew<Value>(
838 proc, BitAnd, Origin(),
839 root->appendNew<Value>(
840 proc, Trunc, Origin(),
841 root->appendNew<ArgumentRegValue>(
842 proc, Origin(), GPRInfo::argumentGPR1)),
843 root->appendNew<Const32Value>(proc, Origin(), 0xff)),
844 root->appendNew<ConstPtrValue>(proc, Origin(), -43),
845 root->appendNew<ConstPtrValue>(proc, Origin(), 36)),
846 root->appendNew<ConstPtrValue>(proc, Origin(), 43)));
847
848 unsigned generationCount2 = 0;
849 check2->setGenerator(
850 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
851 AllowMacroScratchRegisterUsage allowScratch(jit);
852
853 generationCount2++;
854 jit.move(CCallHelpers::TrustedImm32(667), GPRInfo::returnValueGPR);
855 jit.emitFunctionEpilogue();
856 jit.ret();
857 });
858
859 root->appendNewControlValue(
860 proc, Return, Origin(),
861 root->appendNew<Const32Value>(proc, Origin(), 0));
862
863 auto code = compileProc(proc);
864 CHECK(generationCount == 1);
865 CHECK(generationCount2 == 1);
866 CHECK(invoke<int>(*code, true, true) == 0);
867 CHECK(invoke<int>(*code, false, true) == 666);
868 CHECK(invoke<int>(*code, true, false) == 667);
869}
870
871void testCheckSelectAndCSE()
872{
873 Procedure proc;
874 if (proc.optLevel() < 1)
875 return;
876 BasicBlock* root = proc.addBlock();
877
878 auto* selectValue = root->appendNew<Value>(
879 proc, Select, Origin(),
880 root->appendNew<Value>(
881 proc, BitAnd, Origin(),
882 root->appendNew<Value>(
883 proc, Trunc, Origin(),
884 root->appendNew<ArgumentRegValue>(
885 proc, Origin(), GPRInfo::argumentGPR0)),
886 root->appendNew<Const32Value>(proc, Origin(), 0xff)),
887 root->appendNew<ConstPtrValue>(proc, Origin(), -42),
888 root->appendNew<ConstPtrValue>(proc, Origin(), 35));
889
890 auto* constant = root->appendNew<ConstPtrValue>(proc, Origin(), 42);
891 auto* addValue = root->appendNew<Value>(proc, Add, Origin(), selectValue, constant);
892
893 CheckValue* check = root->appendNew<CheckValue>(proc, Check, Origin(), addValue);
894 unsigned generationCount = 0;
895 check->setGenerator(
896 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
897 AllowMacroScratchRegisterUsage allowScratch(jit);
898
899 generationCount++;
900 jit.move(CCallHelpers::TrustedImm32(666), GPRInfo::returnValueGPR);
901 jit.emitFunctionEpilogue();
902 jit.ret();
903 });
904
905 auto* addValue2 = root->appendNew<Value>(proc, Add, Origin(), selectValue, constant);
906
907 root->appendNewControlValue(
908 proc, Return, Origin(),
909 root->appendNew<Value>(proc, Add, Origin(), addValue, addValue2));
910
911 auto code = compileProc(proc);
912 CHECK(generationCount == 1);
913 CHECK(invoke<int>(*code, true) == 0);
914 CHECK(invoke<int>(*code, false) == 666);
915}
916
917double b3Pow(double x, int y)
918{
919 if (y < 0 || y > 1000)
920 return pow(x, y);
921 double result = 1;
922 while (y) {
923 if (y & 1)
924 result *= x;
925 x *= x;
926 y >>= 1;
927 }
928 return result;
929}
930
931void testPowDoubleByIntegerLoop(double xOperand, int32_t yOperand)
932{
933 Procedure proc;
934 BasicBlock* root = proc.addBlock();
935
936 Value* x = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
937 Value* y = root->appendNew<Value>(proc, Trunc, Origin(),
938 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
939 auto result = powDoubleInt32(proc, root, Origin(), x, y);
940 BasicBlock* continuation = result.first;
941 continuation->appendNewControlValue(proc, Return, Origin(), result.second);
942
943 CHECK(isIdentical(compileAndRun<double>(proc, xOperand, yOperand), b3Pow(xOperand, yOperand)));
944}
945
946void testTruncOrHigh()
947{
948 Procedure proc;
949 BasicBlock* root = proc.addBlock();
950
951 root->appendNewControlValue(
952 proc, Return, Origin(),
953 root->appendNew<Value>(
954 proc, Trunc, Origin(),
955 root->appendNew<Value>(
956 proc, BitOr, Origin(),
957 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
958 root->appendNew<Const64Value>(proc, Origin(), 0x100000000))));
959
960 int64_t value = 0x123456781234;
961 CHECK(compileAndRun<int>(proc, value) == 0x56781234);
962}
963
964void testTruncOrLow()
965{
966 Procedure proc;
967 BasicBlock* root = proc.addBlock();
968
969 root->appendNewControlValue(
970 proc, Return, Origin(),
971 root->appendNew<Value>(
972 proc, Trunc, Origin(),
973 root->appendNew<Value>(
974 proc, BitOr, Origin(),
975 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
976 root->appendNew<Const64Value>(proc, Origin(), 0x1000000))));
977
978 int64_t value = 0x123456781234;
979 CHECK(compileAndRun<int>(proc, value) == 0x57781234);
980}
981
982void testBitAndOrHigh()
983{
984 Procedure proc;
985 BasicBlock* root = proc.addBlock();
986
987 root->appendNewControlValue(
988 proc, Return, Origin(),
989 root->appendNew<Value>(
990 proc, BitAnd, Origin(),
991 root->appendNew<Value>(
992 proc, BitOr, Origin(),
993 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
994 root->appendNew<Const64Value>(proc, Origin(), 0x8)),
995 root->appendNew<Const64Value>(proc, Origin(), 0x777777777777)));
996
997 int64_t value = 0x123456781234;
998 CHECK(compileAndRun<int64_t>(proc, value) == 0x123456701234ll);
999}
1000
1001void testBitAndOrLow()
1002{
1003 Procedure proc;
1004 BasicBlock* root = proc.addBlock();
1005
1006 root->appendNewControlValue(
1007 proc, Return, Origin(),
1008 root->appendNew<Value>(
1009 proc, BitAnd, Origin(),
1010 root->appendNew<Value>(
1011 proc, BitOr, Origin(),
1012 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1013 root->appendNew<Const64Value>(proc, Origin(), 0x1)),
1014 root->appendNew<Const64Value>(proc, Origin(), 0x777777777777)));
1015
1016 int64_t value = 0x123456781234;
1017 CHECK(compileAndRun<int64_t>(proc, value) == 0x123456701235ll);
1018}
1019
1020void testBranch64Equal(int64_t left, int64_t right)
1021{
1022 Procedure proc;
1023 BasicBlock* root = proc.addBlock();
1024 BasicBlock* thenCase = proc.addBlock();
1025 BasicBlock* elseCase = proc.addBlock();
1026
1027 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1028 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
1029 root->appendNewControlValue(
1030 proc, Branch, Origin(),
1031 root->appendNew<Value>(proc, Equal, Origin(), arg1, arg2),
1032 FrequentedBlock(thenCase), FrequentedBlock(elseCase));
1033
1034 bool trueResult = true;
1035 thenCase->appendNewControlValue(
1036 proc, Return, Origin(),
1037 thenCase->appendNew<MemoryValue>(
1038 proc, Load8Z, Origin(),
1039 thenCase->appendNew<ConstPtrValue>(proc, Origin(), &trueResult)));
1040
1041 bool elseResult = false;
1042 elseCase->appendNewControlValue(
1043 proc, Return, Origin(),
1044 elseCase->appendNew<MemoryValue>(
1045 proc, Load8Z, Origin(),
1046 elseCase->appendNew<ConstPtrValue>(proc, Origin(), &elseResult)));
1047
1048 CHECK(compileAndRun<bool>(proc, left, right) == (left == right));
1049}
1050
1051void testBranch64EqualImm(int64_t left, int64_t right)
1052{
1053 Procedure proc;
1054 BasicBlock* root = proc.addBlock();
1055 BasicBlock* thenCase = proc.addBlock();
1056 BasicBlock* elseCase = proc.addBlock();
1057
1058 Value* arg1 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1059 Value* arg2 = root->appendNew<ConstPtrValue>(proc, Origin(), right);
1060 root->appendNewControlValue(
1061 proc, Branch, Origin(),
1062 root->appendNew<Value>(proc, Equal, Origin(), arg1, arg2),
1063 FrequentedBlock(thenCase), FrequentedBlock(elseCase));
1064
1065 bool trueResult = true;
1066 thenCase->appendNewControlValue(
1067 proc, Return, Origin(),
1068 thenCase->appendNew<MemoryValue>(
1069 proc, Load8Z, Origin(),
1070 thenCase->appendNew<ConstPtrValue>(proc, Origin(), &trueResult)));
1071
1072 bool elseResult = false;
1073 elseCase->appendNewControlValue(
1074 proc, Return, Origin(),
1075 elseCase->appendNew<MemoryValue>(
1076 proc, Load8Z, Origin(),
1077 elseCase->appendNew<ConstPtrValue>(proc, Origin(), &elseResult)));
1078
1079 CHECK(compileAndRun<bool>(proc, left) == (left == right));
1080}
1081
1082void testBranch64EqualMem(int64_t left, int64_t right)
1083{
1084 Procedure proc;
1085 BasicBlock* root = proc.addBlock();
1086 BasicBlock* thenCase = proc.addBlock();
1087 BasicBlock* elseCase = proc.addBlock();
1088
1089 Value* arg1 = root->appendNew<MemoryValue>(
1090 proc, Load, pointerType(), Origin(),
1091 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1092 Value* arg2 = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
1093 root->appendNewControlValue(
1094 proc, Branch, Origin(),
1095 root->appendNew<Value>(proc, Equal, Origin(), arg1, arg2),
1096 FrequentedBlock(thenCase), FrequentedBlock(elseCase));
1097
1098 bool trueResult = true;
1099 thenCase->appendNewControlValue(
1100 proc, Return, Origin(),
1101 thenCase->appendNew<MemoryValue>(
1102 proc, Load8Z, Origin(),
1103 thenCase->appendNew<ConstPtrValue>(proc, Origin(), &trueResult)));
1104
1105 bool elseResult = false;
1106 elseCase->appendNewControlValue(
1107 proc, Return, Origin(),
1108 elseCase->appendNew<MemoryValue>(
1109 proc, Load8Z, Origin(),
1110 elseCase->appendNew<ConstPtrValue>(proc, Origin(), &elseResult)));
1111
1112 CHECK(compileAndRun<bool>(proc, &left, right) == (left == right));
1113}
1114
1115void testBranch64EqualMemImm(int64_t left, int64_t right)
1116{
1117 Procedure proc;
1118 BasicBlock* root = proc.addBlock();
1119 BasicBlock* thenCase = proc.addBlock();
1120 BasicBlock* elseCase = proc.addBlock();
1121
1122 Value* arg1 = root->appendNew<MemoryValue>(
1123 proc, Load, pointerType(), Origin(),
1124 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1125 Value* arg2 = root->appendNew<ConstPtrValue>(proc, Origin(), right);
1126 root->appendNewControlValue(
1127 proc, Branch, Origin(),
1128 root->appendNew<Value>(proc, Equal, Origin(), arg1, arg2),
1129 FrequentedBlock(thenCase), FrequentedBlock(elseCase));
1130
1131 bool trueResult = true;
1132 thenCase->appendNewControlValue(
1133 proc, Return, Origin(),
1134 thenCase->appendNew<MemoryValue>(
1135 proc, Load8Z, Origin(),
1136 thenCase->appendNew<ConstPtrValue>(proc, Origin(), &trueResult)));
1137
1138 bool elseResult = false;
1139 elseCase->appendNewControlValue(
1140 proc, Return, Origin(),
1141 elseCase->appendNew<MemoryValue>(
1142 proc, Load8Z, Origin(),
1143 elseCase->appendNew<ConstPtrValue>(proc, Origin(), &elseResult)));
1144
1145 CHECK(compileAndRun<bool>(proc, &left) == (left == right));
1146}
1147
1148void testStore8Load8Z(int32_t value)
1149{
1150 Procedure proc;
1151 BasicBlock* root = proc.addBlock();
1152
1153 int8_t byte;
1154 Value* ptr = root->appendNew<ConstPtrValue>(proc, Origin(), &byte);
1155
1156 root->appendNew<MemoryValue>(
1157 proc, Store8, Origin(),
1158 root->appendNew<Value>(
1159 proc, Trunc, Origin(),
1160 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
1161 ptr);
1162
1163 root->appendNewControlValue(
1164 proc, Return, Origin(),
1165 root->appendNew<MemoryValue>(proc, Load8Z, Origin(), ptr));
1166
1167 CHECK(compileAndRun<int32_t>(proc, value) == static_cast<uint8_t>(value));
1168}
1169
1170void testStore16Load16Z(int32_t value)
1171{
1172 Procedure proc;
1173 BasicBlock* root = proc.addBlock();
1174
1175 int16_t byte;
1176 Value* ptr = root->appendNew<ConstPtrValue>(proc, Origin(), &byte);
1177
1178 root->appendNew<MemoryValue>(
1179 proc, Store16, Origin(),
1180 root->appendNew<Value>(
1181 proc, Trunc, Origin(),
1182 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
1183 ptr);
1184
1185 root->appendNewControlValue(
1186 proc, Return, Origin(),
1187 root->appendNew<MemoryValue>(proc, Load16Z, Origin(), ptr));
1188
1189 CHECK(compileAndRun<int32_t>(proc, value) == static_cast<uint16_t>(value));
1190}
1191
1192static void testSShrShl32(int32_t value, int32_t sshrAmount, int32_t shlAmount)
1193{
1194 Procedure proc;
1195 BasicBlock* root = proc.addBlock();
1196
1197 root->appendNewControlValue(
1198 proc, Return, Origin(),
1199 root->appendNew<Value>(
1200 proc, SShr, Origin(),
1201 root->appendNew<Value>(
1202 proc, Shl, Origin(),
1203 root->appendNew<Value>(
1204 proc, Trunc, Origin(),
1205 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
1206 root->appendNew<Const32Value>(proc, Origin(), shlAmount)),
1207 root->appendNew<Const32Value>(proc, Origin(), sshrAmount)));
1208
1209 CHECK(
1210 compileAndRun<int32_t>(proc, value)
1211 == ((value << (shlAmount & 31)) >> (sshrAmount & 31)));
1212}
1213
1214static void testSShrShl64(int64_t value, int32_t sshrAmount, int32_t shlAmount)
1215{
1216 Procedure proc;
1217 BasicBlock* root = proc.addBlock();
1218
1219 root->appendNewControlValue(
1220 proc, Return, Origin(),
1221 root->appendNew<Value>(
1222 proc, SShr, Origin(),
1223 root->appendNew<Value>(
1224 proc, Shl, Origin(),
1225 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1226 root->appendNew<Const32Value>(proc, Origin(), shlAmount)),
1227 root->appendNew<Const32Value>(proc, Origin(), sshrAmount)));
1228
1229 CHECK(
1230 compileAndRun<int64_t>(proc, value)
1231 == ((value << (shlAmount & 63)) >> (sshrAmount & 63)));
1232}
1233
1234void testTrivialInfiniteLoop()
1235{
1236 Procedure proc;
1237 BasicBlock* root = proc.addBlock();
1238 BasicBlock* loop = proc.addBlock();
1239 root->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(loop));
1240 loop->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(loop));
1241
1242 compileProc(proc);
1243}
1244
1245void testFoldPathEqual()
1246{
1247 Procedure proc;
1248 BasicBlock* root = proc.addBlock();
1249 BasicBlock* thenBlock = proc.addBlock();
1250 BasicBlock* elseBlock = proc.addBlock();
1251
1252 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1253
1254 root->appendNewControlValue(
1255 proc, Branch, Origin(), arg, FrequentedBlock(thenBlock), FrequentedBlock(elseBlock));
1256
1257 thenBlock->appendNewControlValue(
1258 proc, Return, Origin(),
1259 thenBlock->appendNew<Value>(
1260 proc, Equal, Origin(), arg, thenBlock->appendNew<ConstPtrValue>(proc, Origin(), 0)));
1261
1262 elseBlock->appendNewControlValue(
1263 proc, Return, Origin(),
1264 elseBlock->appendNew<Value>(
1265 proc, Equal, Origin(), arg, elseBlock->appendNew<ConstPtrValue>(proc, Origin(), 0)));
1266
1267 auto code = compileProc(proc);
1268 CHECK(invoke<intptr_t>(*code, 0) == 1);
1269 CHECK(invoke<intptr_t>(*code, 1) == 0);
1270 CHECK(invoke<intptr_t>(*code, 42) == 0);
1271}
1272
1273void testLShiftSelf32()
1274{
1275 Procedure proc;
1276 BasicBlock* root = proc.addBlock();
1277 Value* arg = root->appendNew<Value>(
1278 proc, Trunc, Origin(),
1279 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1280 root->appendNewControlValue(
1281 proc, Return, Origin(),
1282 root->appendNew<Value>(proc, Shl, Origin(), arg, arg));
1283
1284 auto code = compileProc(proc);
1285
1286 auto check = [&] (int32_t value) {
1287 CHECK(invoke<int32_t>(*code, value) == value << (value & 31));
1288 };
1289
1290 check(0);
1291 check(1);
1292 check(31);
1293 check(32);
1294}
1295
1296void testRShiftSelf32()
1297{
1298 Procedure proc;
1299 BasicBlock* root = proc.addBlock();
1300 Value* arg = root->appendNew<Value>(
1301 proc, Trunc, Origin(),
1302 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1303 root->appendNewControlValue(
1304 proc, Return, Origin(),
1305 root->appendNew<Value>(proc, SShr, Origin(), arg, arg));
1306
1307 auto code = compileProc(proc);
1308
1309 auto check = [&] (int32_t value) {
1310 CHECK(invoke<int32_t>(*code, value) == value >> (value & 31));
1311 };
1312
1313 check(0);
1314 check(1);
1315 check(31);
1316 check(32);
1317}
1318
1319void testURShiftSelf32()
1320{
1321 Procedure proc;
1322 BasicBlock* root = proc.addBlock();
1323 Value* arg = root->appendNew<Value>(
1324 proc, Trunc, Origin(),
1325 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1326 root->appendNewControlValue(
1327 proc, Return, Origin(),
1328 root->appendNew<Value>(proc, ZShr, Origin(), arg, arg));
1329
1330 auto code = compileProc(proc);
1331
1332 auto check = [&] (uint32_t value) {
1333 CHECK(invoke<uint32_t>(*code, value) == value >> (value & 31));
1334 };
1335
1336 check(0);
1337 check(1);
1338 check(31);
1339 check(32);
1340}
1341
1342void testLShiftSelf64()
1343{
1344 Procedure proc;
1345 BasicBlock* root = proc.addBlock();
1346 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1347 root->appendNewControlValue(
1348 proc, Return, Origin(),
1349 root->appendNew<Value>(
1350 proc, Shl, Origin(), arg, root->appendNew<Value>(proc, Trunc, Origin(), arg)));
1351
1352 auto code = compileProc(proc);
1353
1354 auto check = [&] (int64_t value) {
1355 CHECK(invoke<int64_t>(*code, value) == value << (value & 63));
1356 };
1357
1358 check(0);
1359 check(1);
1360 check(31);
1361 check(32);
1362 check(63);
1363 check(64);
1364}
1365
1366void testRShiftSelf64()
1367{
1368 Procedure proc;
1369 BasicBlock* root = proc.addBlock();
1370 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1371 root->appendNewControlValue(
1372 proc, Return, Origin(),
1373 root->appendNew<Value>(
1374 proc, SShr, Origin(), arg, root->appendNew<Value>(proc, Trunc, Origin(), arg)));
1375
1376 auto code = compileProc(proc);
1377
1378 auto check = [&] (int64_t value) {
1379 CHECK(invoke<int64_t>(*code, value) == value >> (value & 63));
1380 };
1381
1382 check(0);
1383 check(1);
1384 check(31);
1385 check(32);
1386 check(63);
1387 check(64);
1388}
1389
1390void testURShiftSelf64()
1391{
1392 Procedure proc;
1393 BasicBlock* root = proc.addBlock();
1394 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1395 root->appendNewControlValue(
1396 proc, Return, Origin(),
1397 root->appendNew<Value>(
1398 proc, ZShr, Origin(), arg, root->appendNew<Value>(proc, Trunc, Origin(), arg)));
1399
1400 auto code = compileProc(proc);
1401
1402 auto check = [&] (uint64_t value) {
1403 CHECK(invoke<uint64_t>(*code, value) == value >> (value & 63));
1404 };
1405
1406 check(0);
1407 check(1);
1408 check(31);
1409 check(32);
1410 check(63);
1411 check(64);
1412}
1413
1414void testPatchpointDoubleRegs()
1415{
1416 Procedure proc;
1417 BasicBlock* root = proc.addBlock();
1418
1419 Value* arg = root->appendNew<ArgumentRegValue>(proc, Origin(), FPRInfo::argumentFPR0);
1420
1421 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Double, Origin());
1422 patchpoint->append(arg, ValueRep(FPRInfo::fpRegT0));
1423 patchpoint->resultConstraints = { ValueRep(FPRInfo::fpRegT0) };
1424
1425 unsigned numCalls = 0;
1426 patchpoint->setGenerator(
1427 [&] (CCallHelpers&, const StackmapGenerationParams&) {
1428 numCalls++;
1429 });
1430
1431 root->appendNewControlValue(proc, Return, Origin(), patchpoint);
1432
1433 auto code = compileProc(proc);
1434 CHECK(numCalls == 1);
1435 CHECK(invoke<double>(*code, 42.5) == 42.5);
1436}
1437
1438void testSpillDefSmallerThanUse()
1439{
1440 Procedure proc;
1441 BasicBlock* root = proc.addBlock();
1442
1443 // Move32.
1444 Value* arg32 = root->appendNew<Value>(
1445 proc, Trunc, Origin(),
1446 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1447 Value* arg64 = root->appendNew<Value>(proc, ZExt32, Origin(), arg32);
1448
1449 // Make sure arg64 is on the stack.
1450 PatchpointValue* forceSpill = root->appendNew<PatchpointValue>(proc, Int64, Origin());
1451 RegisterSet clobberSet = RegisterSet::allGPRs();
1452 clobberSet.exclude(RegisterSet::stackRegisters());
1453 clobberSet.exclude(RegisterSet::reservedHardwareRegisters());
1454 clobberSet.clear(GPRInfo::returnValueGPR); // Force the return value for aliasing below.
1455 forceSpill->clobberLate(clobberSet);
1456 forceSpill->setGenerator(
1457 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1458 AllowMacroScratchRegisterUsage allowScratch(jit);
1459 jit.xor64(params[0].gpr(), params[0].gpr());
1460 });
1461
1462 // On x86, Sub admit an address for any operand. If it uses the stack, the top bits must be zero.
1463 Value* result = root->appendNew<Value>(proc, Sub, Origin(), forceSpill, arg64);
1464 root->appendNewControlValue(proc, Return, Origin(), result);
1465
1466 auto code = compileProc(proc);
1467 CHECK(invoke<int64_t>(*code, 0xffffffff00000000) == 0);
1468}
1469
1470void testSpillUseLargerThanDef()
1471{
1472 Procedure proc;
1473 BasicBlock* root = proc.addBlock();
1474 BasicBlock* thenCase = proc.addBlock();
1475 BasicBlock* elseCase = proc.addBlock();
1476 BasicBlock* tail = proc.addBlock();
1477
1478 RegisterSet clobberSet = RegisterSet::allGPRs();
1479 clobberSet.exclude(RegisterSet::stackRegisters());
1480 clobberSet.exclude(RegisterSet::reservedHardwareRegisters());
1481
1482 Value* condition = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1483 Value* argument = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1);
1484 root->appendNewControlValue(
1485 proc, Branch, Origin(),
1486 root->appendNew<Value>(
1487 proc, Trunc, Origin(),
1488 condition),
1489 FrequentedBlock(thenCase), FrequentedBlock(elseCase));
1490
1491 Value* truncated = thenCase->appendNew<Value>(proc, ZExt32, Origin(),
1492 thenCase->appendNew<Value>(proc, Trunc, Origin(), argument));
1493 UpsilonValue* thenResult = thenCase->appendNew<UpsilonValue>(proc, Origin(), truncated);
1494 thenCase->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(tail));
1495
1496 UpsilonValue* elseResult = elseCase->appendNew<UpsilonValue>(proc, Origin(), argument);
1497 elseCase->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(tail));
1498
1499 for (unsigned i = 0; i < 100; ++i) {
1500 PatchpointValue* preventTailDuplication = tail->appendNew<PatchpointValue>(proc, Void, Origin());
1501 preventTailDuplication->clobberLate(clobberSet);
1502 preventTailDuplication->setGenerator([] (CCallHelpers&, const StackmapGenerationParams&) { });
1503 }
1504
1505 PatchpointValue* forceSpill = tail->appendNew<PatchpointValue>(proc, Void, Origin());
1506 forceSpill->clobberLate(clobberSet);
1507 forceSpill->setGenerator(
1508 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
1509 AllowMacroScratchRegisterUsage allowScratch(jit);
1510 clobberSet.forEach([&] (Reg reg) {
1511 jit.move(CCallHelpers::TrustedImm64(0xffffffffffffffff), reg.gpr());
1512 });
1513 });
1514
1515 Value* phi = tail->appendNew<Value>(proc, Phi, Int64, Origin());
1516 thenResult->setPhi(phi);
1517 elseResult->setPhi(phi);
1518 tail->appendNewControlValue(proc, Return, Origin(), phi);
1519
1520 auto code = compileProc(proc);
1521 CHECK(invoke<uint64_t>(*code, 1, 0xffffffff00000000) == 0);
1522 CHECK(invoke<uint64_t>(*code, 0, 0xffffffff00000000) == 0xffffffff00000000);
1523
1524 // A second time since the previous run is still on the stack.
1525 CHECK(invoke<uint64_t>(*code, 1, 0xffffffff00000000) == 0);
1526
1527}
1528
1529void testLateRegister()
1530{
1531 Procedure proc;
1532
1533 if (!proc.optLevel()) {
1534 // FIXME: Make O0 handle such situations:
1535 // https://bugs.webkit.org/show_bug.cgi?id=194633
1536 return;
1537 }
1538
1539 BasicBlock* root = proc.addBlock();
1540
1541 // This works by making all but 1 register be input to the first patchpoint as LateRegister.
1542 // The other 1 register is just a regular Register input. We assert our result is the regular
1543 // register input. There would be no other way for the register allocator to arrange things
1544 // because LateRegister interferes with the result.
1545 // Then, the second patchpoint takes the result of the first as an argument and asks for
1546 // it in a register that was a LateRegister. This is to incentivize the register allocator
1547 // to use that LateRegister as the result for the first patchpoint. But of course it can not do that.
1548 // So it must issue a mov after the first patchpoint from the first's result into the second's input.
1549
1550 RegisterSet regs = RegisterSet::allGPRs();
1551 regs.exclude(RegisterSet::stackRegisters());
1552 regs.exclude(RegisterSet::reservedHardwareRegisters());
1553 Vector<Value*> lateUseArgs;
1554 unsigned result = 0;
1555 for (GPRReg reg = CCallHelpers::firstRegister(); reg <= CCallHelpers::lastRegister(); reg = CCallHelpers::nextRegister(reg)) {
1556 if (!regs.get(reg))
1557 continue;
1558 result++;
1559 if (reg == GPRInfo::regT0)
1560 continue;
1561 Value* value = root->appendNew<Const64Value>(proc, Origin(), 1);
1562 lateUseArgs.append(value);
1563 }
1564 Value* regularUse = root->appendNew<Const64Value>(proc, Origin(), 1);
1565 PatchpointValue* firstPatchpoint = root->appendNew<PatchpointValue>(proc, Int64, Origin());
1566 {
1567 unsigned i = 0;
1568 for (GPRReg reg = CCallHelpers::firstRegister(); reg <= CCallHelpers::lastRegister(); reg = CCallHelpers::nextRegister(reg)) {
1569 if (!regs.get(reg))
1570 continue;
1571 if (reg == GPRInfo::regT0)
1572 continue;
1573 Value* value = lateUseArgs[i++];
1574 firstPatchpoint->append(value, ValueRep::lateReg(reg));
1575 }
1576 firstPatchpoint->append(regularUse, ValueRep::reg(GPRInfo::regT0));
1577 }
1578
1579 firstPatchpoint->setGenerator(
1580 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1581 AllowMacroScratchRegisterUsage allowScratch(jit);
1582 CHECK(params[0].gpr() == GPRInfo::regT0);
1583 // Note that regT0 should also start off as 1, so we're implicitly starting our add with 1, which is also an argument.
1584 unsigned skipped = 0;
1585 for (unsigned i = 1; i < params.size(); i++) {
1586 if (params[i].gpr() == params[0].gpr()) {
1587 skipped = i;
1588 continue;
1589 }
1590 jit.add64(params[i].gpr(), params[0].gpr());
1591 }
1592 CHECK(!!skipped);
1593 });
1594
1595 PatchpointValue* secondPatchpoint = root->appendNew<PatchpointValue>(proc, Int64, Origin());
1596 secondPatchpoint->append(firstPatchpoint, ValueRep::reg(GPRInfo::regT1));
1597 secondPatchpoint->setGenerator(
1598 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1599 AllowMacroScratchRegisterUsage allowScratch(jit);
1600 CHECK(params[1].gpr() == GPRInfo::regT1);
1601 jit.nop();
1602 jit.nop();
1603 jit.move(params[1].gpr(), params[0].gpr());
1604 jit.nop();
1605 jit.nop();
1606 });
1607 root->appendNewControlValue(proc, Return, Origin(), secondPatchpoint);
1608
1609 auto code = compileProc(proc);
1610 CHECK(invoke<uint64_t>(*code) == result);
1611}
1612
1613void interpreterPrint(Vector<intptr_t>* stream, intptr_t value)
1614{
1615 stream->append(value);
1616}
1617
1618void testInterpreter()
1619{
1620 // This implements a silly interpreter to test building custom switch statements using
1621 // Patchpoint.
1622
1623 Procedure proc;
1624
1625 BasicBlock* root = proc.addBlock();
1626 BasicBlock* dispatch = proc.addBlock();
1627 BasicBlock* addToDataPointer = proc.addBlock();
1628 BasicBlock* addToCodePointer = proc.addBlock();
1629 BasicBlock* addToCodePointerTaken = proc.addBlock();
1630 BasicBlock* addToCodePointerNotTaken = proc.addBlock();
1631 BasicBlock* addToData = proc.addBlock();
1632 BasicBlock* print = proc.addBlock();
1633 BasicBlock* stop = proc.addBlock();
1634
1635 Variable* dataPointer = proc.addVariable(pointerType());
1636 Variable* codePointer = proc.addVariable(pointerType());
1637
1638 root->appendNew<VariableValue>(
1639 proc, Set, Origin(), dataPointer,
1640 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
1641 root->appendNew<VariableValue>(
1642 proc, Set, Origin(), codePointer,
1643 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1));
1644 Value* context = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2);
1645 root->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(dispatch));
1646
1647 // NOTE: It's totally valid for this patchpoint to be tail-duplicated.
1648 Value* codePointerValue =
1649 dispatch->appendNew<VariableValue>(proc, B3::Get, Origin(), codePointer);
1650 Value* opcode = dispatch->appendNew<MemoryValue>(
1651 proc, Load, pointerType(), Origin(), codePointerValue);
1652 PatchpointValue* polyJump = dispatch->appendNew<PatchpointValue>(proc, Void, Origin());
1653 polyJump->effects = Effects();
1654 polyJump->effects.terminal = true;
1655 polyJump->appendSomeRegister(opcode);
1656 polyJump->clobber(RegisterSet::macroScratchRegisters());
1657 polyJump->numGPScratchRegisters = 2;
1658 dispatch->appendSuccessor(FrequentedBlock(addToDataPointer));
1659 dispatch->appendSuccessor(FrequentedBlock(addToCodePointer));
1660 dispatch->appendSuccessor(FrequentedBlock(addToData));
1661 dispatch->appendSuccessor(FrequentedBlock(print));
1662 dispatch->appendSuccessor(FrequentedBlock(stop));
1663
1664 // Our "opcodes".
1665 static constexpr intptr_t AddDP = 0;
1666 static constexpr intptr_t AddCP = 1;
1667 static constexpr intptr_t Add = 2;
1668 static constexpr intptr_t Print = 3;
1669 static constexpr intptr_t Stop = 4;
1670
1671 polyJump->setGenerator(
1672 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1673 AllowMacroScratchRegisterUsage allowScratch(jit);
1674 Vector<Box<CCallHelpers::Label>> labels = params.successorLabels();
1675
1676 MacroAssemblerCodePtr<B3CompilationPtrTag>* jumpTable = bitwise_cast<MacroAssemblerCodePtr<B3CompilationPtrTag>*>(
1677 params.proc().addDataSection(sizeof(MacroAssemblerCodePtr<B3CompilationPtrTag>) * labels.size()));
1678
1679 GPRReg scratch = params.gpScratch(0);
1680
1681 jit.move(CCallHelpers::TrustedImmPtr(jumpTable), scratch);
1682 jit.load64(CCallHelpers::BaseIndex(scratch, params[0].gpr(), CCallHelpers::timesPtr()), scratch);
1683 jit.farJump(scratch, B3CompilationPtrTag);
1684
1685 jit.addLinkTask(
1686 [&, jumpTable, labels] (LinkBuffer& linkBuffer) {
1687 for (unsigned i = labels.size(); i--;)
1688 jumpTable[i] = linkBuffer.locationOf<B3CompilationPtrTag>(*labels[i]);
1689 });
1690 });
1691
1692 // AddDP <operand>: adds <operand> to DP.
1693 codePointerValue =
1694 addToDataPointer->appendNew<VariableValue>(proc, B3::Get, Origin(), codePointer);
1695 addToDataPointer->appendNew<VariableValue>(
1696 proc, Set, Origin(), dataPointer,
1697 addToDataPointer->appendNew<Value>(
1698 proc, B3::Add, Origin(),
1699 addToDataPointer->appendNew<VariableValue>(proc, B3::Get, Origin(), dataPointer),
1700 addToDataPointer->appendNew<Value>(
1701 proc, Mul, Origin(),
1702 addToDataPointer->appendNew<MemoryValue>(
1703 proc, Load, pointerType(), Origin(), codePointerValue, static_cast<int32_t>(sizeof(intptr_t))),
1704 addToDataPointer->appendIntConstant(
1705 proc, Origin(), pointerType(), sizeof(intptr_t)))));
1706 addToDataPointer->appendNew<VariableValue>(
1707 proc, Set, Origin(), codePointer,
1708 addToDataPointer->appendNew<Value>(
1709 proc, B3::Add, Origin(), codePointerValue,
1710 addToDataPointer->appendIntConstant(
1711 proc, Origin(), pointerType(), sizeof(intptr_t) * 2)));
1712 addToDataPointer->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(dispatch));
1713
1714 // AddCP <operand>: adds <operand> to CP if the current value at DP is non-zero, otherwise
1715 // falls through normally.
1716 codePointerValue =
1717 addToCodePointer->appendNew<VariableValue>(proc, B3::Get, Origin(), codePointer);
1718 Value* dataPointerValue =
1719 addToCodePointer->appendNew<VariableValue>(proc, B3::Get, Origin(), dataPointer);
1720 addToCodePointer->appendNewControlValue(
1721 proc, Branch, Origin(),
1722 addToCodePointer->appendNew<MemoryValue>(
1723 proc, Load, pointerType(), Origin(), dataPointerValue),
1724 FrequentedBlock(addToCodePointerTaken), FrequentedBlock(addToCodePointerNotTaken));
1725 addToCodePointerTaken->appendNew<VariableValue>(
1726 proc, Set, Origin(), codePointer,
1727 addToCodePointerTaken->appendNew<Value>(
1728 proc, B3::Add, Origin(), codePointerValue,
1729 addToCodePointerTaken->appendNew<Value>(
1730 proc, Mul, Origin(),
1731 addToCodePointerTaken->appendNew<MemoryValue>(
1732 proc, Load, pointerType(), Origin(), codePointerValue, static_cast<int32_t>(sizeof(intptr_t))),
1733 addToCodePointerTaken->appendIntConstant(
1734 proc, Origin(), pointerType(), sizeof(intptr_t)))));
1735 addToCodePointerTaken->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(dispatch));
1736 addToCodePointerNotTaken->appendNew<VariableValue>(
1737 proc, Set, Origin(), codePointer,
1738 addToCodePointerNotTaken->appendNew<Value>(
1739 proc, B3::Add, Origin(), codePointerValue,
1740 addToCodePointerNotTaken->appendIntConstant(
1741 proc, Origin(), pointerType(), sizeof(intptr_t) * 2)));
1742 addToCodePointerNotTaken->appendNewControlValue(
1743 proc, Jump, Origin(), FrequentedBlock(dispatch));
1744
1745 // Add <operand>: adds <operand> to the slot pointed to by DP.
1746 codePointerValue = addToData->appendNew<VariableValue>(proc, B3::Get, Origin(), codePointer);
1747 dataPointerValue = addToData->appendNew<VariableValue>(proc, B3::Get, Origin(), dataPointer);
1748 addToData->appendNew<MemoryValue>(
1749 proc, Store, Origin(),
1750 addToData->appendNew<Value>(
1751 proc, B3::Add, Origin(),
1752 addToData->appendNew<MemoryValue>(
1753 proc, Load, pointerType(), Origin(), dataPointerValue),
1754 addToData->appendNew<MemoryValue>(
1755 proc, Load, pointerType(), Origin(), codePointerValue, static_cast<int32_t>(sizeof(intptr_t)))),
1756 dataPointerValue);
1757 addToData->appendNew<VariableValue>(
1758 proc, Set, Origin(), codePointer,
1759 addToData->appendNew<Value>(
1760 proc, B3::Add, Origin(), codePointerValue,
1761 addToData->appendIntConstant(proc, Origin(), pointerType(), sizeof(intptr_t) * 2)));
1762 addToData->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(dispatch));
1763
1764 // Print: "prints" the value pointed to by DP. What this actually means is that the value is
1765 // appended to the stream vector by the interpreterPrint function.
1766 codePointerValue = print->appendNew<VariableValue>(proc, B3::Get, Origin(), codePointer);
1767 dataPointerValue = print->appendNew<VariableValue>(proc, B3::Get, Origin(), dataPointer);
1768 print->appendNew<CCallValue>(
1769 proc, Void, Origin(),
1770 print->appendNew<ConstPtrValue>(
1771 proc, Origin(), tagCFunctionPtr<void*>(interpreterPrint, B3CCallPtrTag)),
1772 context,
1773 print->appendNew<MemoryValue>(proc, Load, pointerType(), Origin(), dataPointerValue));
1774 print->appendNew<VariableValue>(
1775 proc, Set, Origin(), codePointer,
1776 print->appendNew<Value>(
1777 proc, B3::Add, Origin(), codePointerValue,
1778 print->appendIntConstant(proc, Origin(), pointerType(), sizeof(intptr_t))));
1779 print->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(dispatch));
1780
1781 // Stop: returns.
1782 stop->appendNewControlValue(
1783 proc, Return, Origin(),
1784 stop->appendIntConstant(proc, Origin(), pointerType(), 0));
1785
1786 auto interpreter = compileProc(proc);
1787
1788 Vector<uintptr_t> data;
1789 Vector<uintptr_t> code;
1790 Vector<uintptr_t> stream;
1791
1792 data.append(1);
1793 data.append(0);
1794
1795 if (shouldBeVerbose())
1796 dataLog("data = ", listDump(data), "\n");
1797
1798 // We'll write a program that prints the numbers 1..100.
1799 // We expect DP to point at #0.
1800 code.append(AddCP);
1801 code.append(6); // go to loop body
1802
1803 // Loop re-entry:
1804 // We expect DP to point at #1 and for #1 to be offset by -100.
1805 code.append(Add);
1806 code.append(100);
1807
1808 code.append(AddDP);
1809 code.append(-1);
1810
1811 // Loop header:
1812 // We expect DP to point at #0.
1813 code.append(AddDP);
1814 code.append(1);
1815
1816 code.append(Add);
1817 code.append(1);
1818
1819 code.append(Print);
1820
1821 code.append(Add);
1822 code.append(-100);
1823
1824 // We want to stop if it's zero and continue if it's non-zero. AddCP takes the branch if it's
1825 // non-zero.
1826 code.append(AddCP);
1827 code.append(-11); // go to loop re-entry.
1828
1829 code.append(Stop);
1830
1831 if (shouldBeVerbose())
1832 dataLog("code = ", listDump(code), "\n");
1833
1834 CHECK(!invoke<intptr_t>(*interpreter, data.data(), code.data(), &stream));
1835
1836 CHECK(stream.size() == 100);
1837 for (unsigned i = 0; i < 100; ++i)
1838 CHECK(stream[i] == i + 1);
1839
1840 if (shouldBeVerbose())
1841 dataLog("stream = ", listDump(stream), "\n");
1842}
1843
1844void testReduceStrengthCheckBottomUseInAnotherBlock()
1845{
1846 Procedure proc;
1847 if (proc.optLevel() < 1)
1848 return;
1849
1850 BasicBlock* one = proc.addBlock();
1851 BasicBlock* two = proc.addBlock();
1852
1853 CheckValue* check = one->appendNew<CheckValue>(
1854 proc, Check, Origin(), one->appendNew<Const32Value>(proc, Origin(), 1));
1855 check->setGenerator(
1856 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
1857 AllowMacroScratchRegisterUsage allowScratch(jit);
1858
1859 jit.move(CCallHelpers::TrustedImm32(666), GPRInfo::returnValueGPR);
1860 jit.emitFunctionEpilogue();
1861 jit.ret();
1862 });
1863 Value* arg = one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
1864 one->appendNewControlValue(proc, Jump, Origin(), FrequentedBlock(two));
1865
1866 check = two->appendNew<CheckValue>(
1867 proc, CheckAdd, Origin(), arg,
1868 two->appendNew<ConstPtrValue>(proc, Origin(), 1));
1869 check->setGenerator(
1870 [&] (CCallHelpers&, const StackmapGenerationParams&) {
1871 CHECK(!"Should not execute");
1872 });
1873 two->appendNewControlValue(proc, Return, Origin(), check);
1874
1875 proc.resetReachability();
1876 reduceStrength(proc);
1877}
1878
1879void testResetReachabilityDanglingReference()
1880{
1881 Procedure proc;
1882
1883 BasicBlock* one = proc.addBlock();
1884 BasicBlock* two = proc.addBlock();
1885
1886 UpsilonValue* upsilon = one->appendNew<UpsilonValue>(
1887 proc, Origin(), one->appendNew<Const32Value>(proc, Origin(), 42));
1888 one->appendNewControlValue(proc, Oops, Origin());
1889
1890 Value* phi = two->appendNew<Value>(proc, Phi, Int32, Origin());
1891 upsilon->setPhi(phi);
1892 two->appendNewControlValue(proc, Oops, Origin());
1893
1894 proc.resetReachability();
1895 validate(proc);
1896}
1897
1898void testEntrySwitchSimple()
1899{
1900 Procedure proc;
1901 proc.setNumEntrypoints(3);
1902
1903 BasicBlock* root = proc.addBlock();
1904 BasicBlock* one = proc.addBlock();
1905 BasicBlock* two = proc.addBlock();
1906 BasicBlock* three = proc.addBlock();
1907
1908 root->appendNew<Value>(proc, EntrySwitch, Origin());
1909 root->appendSuccessor(FrequentedBlock(one));
1910 root->appendSuccessor(FrequentedBlock(two));
1911 root->appendSuccessor(FrequentedBlock(three));
1912
1913 one->appendNew<Value>(
1914 proc, Return, Origin(),
1915 one->appendNew<Value>(
1916 proc, Add, Origin(),
1917 one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1918 one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
1919
1920 two->appendNew<Value>(
1921 proc, Return, Origin(),
1922 two->appendNew<Value>(
1923 proc, Sub, Origin(),
1924 two->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1925 two->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
1926
1927 three->appendNew<Value>(
1928 proc, Return, Origin(),
1929 three->appendNew<Value>(
1930 proc, Mul, Origin(),
1931 three->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1932 three->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
1933
1934 prepareForGeneration(proc);
1935
1936 CCallHelpers jit;
1937 generate(proc, jit);
1938 LinkBuffer linkBuffer(jit, nullptr);
1939 CodeLocationLabel<B3CompilationPtrTag> labelOne = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(0));
1940 CodeLocationLabel<B3CompilationPtrTag> labelTwo = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(1));
1941 CodeLocationLabel<B3CompilationPtrTag> labelThree = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(2));
1942
1943 MacroAssemblerCodeRef<B3CompilationPtrTag> codeRef = FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testb3 compilation");
1944
1945 CHECK(invoke<int>(labelOne, 1, 2) == 3);
1946 CHECK(invoke<int>(labelTwo, 1, 2) == -1);
1947 CHECK(invoke<int>(labelThree, 1, 2) == 2);
1948 CHECK(invoke<int>(labelOne, -1, 2) == 1);
1949 CHECK(invoke<int>(labelTwo, -1, 2) == -3);
1950 CHECK(invoke<int>(labelThree, -1, 2) == -2);
1951}
1952
1953void testEntrySwitchNoEntrySwitch()
1954{
1955 Procedure proc;
1956 proc.setNumEntrypoints(3);
1957
1958 BasicBlock* root = proc.addBlock();
1959
1960 root->appendNew<Value>(
1961 proc, Return, Origin(),
1962 root->appendNew<Value>(
1963 proc, Add, Origin(),
1964 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0),
1965 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1)));
1966
1967 prepareForGeneration(proc);
1968
1969 CCallHelpers jit;
1970 generate(proc, jit);
1971 LinkBuffer linkBuffer(jit, nullptr);
1972 CodeLocationLabel<B3CompilationPtrTag> labelOne = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(0));
1973 CodeLocationLabel<B3CompilationPtrTag> labelTwo = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(1));
1974 CodeLocationLabel<B3CompilationPtrTag> labelThree = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(2));
1975
1976 MacroAssemblerCodeRef<B3CompilationPtrTag> codeRef = FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testb3 compilation");
1977
1978 CHECK_EQ(invoke<int>(labelOne, 1, 2), 3);
1979 CHECK_EQ(invoke<int>(labelTwo, 1, 2), 3);
1980 CHECK_EQ(invoke<int>(labelThree, 1, 2), 3);
1981 CHECK_EQ(invoke<int>(labelOne, -1, 2), 1);
1982 CHECK_EQ(invoke<int>(labelTwo, -1, 2), 1);
1983 CHECK_EQ(invoke<int>(labelThree, -1, 2), 1);
1984}
1985
1986void testEntrySwitchWithCommonPaths()
1987{
1988 Procedure proc;
1989 proc.setNumEntrypoints(3);
1990
1991 BasicBlock* root = proc.addBlock();
1992 BasicBlock* one = proc.addBlock();
1993 BasicBlock* two = proc.addBlock();
1994 BasicBlock* three = proc.addBlock();
1995 BasicBlock* end = proc.addBlock();
1996
1997 root->appendNew<Value>(proc, EntrySwitch, Origin());
1998 root->appendSuccessor(FrequentedBlock(one));
1999 root->appendSuccessor(FrequentedBlock(two));
2000 root->appendSuccessor(FrequentedBlock(three));
2001
2002 UpsilonValue* upsilonOne = one->appendNew<UpsilonValue>(
2003 proc, Origin(),
2004 one->appendNew<Value>(
2005 proc, Add, Origin(),
2006 one->appendNew<Value>(
2007 proc, Trunc, Origin(),
2008 one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
2009 one->appendNew<Value>(
2010 proc, Trunc, Origin(),
2011 one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2012 one->appendNew<Value>(proc, Jump, Origin());
2013 one->setSuccessors(FrequentedBlock(end));
2014
2015 UpsilonValue* upsilonTwo = two->appendNew<UpsilonValue>(
2016 proc, Origin(),
2017 two->appendNew<Value>(
2018 proc, Sub, Origin(),
2019 two->appendNew<Value>(
2020 proc, Trunc, Origin(),
2021 two->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
2022 two->appendNew<Value>(
2023 proc, Trunc, Origin(),
2024 two->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2025 two->appendNew<Value>(proc, Jump, Origin());
2026 two->setSuccessors(FrequentedBlock(end));
2027
2028 UpsilonValue* upsilonThree = three->appendNew<UpsilonValue>(
2029 proc, Origin(),
2030 three->appendNew<Value>(
2031 proc, Mul, Origin(),
2032 three->appendNew<Value>(
2033 proc, Trunc, Origin(),
2034 three->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)),
2035 three->appendNew<Value>(
2036 proc, Trunc, Origin(),
2037 three->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2038 three->appendNew<Value>(proc, Jump, Origin());
2039 three->setSuccessors(FrequentedBlock(end));
2040
2041 Value* phi = end->appendNew<Value>(proc, Phi, Int32, Origin());
2042 upsilonOne->setPhi(phi);
2043 upsilonTwo->setPhi(phi);
2044 upsilonThree->setPhi(phi);
2045
2046 end->appendNew<Value>(
2047 proc, Return, Origin(),
2048 end->appendNew<Value>(
2049 proc, chill(Mod), Origin(),
2050 phi, end->appendNew<Value>(
2051 proc, Trunc, Origin(),
2052 end->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2))));
2053
2054 prepareForGeneration(proc);
2055
2056 CCallHelpers jit;
2057 generate(proc, jit);
2058 LinkBuffer linkBuffer(jit, nullptr);
2059 CodeLocationLabel<B3CompilationPtrTag> labelOne = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(0));
2060 CodeLocationLabel<B3CompilationPtrTag> labelTwo = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(1));
2061 CodeLocationLabel<B3CompilationPtrTag> labelThree = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(2));
2062
2063 MacroAssemblerCodeRef<B3CompilationPtrTag> codeRef = FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testb3 compilation");
2064
2065 CHECK_EQ(invoke<int>(labelOne, 1, 2, 10), 3);
2066 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 10), -1);
2067 CHECK_EQ(invoke<int>(labelThree, 1, 2, 10), 2);
2068 CHECK_EQ(invoke<int>(labelOne, -1, 2, 10), 1);
2069 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 10), -3);
2070 CHECK_EQ(invoke<int>(labelThree, -1, 2, 10), -2);
2071 CHECK_EQ(invoke<int>(labelOne, 1, 2, 2), 1);
2072 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 2), -1);
2073 CHECK_EQ(invoke<int>(labelThree, 1, 2, 2), 0);
2074 CHECK_EQ(invoke<int>(labelOne, -1, 2, 2), 1);
2075 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 2), -1);
2076 CHECK_EQ(invoke<int>(labelThree, -1, 2, 2), 0);
2077 CHECK_EQ(invoke<int>(labelOne, 1, 2, 0), 0);
2078 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 0), 0);
2079 CHECK_EQ(invoke<int>(labelThree, 1, 2, 0), 0);
2080 CHECK_EQ(invoke<int>(labelOne, -1, 2, 0), 0);
2081 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 0), 0);
2082 CHECK_EQ(invoke<int>(labelThree, -1, 2, 0), 0);
2083}
2084
2085void testEntrySwitchWithCommonPathsAndNonTrivialEntrypoint()
2086{
2087 Procedure proc;
2088 proc.setNumEntrypoints(3);
2089
2090 BasicBlock* root = proc.addBlock();
2091 BasicBlock* negate = proc.addBlock();
2092 BasicBlock* dispatch = proc.addBlock();
2093 BasicBlock* one = proc.addBlock();
2094 BasicBlock* two = proc.addBlock();
2095 BasicBlock* three = proc.addBlock();
2096 BasicBlock* end = proc.addBlock();
2097
2098 UpsilonValue* upsilonBase = root->appendNew<UpsilonValue>(
2099 proc, Origin(), root->appendNew<Value>(
2100 proc, Trunc, Origin(),
2101 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
2102 root->appendNew<Value>(
2103 proc, Branch, Origin(),
2104 root->appendNew<Value>(
2105 proc, BitAnd, Origin(),
2106 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR3),
2107 root->appendNew<ConstPtrValue>(proc, Origin(), 0xff)));
2108 root->setSuccessors(FrequentedBlock(negate), FrequentedBlock(dispatch));
2109
2110 UpsilonValue* upsilonNegate = negate->appendNew<UpsilonValue>(
2111 proc, Origin(),
2112 negate->appendNew<Value>(
2113 proc, Neg, Origin(),
2114 negate->appendNew<Value>(
2115 proc, Trunc, Origin(),
2116 negate->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0))));
2117 negate->appendNew<Value>(proc, Jump, Origin());
2118 negate->setSuccessors(FrequentedBlock(dispatch));
2119
2120 Value* arg0 = dispatch->appendNew<Value>(proc, Phi, Int32, Origin());
2121 upsilonBase->setPhi(arg0);
2122 upsilonNegate->setPhi(arg0);
2123 dispatch->appendNew<Value>(proc, EntrySwitch, Origin());
2124 dispatch->appendSuccessor(FrequentedBlock(one));
2125 dispatch->appendSuccessor(FrequentedBlock(two));
2126 dispatch->appendSuccessor(FrequentedBlock(three));
2127
2128 UpsilonValue* upsilonOne = one->appendNew<UpsilonValue>(
2129 proc, Origin(),
2130 one->appendNew<Value>(
2131 proc, Add, Origin(),
2132 arg0, one->appendNew<Value>(
2133 proc, Trunc, Origin(),
2134 one->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2135 one->appendNew<Value>(proc, Jump, Origin());
2136 one->setSuccessors(FrequentedBlock(end));
2137
2138 UpsilonValue* upsilonTwo = two->appendNew<UpsilonValue>(
2139 proc, Origin(),
2140 two->appendNew<Value>(
2141 proc, Sub, Origin(),
2142 arg0, two->appendNew<Value>(
2143 proc, Trunc, Origin(),
2144 two->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2145 two->appendNew<Value>(proc, Jump, Origin());
2146 two->setSuccessors(FrequentedBlock(end));
2147
2148 UpsilonValue* upsilonThree = three->appendNew<UpsilonValue>(
2149 proc, Origin(),
2150 three->appendNew<Value>(
2151 proc, Mul, Origin(),
2152 arg0, three->appendNew<Value>(
2153 proc, Trunc, Origin(),
2154 three->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR1))));
2155 three->appendNew<Value>(proc, Jump, Origin());
2156 three->setSuccessors(FrequentedBlock(end));
2157
2158 Value* phi = end->appendNew<Value>(proc, Phi, Int32, Origin());
2159 upsilonOne->setPhi(phi);
2160 upsilonTwo->setPhi(phi);
2161 upsilonThree->setPhi(phi);
2162
2163 end->appendNew<Value>(
2164 proc, Return, Origin(),
2165 end->appendNew<Value>(
2166 proc, chill(Mod), Origin(),
2167 phi, end->appendNew<Value>(
2168 proc, Trunc, Origin(),
2169 end->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR2))));
2170
2171 prepareForGeneration(proc);
2172
2173 CCallHelpers jit;
2174 generate(proc, jit);
2175 LinkBuffer linkBuffer(jit, nullptr);
2176 CodeLocationLabel<B3CompilationPtrTag> labelOne = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(0));
2177 CodeLocationLabel<B3CompilationPtrTag> labelTwo = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(1));
2178 CodeLocationLabel<B3CompilationPtrTag> labelThree = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(2));
2179
2180 MacroAssemblerCodeRef<B3CompilationPtrTag> codeRef = FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testb3 compilation");
2181
2182 CHECK_EQ(invoke<int>(labelOne, 1, 2, 10, false), 3);
2183 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 10, false), -1);
2184 CHECK_EQ(invoke<int>(labelThree, 1, 2, 10, false), 2);
2185 CHECK_EQ(invoke<int>(labelOne, -1, 2, 10, false), 1);
2186 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 10, false), -3);
2187 CHECK_EQ(invoke<int>(labelThree, -1, 2, 10, false), -2);
2188 CHECK_EQ(invoke<int>(labelOne, 1, 2, 10, true), 1);
2189 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 10, true), -3);
2190 CHECK_EQ(invoke<int>(labelThree, 1, 2, 10, true), -2);
2191 CHECK_EQ(invoke<int>(labelOne, -1, 2, 10, true), 3);
2192 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 10, true), -1);
2193 CHECK_EQ(invoke<int>(labelThree, -1, 2, 10, true), 2);
2194 CHECK_EQ(invoke<int>(labelOne, 1, 2, 2, false), 1);
2195 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 2, false), -1);
2196 CHECK_EQ(invoke<int>(labelThree, 1, 2, 2, false), 0);
2197 CHECK_EQ(invoke<int>(labelOne, -1, 2, 2, false), 1);
2198 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 2, false), -1);
2199 CHECK_EQ(invoke<int>(labelThree, -1, 2, 2, false), 0);
2200 CHECK_EQ(invoke<int>(labelOne, 1, 2, 0, false), 0);
2201 CHECK_EQ(invoke<int>(labelTwo, 1, 2, 0, false), 0);
2202 CHECK_EQ(invoke<int>(labelThree, 1, 2, 0, false), 0);
2203 CHECK_EQ(invoke<int>(labelOne, -1, 2, 0, false), 0);
2204 CHECK_EQ(invoke<int>(labelTwo, -1, 2, 0, false), 0);
2205 CHECK_EQ(invoke<int>(labelThree, -1, 2, 0, false), 0);
2206}
2207
2208void testEntrySwitchLoop()
2209{
2210 // This is a completely absurd use of EntrySwitch, where it impacts the loop condition. This
2211 // should cause duplication of either nearly the entire Procedure. At time of writing, we ended
2212 // up duplicating all of it, which is fine. It's important to test this case, to make sure that
2213 // the duplication algorithm can handle interesting control flow.
2214
2215 Procedure proc;
2216 proc.setNumEntrypoints(2);
2217
2218 BasicBlock* root = proc.addBlock();
2219 BasicBlock* loopHeader = proc.addBlock();
2220 BasicBlock* loopFooter = proc.addBlock();
2221 BasicBlock* end = proc.addBlock();
2222
2223 UpsilonValue* initialValue = root->appendNew<UpsilonValue>(
2224 proc, Origin(), root->appendNew<Value>(
2225 proc, Trunc, Origin(),
2226 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0)));
2227 root->appendNew<Value>(proc, Jump, Origin());
2228 root->setSuccessors(loopHeader);
2229
2230 Value* valueInLoop = loopHeader->appendNew<Value>(proc, Phi, Int32, Origin());
2231 initialValue->setPhi(valueInLoop);
2232 Value* newValue = loopHeader->appendNew<Value>(
2233 proc, Add, Origin(), valueInLoop,
2234 loopHeader->appendNew<Const32Value>(proc, Origin(), 1));
2235 loopHeader->appendNew<Value>(proc, EntrySwitch, Origin());
2236 loopHeader->appendSuccessor(end);
2237 loopHeader->appendSuccessor(loopFooter);
2238
2239 loopFooter->appendNew<UpsilonValue>(proc, Origin(), newValue, valueInLoop);
2240 loopFooter->appendNew<Value>(
2241 proc, Branch, Origin(),
2242 loopFooter->appendNew<Value>(
2243 proc, LessThan, Origin(), newValue,
2244 loopFooter->appendNew<Const32Value>(proc, Origin(), 100)));
2245 loopFooter->setSuccessors(loopHeader, end);
2246
2247 end->appendNew<Value>(proc, Return, Origin(), newValue);
2248
2249 prepareForGeneration(proc);
2250
2251 CCallHelpers jit;
2252 generate(proc, jit);
2253 LinkBuffer linkBuffer(jit, nullptr);
2254 CodeLocationLabel<B3CompilationPtrTag> labelOne = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(0));
2255 CodeLocationLabel<B3CompilationPtrTag> labelTwo = linkBuffer.locationOf<B3CompilationPtrTag>(proc.entrypointLabel(1));
2256
2257 MacroAssemblerCodeRef<B3CompilationPtrTag> codeRef = FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "testb3 compilation");
2258
2259 CHECK(invoke<int>(labelOne, 0) == 1);
2260 CHECK(invoke<int>(labelOne, 42) == 43);
2261 CHECK(invoke<int>(labelOne, 1000) == 1001);
2262
2263 CHECK(invoke<int>(labelTwo, 0) == 100);
2264 CHECK(invoke<int>(labelTwo, 42) == 100);
2265 CHECK(invoke<int>(labelTwo, 1000) == 1001);
2266}
2267
2268void testSomeEarlyRegister()
2269{
2270 auto run = [&] (bool succeed) {
2271 Procedure proc;
2272
2273 BasicBlock* root = proc.addBlock();
2274
2275 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
2276 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
2277 bool ranFirstPatchpoint = false;
2278 patchpoint->setGenerator(
2279 [&] (CCallHelpers&, const StackmapGenerationParams& params) {
2280 CHECK(params[0].gpr() == GPRInfo::returnValueGPR);
2281 ranFirstPatchpoint = true;
2282 });
2283
2284 Value* arg = patchpoint;
2285
2286 patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
2287 patchpoint->appendSomeRegister(arg);
2288 if (succeed)
2289 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
2290 bool ranSecondPatchpoint = false;
2291 unsigned optLevel = proc.optLevel();
2292 patchpoint->setGenerator(
2293 [&] (CCallHelpers&, const StackmapGenerationParams& params) {
2294 if (succeed)
2295 CHECK(params[0].gpr() != params[1].gpr());
2296 else if (optLevel > 1)
2297 CHECK(params[0].gpr() == params[1].gpr());
2298 ranSecondPatchpoint = true;
2299 });
2300
2301 root->appendNew<Value>(proc, Return, Origin(), patchpoint);
2302
2303 compileProc(proc);
2304 CHECK(ranFirstPatchpoint);
2305 CHECK(ranSecondPatchpoint);
2306 };
2307
2308 run(true);
2309 run(false);
2310}
2311
2312void testBranchBitAndImmFusion(
2313 B3::Opcode valueModifier, Type valueType, int64_t constant,
2314 Air::Opcode expectedOpcode, Air::Arg::Kind firstKind)
2315{
2316 // Currently this test should pass on all CPUs. But some CPUs may not support this fused
2317 // instruction. It's OK to skip this test on those CPUs.
2318
2319 Procedure proc;
2320
2321 BasicBlock* root = proc.addBlock();
2322 BasicBlock* one = proc.addBlock();
2323 BasicBlock* two = proc.addBlock();
2324
2325 Value* left = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
2326
2327 if (valueModifier != Identity) {
2328 if (MemoryValue::accepts(valueModifier))
2329 left = root->appendNew<MemoryValue>(proc, valueModifier, valueType, Origin(), left);
2330 else
2331 left = root->appendNew<Value>(proc, valueModifier, valueType, Origin(), left);
2332 }
2333
2334 root->appendNew<Value>(
2335 proc, Branch, Origin(),
2336 root->appendNew<Value>(
2337 proc, BitAnd, Origin(), left,
2338 root->appendIntConstant(proc, Origin(), valueType, constant)));
2339 root->setSuccessors(FrequentedBlock(one), FrequentedBlock(two));
2340
2341 one->appendNew<Value>(proc, Oops, Origin());
2342 two->appendNew<Value>(proc, Oops, Origin());
2343
2344 lowerToAirForTesting(proc);
2345
2346 // The first basic block must end in a BranchTest64(resCond, tmp, bitImm).
2347 Air::Inst terminal = proc.code()[0]->last();
2348 CHECK_EQ(terminal.kind.opcode, expectedOpcode);
2349 CHECK_EQ(terminal.args[0].kind(), Air::Arg::ResCond);
2350 CHECK_EQ(terminal.args[1].kind(), firstKind);
2351 CHECK(terminal.args[2].kind() == Air::Arg::BitImm || terminal.args[2].kind() == Air::Arg::BitImm64);
2352}
2353
2354void testTerminalPatchpointThatNeedsToBeSpilled()
2355{
2356 // This is a unit test for how FTL's heap allocation fast paths behave.
2357 Procedure proc;
2358
2359 BasicBlock* root = proc.addBlock();
2360 BasicBlock* success = proc.addBlock();
2361 BasicBlock* slowPath = proc.addBlock();
2362
2363 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
2364 patchpoint->effects.terminal = true;
2365 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2366
2367 root->appendSuccessor(success);
2368 root->appendSuccessor(FrequentedBlock(slowPath, FrequencyClass::Rare));
2369
2370 patchpoint->setGenerator(
2371 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2372 AllowMacroScratchRegisterUsage allowScratch(jit);
2373 jit.move(CCallHelpers::TrustedImm32(42), params[0].gpr());
2374
2375 CCallHelpers::Jump jumpToSuccess;
2376 if (!params.fallsThroughToSuccessor(0))
2377 jumpToSuccess = jit.jump();
2378
2379 Vector<Box<CCallHelpers::Label>> labels = params.successorLabels();
2380
2381 params.addLatePath(
2382 [=] (CCallHelpers& jit) {
2383 if (jumpToSuccess.isSet())
2384 jumpToSuccess.linkTo(*labels[0], &jit);
2385 });
2386 });
2387
2388 Vector<Value*> args;
2389 {
2390 RegisterSet fillAllGPRsSet = proc.mutableGPRs();
2391 for (unsigned i = 0; i < fillAllGPRsSet.numberOfSetRegisters(); i++)
2392 args.append(success->appendNew<Const32Value>(proc, Origin(), i));
2393 }
2394
2395 {
2396 // Now force all values into every available register.
2397 PatchpointValue* p = success->appendNew<PatchpointValue>(proc, Void, Origin());
2398 for (Value* v : args)
2399 p->append(v, ValueRep::SomeRegister);
2400 p->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
2401 }
2402
2403 {
2404 // Now require the original patchpoint to be materialized into a register.
2405 PatchpointValue* p = success->appendNew<PatchpointValue>(proc, Void, Origin());
2406 p->append(patchpoint, ValueRep::SomeRegister);
2407 p->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
2408 }
2409
2410 success->appendNew<Value>(proc, Return, Origin(), success->appendNew<Const32Value>(proc, Origin(), 10));
2411
2412 slowPath->appendNew<Value>(proc, Return, Origin(), slowPath->appendNew<Const32Value>(proc, Origin(), 20));
2413
2414 auto code = compileProc(proc);
2415 CHECK_EQ(invoke<int>(*code), 10);
2416}
2417
2418void testTerminalPatchpointThatNeedsToBeSpilled2()
2419{
2420 // This is a unit test for how FTL's heap allocation fast paths behave.
2421 Procedure proc;
2422
2423 // FIXME: Air O0/O1 allocator can't handle such programs. We rely on WasmAirIRGenerator
2424 // to not use any such constructs where the register allocator is cornered in such
2425 // a way.
2426 // https://bugs.webkit.org/show_bug.cgi?id=194633
2427 if (proc.optLevel() < 2)
2428 return;
2429
2430 BasicBlock* root = proc.addBlock();
2431 BasicBlock* one = proc.addBlock();
2432 BasicBlock* success = proc.addBlock();
2433 BasicBlock* slowPath = proc.addBlock();
2434
2435 Value* arg = root->appendNew<Value>(
2436 proc, Trunc, Origin(),
2437 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
2438
2439 root->appendNew<Value>(
2440 proc, Branch, Origin(), arg);
2441 root->appendSuccessor(one);
2442 root->appendSuccessor(FrequentedBlock(slowPath, FrequencyClass::Rare));
2443
2444 PatchpointValue* patchpoint = one->appendNew<PatchpointValue>(proc, Int32, Origin());
2445 patchpoint->effects.terminal = true;
2446 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2447 patchpoint->append(arg, ValueRep::SomeRegister);
2448
2449 one->appendSuccessor(success);
2450 one->appendSuccessor(FrequentedBlock(slowPath, FrequencyClass::Rare));
2451
2452 patchpoint->setGenerator(
2453 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2454 AllowMacroScratchRegisterUsage allowScratch(jit);
2455 jit.move(CCallHelpers::TrustedImm32(666), params[0].gpr());
2456 auto goToFastPath = jit.branch32(CCallHelpers::Equal, params[1].gpr(), CCallHelpers::TrustedImm32(42));
2457 auto jumpToSlow = jit.jump();
2458
2459 // Make sure the asserts here pass.
2460 params.fallsThroughToSuccessor(0);
2461 params.fallsThroughToSuccessor(1);
2462
2463 Vector<Box<CCallHelpers::Label>> labels = params.successorLabels();
2464
2465 params.addLatePath(
2466 [=] (CCallHelpers& jit) {
2467 goToFastPath.linkTo(*labels[0], &jit);
2468 jumpToSlow.linkTo(*labels[1], &jit);
2469 });
2470 });
2471
2472 Vector<Value*> args;
2473 {
2474 RegisterSet fillAllGPRsSet = proc.mutableGPRs();
2475 for (unsigned i = 0; i < fillAllGPRsSet.numberOfSetRegisters(); i++)
2476 args.append(success->appendNew<Const32Value>(proc, Origin(), i));
2477 }
2478
2479 {
2480 // Now force all values into every available register.
2481 PatchpointValue* p = success->appendNew<PatchpointValue>(proc, Void, Origin());
2482 for (Value* v : args)
2483 p->append(v, ValueRep::SomeRegister);
2484 p->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
2485 }
2486
2487 {
2488 // Now require the original patchpoint to be materialized into a register.
2489 PatchpointValue* p = success->appendNew<PatchpointValue>(proc, Void, Origin());
2490 p->append(patchpoint, ValueRep::SomeRegister);
2491 p->setGenerator([&] (CCallHelpers&, const StackmapGenerationParams&) { });
2492 }
2493
2494 success->appendNew<Value>(proc, Return, Origin(), patchpoint);
2495
2496 slowPath->appendNew<Value>(proc, Return, Origin(), arg);
2497
2498 auto original1 = Options::maxB3TailDupBlockSize();
2499 auto original2 = Options::maxB3TailDupBlockSuccessors();
2500
2501 // Tail duplication will break the critical edge we're trying to test because it
2502 // will clone the slowPath block for both edges to it!
2503 Options::maxB3TailDupBlockSize() = 0;
2504 Options::maxB3TailDupBlockSuccessors() = 0;
2505
2506 auto code = compileProc(proc);
2507 CHECK_EQ(invoke<int>(*code, 1), 1);
2508 CHECK_EQ(invoke<int>(*code, 0), 0);
2509 CHECK_EQ(invoke<int>(*code, 42), 666);
2510
2511 Options::maxB3TailDupBlockSize() = original1;
2512 Options::maxB3TailDupBlockSuccessors() = original2;
2513}
2514
2515void testPatchpointTerminalReturnValue(bool successIsRare)
2516{
2517 // This is a unit test for how FTL's heap allocation fast paths behave.
2518 Procedure proc;
2519
2520 BasicBlock* root = proc.addBlock();
2521 BasicBlock* success = proc.addBlock();
2522 BasicBlock* slowPath = proc.addBlock();
2523 BasicBlock* continuation = proc.addBlock();
2524
2525 Value* arg = root->appendNew<Value>(
2526 proc, Trunc, Origin(),
2527 root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0));
2528
2529 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Int32, Origin());
2530 patchpoint->effects.terminal = true;
2531 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2532
2533 if (successIsRare) {
2534 root->appendSuccessor(FrequentedBlock(success, FrequencyClass::Rare));
2535 root->appendSuccessor(slowPath);
2536 } else {
2537 root->appendSuccessor(success);
2538 root->appendSuccessor(FrequentedBlock(slowPath, FrequencyClass::Rare));
2539 }
2540
2541 patchpoint->appendSomeRegister(arg);
2542
2543 patchpoint->setGenerator(
2544 [&] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2545 AllowMacroScratchRegisterUsage allowScratch(jit);
2546
2547 CCallHelpers::Jump jumpToSlow =
2548 jit.branch32(CCallHelpers::Above, params[1].gpr(), CCallHelpers::TrustedImm32(42));
2549
2550 jit.add32(CCallHelpers::TrustedImm32(31), params[1].gpr(), params[0].gpr());
2551
2552 CCallHelpers::Jump jumpToSuccess;
2553 if (!params.fallsThroughToSuccessor(0))
2554 jumpToSuccess = jit.jump();
2555
2556 Vector<Box<CCallHelpers::Label>> labels = params.successorLabels();
2557
2558 params.addLatePath(
2559 [=] (CCallHelpers& jit) {
2560 jumpToSlow.linkTo(*labels[1], &jit);
2561 if (jumpToSuccess.isSet())
2562 jumpToSuccess.linkTo(*labels[0], &jit);
2563 });
2564 });
2565
2566 UpsilonValue* successUpsilon = success->appendNew<UpsilonValue>(proc, Origin(), patchpoint);
2567 success->appendNew<Value>(proc, Jump, Origin());
2568 success->setSuccessors(continuation);
2569
2570 UpsilonValue* slowPathUpsilon = slowPath->appendNew<UpsilonValue>(
2571 proc, Origin(), slowPath->appendNew<Const32Value>(proc, Origin(), 666));
2572 slowPath->appendNew<Value>(proc, Jump, Origin());
2573 slowPath->setSuccessors(continuation);
2574
2575 Value* phi = continuation->appendNew<Value>(proc, Phi, Int32, Origin());
2576 successUpsilon->setPhi(phi);
2577 slowPathUpsilon->setPhi(phi);
2578 continuation->appendNew<Value>(proc, Return, Origin(), phi);
2579
2580 auto code = compileProc(proc);
2581 CHECK_EQ(invoke<int>(*code, 0), 31);
2582 CHECK_EQ(invoke<int>(*code, 1), 32);
2583 CHECK_EQ(invoke<int>(*code, 41), 72);
2584 CHECK_EQ(invoke<int>(*code, 42), 73);
2585 CHECK_EQ(invoke<int>(*code, 43), 666);
2586 CHECK_EQ(invoke<int>(*code, -1), 666);
2587}
2588
2589void testMemoryFence()
2590{
2591 Procedure proc;
2592
2593 BasicBlock* root = proc.addBlock();
2594
2595 root->appendNew<FenceValue>(proc, Origin());
2596 root->appendNew<Value>(proc, Return, Origin(), root->appendIntConstant(proc, Origin(), Int32, 42));
2597
2598 auto code = compileProc(proc);
2599 CHECK_EQ(invoke<int>(*code), 42);
2600 if (isX86())
2601 checkUsesInstruction(*code, "lock or $0x0, (%rsp)");
2602 if (isARM64())
2603 checkUsesInstruction(*code, "dmb ish");
2604 checkDoesNotUseInstruction(*code, "mfence");
2605 checkDoesNotUseInstruction(*code, "dmb ishst");
2606}
2607
2608void testStoreFence()
2609{
2610 Procedure proc;
2611
2612 BasicBlock* root = proc.addBlock();
2613
2614 root->appendNew<FenceValue>(proc, Origin(), HeapRange::top(), HeapRange());
2615 root->appendNew<Value>(proc, Return, Origin(), root->appendIntConstant(proc, Origin(), Int32, 42));
2616
2617 auto code = compileProc(proc);
2618 CHECK_EQ(invoke<int>(*code), 42);
2619 checkDoesNotUseInstruction(*code, "lock");
2620 checkDoesNotUseInstruction(*code, "mfence");
2621 if (isARM64())
2622 checkUsesInstruction(*code, "dmb ishst");
2623}
2624
2625void testLoadFence()
2626{
2627 Procedure proc;
2628
2629 BasicBlock* root = proc.addBlock();
2630
2631 root->appendNew<FenceValue>(proc, Origin(), HeapRange(), HeapRange::top());
2632 root->appendNew<Value>(proc, Return, Origin(), root->appendIntConstant(proc, Origin(), Int32, 42));
2633
2634 auto code = compileProc(proc);
2635 CHECK_EQ(invoke<int>(*code), 42);
2636 checkDoesNotUseInstruction(*code, "lock");
2637 checkDoesNotUseInstruction(*code, "mfence");
2638 if (isARM64())
2639 checkUsesInstruction(*code, "dmb ish");
2640 checkDoesNotUseInstruction(*code, "dmb ishst");
2641}
2642
2643void testTrappingLoad()
2644{
2645 Procedure proc;
2646 BasicBlock* root = proc.addBlock();
2647 int x = 42;
2648 MemoryValue* value = root->appendNew<MemoryValue>(
2649 proc, trapping(Load), Int32, Origin(),
2650 root->appendNew<ConstPtrValue>(proc, Origin(), &x));
2651 Effects expectedEffects;
2652 expectedEffects.exitsSideways = true;
2653 expectedEffects.controlDependent= true;
2654 expectedEffects.reads = HeapRange::top();
2655 CHECK_EQ(value->range(), HeapRange::top());
2656 CHECK_EQ(value->effects(), expectedEffects);
2657 value->setRange(HeapRange(0));
2658 CHECK_EQ(value->range(), HeapRange(0));
2659 CHECK_EQ(value->effects(), expectedEffects); // We still reads top!
2660 root->appendNew<Value>(proc, Return, Origin(), value);
2661 CHECK_EQ(compileAndRun<int>(proc), 42);
2662 unsigned trapsCount = 0;
2663 for (Air::BasicBlock* block : proc.code()) {
2664 for (Air::Inst& inst : *block) {
2665 if (inst.kind.effects)
2666 trapsCount++;
2667 }
2668 }
2669 CHECK_EQ(trapsCount, 1u);
2670}
2671
2672void testTrappingStore()
2673{
2674 Procedure proc;
2675 BasicBlock* root = proc.addBlock();
2676 int x = 42;
2677 MemoryValue* value = root->appendNew<MemoryValue>(
2678 proc, trapping(Store), Origin(),
2679 root->appendNew<Const32Value>(proc, Origin(), 111),
2680 root->appendNew<ConstPtrValue>(proc, Origin(), &x), 0);
2681 Effects expectedEffects;
2682 expectedEffects.exitsSideways = true;
2683 expectedEffects.controlDependent= true;
2684 expectedEffects.reads = HeapRange::top();
2685 expectedEffects.writes = HeapRange::top();
2686 CHECK_EQ(value->range(), HeapRange::top());
2687 CHECK_EQ(value->effects(), expectedEffects);
2688 value->setRange(HeapRange(0));
2689 CHECK_EQ(value->range(), HeapRange(0));
2690 expectedEffects.writes = HeapRange(0);
2691 CHECK_EQ(value->effects(), expectedEffects); // We still reads top!
2692 root->appendNew<Value>(proc, Return, Origin());
2693 compileAndRun<int>(proc);
2694 CHECK_EQ(x, 111);
2695 unsigned trapsCount = 0;
2696 for (Air::BasicBlock* block : proc.code()) {
2697 for (Air::Inst& inst : *block) {
2698 if (inst.kind.effects)
2699 trapsCount++;
2700 }
2701 }
2702 CHECK_EQ(trapsCount, 1u);
2703}
2704
2705void testTrappingLoadAddStore()
2706{
2707 Procedure proc;
2708 BasicBlock* root = proc.addBlock();
2709 int x = 42;
2710 ConstPtrValue* ptr = root->appendNew<ConstPtrValue>(proc, Origin(), &x);
2711 root->appendNew<MemoryValue>(
2712 proc, trapping(Store), Origin(),
2713 root->appendNew<Value>(
2714 proc, Add, Origin(),
2715 root->appendNew<MemoryValue>(proc, trapping(Load), Int32, Origin(), ptr),
2716 root->appendNew<Const32Value>(proc, Origin(), 3)),
2717 ptr, 0);
2718 root->appendNew<Value>(proc, Return, Origin());
2719 compileAndRun<int>(proc);
2720 CHECK_EQ(x, 45);
2721 bool traps = false;
2722 for (Air::BasicBlock* block : proc.code()) {
2723 for (Air::Inst& inst : *block) {
2724 if (inst.kind.effects)
2725 traps = true;
2726 }
2727 }
2728 CHECK(traps);
2729}
2730
2731void testTrappingLoadDCE()
2732{
2733 Procedure proc;
2734 BasicBlock* root = proc.addBlock();
2735 int x = 42;
2736 root->appendNew<MemoryValue>(
2737 proc, trapping(Load), Int32, Origin(),
2738 root->appendNew<ConstPtrValue>(proc, Origin(), &x));
2739 root->appendNew<Value>(proc, Return, Origin());
2740 compileAndRun<int>(proc);
2741 unsigned trapsCount = 0;
2742 for (Air::BasicBlock* block : proc.code()) {
2743 for (Air::Inst& inst : *block) {
2744 if (inst.kind.effects)
2745 trapsCount++;
2746 }
2747 }
2748 CHECK_EQ(trapsCount, 1u);
2749}
2750
2751void testTrappingStoreElimination()
2752{
2753 Procedure proc;
2754 BasicBlock* root = proc.addBlock();
2755 int x = 42;
2756 Value* ptr = root->appendNew<ConstPtrValue>(proc, Origin(), &x);
2757 root->appendNew<MemoryValue>(
2758 proc, trapping(Store), Origin(),
2759 root->appendNew<Const32Value>(proc, Origin(), 43),
2760 ptr);
2761 root->appendNew<MemoryValue>(
2762 proc, trapping(Store), Origin(),
2763 root->appendNew<Const32Value>(proc, Origin(), 44),
2764 ptr);
2765 root->appendNew<Value>(proc, Return, Origin());
2766 compileAndRun<int>(proc);
2767 unsigned storeCount = 0;
2768 for (Value* value : proc.values()) {
2769 if (isStore(value->opcode()))
2770 storeCount++;
2771 }
2772 CHECK_EQ(storeCount, 2u);
2773}
2774
2775void testMoveConstants()
2776{
2777 auto check = [] (Procedure& proc) {
2778 proc.resetReachability();
2779
2780 if (shouldBeVerbose()) {
2781 dataLog("IR before:\n");
2782 dataLog(proc);
2783 }
2784
2785 moveConstants(proc);
2786
2787 if (shouldBeVerbose()) {
2788 dataLog("IR after:\n");
2789 dataLog(proc);
2790 }
2791
2792 UseCounts useCounts(proc);
2793 unsigned count = 0;
2794 for (Value* value : proc.values()) {
2795 if (useCounts.numUses(value) && value->hasInt64())
2796 count++;
2797 }
2798
2799 if (count == 1)
2800 return;
2801
2802 crashLock.lock();
2803 dataLog("Fail in testMoveConstants: got more than one Const64:\n");
2804 dataLog(proc);
2805 CRASH();
2806 };
2807
2808 {
2809 Procedure proc;
2810 BasicBlock* root = proc.addBlock();
2811 Value* a = root->appendNew<MemoryValue>(
2812 proc, Load, pointerType(), Origin(),
2813 root->appendNew<ConstPtrValue>(proc, Origin(), 0x123412341234));
2814 Value* b = root->appendNew<MemoryValue>(
2815 proc, Load, pointerType(), Origin(),
2816 root->appendNew<ConstPtrValue>(proc, Origin(), 0x123412341334));
2817 root->appendNew<CCallValue>(proc, Void, Origin(), a, b);
2818 root->appendNew<Value>(proc, Return, Origin());
2819 check(proc);
2820 }
2821
2822 {
2823 Procedure proc;
2824 BasicBlock* root = proc.addBlock();
2825 Value* x = root->appendNew<ArgumentRegValue>(proc, Origin(), GPRInfo::argumentGPR0);
2826 Value* a = root->appendNew<Value>(
2827 proc, Add, Origin(), x, root->appendNew<ConstPtrValue>(proc, Origin(), 0x123412341234));
2828 Value* b = root->appendNew<Value>(
2829 proc, Add, Origin(), x, root->appendNew<ConstPtrValue>(proc, Origin(), -0x123412341234));
2830 root->appendNew<CCallValue>(proc, Void, Origin(), a, b);
2831 root->appendNew<Value>(proc, Return, Origin());
2832 check(proc);
2833 }
2834}
2835
2836void testPCOriginMapDoesntInsertNops()
2837{
2838 Procedure proc;
2839 BasicBlock* root = proc.addBlock();
2840
2841 CCallHelpers::Label watchpointLabel;
2842
2843 PatchpointValue* patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
2844 patchpoint->setGenerator(
2845 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
2846 watchpointLabel = jit.watchpointLabel();
2847 });
2848
2849 patchpoint = root->appendNew<PatchpointValue>(proc, Void, Origin());
2850 patchpoint->setGenerator(
2851 [&] (CCallHelpers& jit, const StackmapGenerationParams&) {
2852 CCallHelpers::Label labelIgnoringWatchpoints = jit.labelIgnoringWatchpoints();
2853
2854 CHECK(watchpointLabel == labelIgnoringWatchpoints);
2855 });
2856
2857 root->appendNew<Value>(proc, Return, Origin());
2858
2859 compileProc(proc);
2860}
2861
2862void addSShrShTests(const char* filter, Deque<RefPtr<SharedTask<void()>>>& tasks)
2863{
2864 RUN(testSShrShl32(42, 24, 24));
2865 RUN(testSShrShl32(-42, 24, 24));
2866 RUN(testSShrShl32(4200, 24, 24));
2867 RUN(testSShrShl32(-4200, 24, 24));
2868 RUN(testSShrShl32(4200000, 24, 24));
2869 RUN(testSShrShl32(-4200000, 24, 24));
2870
2871 RUN(testSShrShl32(42, 16, 16));
2872 RUN(testSShrShl32(-42, 16, 16));
2873 RUN(testSShrShl32(4200, 16, 16));
2874 RUN(testSShrShl32(-4200, 16, 16));
2875 RUN(testSShrShl32(4200000, 16, 16));
2876 RUN(testSShrShl32(-4200000, 16, 16));
2877
2878 RUN(testSShrShl32(42, 8, 8));
2879 RUN(testSShrShl32(-42, 8, 8));
2880 RUN(testSShrShl32(4200, 8, 8));
2881 RUN(testSShrShl32(-4200, 8, 8));
2882 RUN(testSShrShl32(4200000, 8, 8));
2883 RUN(testSShrShl32(-4200000, 8, 8));
2884 RUN(testSShrShl32(420000000, 8, 8));
2885 RUN(testSShrShl32(-420000000, 8, 8));
2886
2887 RUN(testSShrShl64(42, 56, 56));
2888 RUN(testSShrShl64(-42, 56, 56));
2889 RUN(testSShrShl64(4200, 56, 56));
2890 RUN(testSShrShl64(-4200, 56, 56));
2891 RUN(testSShrShl64(4200000, 56, 56));
2892 RUN(testSShrShl64(-4200000, 56, 56));
2893 RUN(testSShrShl64(420000000, 56, 56));
2894 RUN(testSShrShl64(-420000000, 56, 56));
2895 RUN(testSShrShl64(42000000000, 56, 56));
2896 RUN(testSShrShl64(-42000000000, 56, 56));
2897
2898 RUN(testSShrShl64(42, 48, 48));
2899 RUN(testSShrShl64(-42, 48, 48));
2900 RUN(testSShrShl64(4200, 48, 48));
2901 RUN(testSShrShl64(-4200, 48, 48));
2902 RUN(testSShrShl64(4200000, 48, 48));
2903 RUN(testSShrShl64(-4200000, 48, 48));
2904 RUN(testSShrShl64(420000000, 48, 48));
2905 RUN(testSShrShl64(-420000000, 48, 48));
2906 RUN(testSShrShl64(42000000000, 48, 48));
2907 RUN(testSShrShl64(-42000000000, 48, 48));
2908
2909 RUN(testSShrShl64(42, 32, 32));
2910 RUN(testSShrShl64(-42, 32, 32));
2911 RUN(testSShrShl64(4200, 32, 32));
2912 RUN(testSShrShl64(-4200, 32, 32));
2913 RUN(testSShrShl64(4200000, 32, 32));
2914 RUN(testSShrShl64(-4200000, 32, 32));
2915 RUN(testSShrShl64(420000000, 32, 32));
2916 RUN(testSShrShl64(-420000000, 32, 32));
2917 RUN(testSShrShl64(42000000000, 32, 32));
2918 RUN(testSShrShl64(-42000000000, 32, 32));
2919
2920 RUN(testSShrShl64(42, 24, 24));
2921 RUN(testSShrShl64(-42, 24, 24));
2922 RUN(testSShrShl64(4200, 24, 24));
2923 RUN(testSShrShl64(-4200, 24, 24));
2924 RUN(testSShrShl64(4200000, 24, 24));
2925 RUN(testSShrShl64(-4200000, 24, 24));
2926 RUN(testSShrShl64(420000000, 24, 24));
2927 RUN(testSShrShl64(-420000000, 24, 24));
2928 RUN(testSShrShl64(42000000000, 24, 24));
2929 RUN(testSShrShl64(-42000000000, 24, 24));
2930
2931 RUN(testSShrShl64(42, 16, 16));
2932 RUN(testSShrShl64(-42, 16, 16));
2933 RUN(testSShrShl64(4200, 16, 16));
2934 RUN(testSShrShl64(-4200, 16, 16));
2935 RUN(testSShrShl64(4200000, 16, 16));
2936 RUN(testSShrShl64(-4200000, 16, 16));
2937 RUN(testSShrShl64(420000000, 16, 16));
2938 RUN(testSShrShl64(-420000000, 16, 16));
2939 RUN(testSShrShl64(42000000000, 16, 16));
2940 RUN(testSShrShl64(-42000000000, 16, 16));
2941
2942 RUN(testSShrShl64(42, 8, 8));
2943 RUN(testSShrShl64(-42, 8, 8));
2944 RUN(testSShrShl64(4200, 8, 8));
2945 RUN(testSShrShl64(-4200, 8, 8));
2946 RUN(testSShrShl64(4200000, 8, 8));
2947 RUN(testSShrShl64(-4200000, 8, 8));
2948 RUN(testSShrShl64(420000000, 8, 8));
2949 RUN(testSShrShl64(-420000000, 8, 8));
2950 RUN(testSShrShl64(42000000000, 8, 8));
2951 RUN(testSShrShl64(-42000000000, 8, 8));
2952}
2953
2954#endif // ENABLE(B3_JIT)
2955