1/*
2 * Copyright (C) 2013-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27
28#if ENABLE(ASSEMBLER) && (CPU(X86) || CPU(X86_64))
29#include "MacroAssembler.h"
30
31#include "ProbeContext.h"
32#include <wtf/InlineASM.h>
33
34#if COMPILER(MSVC)
35#include <intrin.h>
36#endif
37
38namespace JSC {
39
40#if ENABLE(MASM_PROBE)
41
42extern "C" void ctiMasmProbeTrampoline();
43
44// The following are offsets for Probe::State fields accessed by the ctiMasmProbeTrampoline stub.
45
46#if CPU(X86)
47#define PTR_SIZE 4
48#else // CPU(X86_64)
49#define PTR_SIZE 8
50#endif
51
52#define PROBE_PROBE_FUNCTION_OFFSET (0 * PTR_SIZE)
53#define PROBE_ARG_OFFSET (1 * PTR_SIZE)
54#define PROBE_INIT_STACK_FUNCTION_OFFSET (2 * PTR_SIZE)
55#define PROBE_INIT_STACK_ARG_OFFSET (3 * PTR_SIZE)
56
57#define PROBE_FIRST_GPR_OFFSET (4 * PTR_SIZE)
58#define PROBE_CPU_EAX_OFFSET (PROBE_FIRST_GPR_OFFSET + (0 * PTR_SIZE))
59#define PROBE_CPU_ECX_OFFSET (PROBE_FIRST_GPR_OFFSET + (1 * PTR_SIZE))
60#define PROBE_CPU_EDX_OFFSET (PROBE_FIRST_GPR_OFFSET + (2 * PTR_SIZE))
61#define PROBE_CPU_EBX_OFFSET (PROBE_FIRST_GPR_OFFSET + (3 * PTR_SIZE))
62#define PROBE_CPU_ESP_OFFSET (PROBE_FIRST_GPR_OFFSET + (4 * PTR_SIZE))
63#define PROBE_CPU_EBP_OFFSET (PROBE_FIRST_GPR_OFFSET + (5 * PTR_SIZE))
64#define PROBE_CPU_ESI_OFFSET (PROBE_FIRST_GPR_OFFSET + (6 * PTR_SIZE))
65#define PROBE_CPU_EDI_OFFSET (PROBE_FIRST_GPR_OFFSET + (7 * PTR_SIZE))
66
67#if CPU(X86)
68#define PROBE_FIRST_SPR_OFFSET (PROBE_FIRST_GPR_OFFSET + (8 * PTR_SIZE))
69#else // CPU(X86_64)
70#define PROBE_CPU_R8_OFFSET (PROBE_FIRST_GPR_OFFSET + (8 * PTR_SIZE))
71#define PROBE_CPU_R9_OFFSET (PROBE_FIRST_GPR_OFFSET + (9 * PTR_SIZE))
72#define PROBE_CPU_R10_OFFSET (PROBE_FIRST_GPR_OFFSET + (10 * PTR_SIZE))
73#define PROBE_CPU_R11_OFFSET (PROBE_FIRST_GPR_OFFSET + (11 * PTR_SIZE))
74#define PROBE_CPU_R12_OFFSET (PROBE_FIRST_GPR_OFFSET + (12 * PTR_SIZE))
75#define PROBE_CPU_R13_OFFSET (PROBE_FIRST_GPR_OFFSET + (13 * PTR_SIZE))
76#define PROBE_CPU_R14_OFFSET (PROBE_FIRST_GPR_OFFSET + (14 * PTR_SIZE))
77#define PROBE_CPU_R15_OFFSET (PROBE_FIRST_GPR_OFFSET + (15 * PTR_SIZE))
78#define PROBE_FIRST_SPR_OFFSET (PROBE_FIRST_GPR_OFFSET + (16 * PTR_SIZE))
79#endif // CPU(X86_64)
80
81#define PROBE_CPU_EIP_OFFSET (PROBE_FIRST_SPR_OFFSET + (0 * PTR_SIZE))
82#define PROBE_CPU_EFLAGS_OFFSET (PROBE_FIRST_SPR_OFFSET + (1 * PTR_SIZE))
83#define PROBE_FIRST_XMM_OFFSET (PROBE_FIRST_SPR_OFFSET + (2 * PTR_SIZE))
84
85#define XMM_SIZE 8
86#define PROBE_CPU_XMM0_OFFSET (PROBE_FIRST_XMM_OFFSET + (0 * XMM_SIZE))
87#define PROBE_CPU_XMM1_OFFSET (PROBE_FIRST_XMM_OFFSET + (1 * XMM_SIZE))
88#define PROBE_CPU_XMM2_OFFSET (PROBE_FIRST_XMM_OFFSET + (2 * XMM_SIZE))
89#define PROBE_CPU_XMM3_OFFSET (PROBE_FIRST_XMM_OFFSET + (3 * XMM_SIZE))
90#define PROBE_CPU_XMM4_OFFSET (PROBE_FIRST_XMM_OFFSET + (4 * XMM_SIZE))
91#define PROBE_CPU_XMM5_OFFSET (PROBE_FIRST_XMM_OFFSET + (5 * XMM_SIZE))
92#define PROBE_CPU_XMM6_OFFSET (PROBE_FIRST_XMM_OFFSET + (6 * XMM_SIZE))
93#define PROBE_CPU_XMM7_OFFSET (PROBE_FIRST_XMM_OFFSET + (7 * XMM_SIZE))
94
95#if CPU(X86)
96#define PROBE_SIZE (PROBE_CPU_XMM7_OFFSET + XMM_SIZE)
97#else // CPU(X86_64)
98#define PROBE_CPU_XMM8_OFFSET (PROBE_FIRST_XMM_OFFSET + (8 * XMM_SIZE))
99#define PROBE_CPU_XMM9_OFFSET (PROBE_FIRST_XMM_OFFSET + (9 * XMM_SIZE))
100#define PROBE_CPU_XMM10_OFFSET (PROBE_FIRST_XMM_OFFSET + (10 * XMM_SIZE))
101#define PROBE_CPU_XMM11_OFFSET (PROBE_FIRST_XMM_OFFSET + (11 * XMM_SIZE))
102#define PROBE_CPU_XMM12_OFFSET (PROBE_FIRST_XMM_OFFSET + (12 * XMM_SIZE))
103#define PROBE_CPU_XMM13_OFFSET (PROBE_FIRST_XMM_OFFSET + (13 * XMM_SIZE))
104#define PROBE_CPU_XMM14_OFFSET (PROBE_FIRST_XMM_OFFSET + (14 * XMM_SIZE))
105#define PROBE_CPU_XMM15_OFFSET (PROBE_FIRST_XMM_OFFSET + (15 * XMM_SIZE))
106#define PROBE_SIZE (PROBE_CPU_XMM15_OFFSET + XMM_SIZE)
107#endif // CPU(X86_64)
108
109#define PROBE_EXECUTOR_OFFSET PROBE_SIZE // Stash the executeProbe function pointer at the end of the ProbeContext.
110
111// The outgoing record to be popped off the stack at the end consists of:
112// eflags, eax, ecx, ebp, eip.
113#define OUT_SIZE (5 * PTR_SIZE)
114
115// These ASSERTs remind you that if you change the layout of Probe::State,
116// you need to change ctiMasmProbeTrampoline offsets above to match.
117#define PROBE_OFFSETOF(x) offsetof(struct Probe::State, x)
118#define PROBE_OFFSETOF_REG(x, reg) offsetof(struct Probe::State, x) + reg * sizeof((reinterpret_cast<Probe::State*>(0))->x[reg])
119static_assert(PROBE_OFFSETOF(probeFunction) == PROBE_PROBE_FUNCTION_OFFSET, "Probe::State::probeFunction's offset matches ctiMasmProbeTrampoline");
120static_assert(PROBE_OFFSETOF(arg) == PROBE_ARG_OFFSET, "Probe::State::arg's offset matches ctiMasmProbeTrampoline");
121static_assert(PROBE_OFFSETOF(initializeStackFunction) == PROBE_INIT_STACK_FUNCTION_OFFSET, "Probe::State::initializeStackFunction's offset matches ctiMasmProbeTrampoline");
122static_assert(PROBE_OFFSETOF(initializeStackArg) == PROBE_INIT_STACK_ARG_OFFSET, "Probe::State::initializeStackArg's offset matches ctiMasmProbeTrampoline");
123
124static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::eax) == PROBE_CPU_EAX_OFFSET, "Probe::State::cpu.gprs[eax]'s offset matches ctiMasmProbeTrampoline");
125static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::ecx) == PROBE_CPU_ECX_OFFSET, "Probe::State::cpu.gprs[ecx]'s offset matches ctiMasmProbeTrampoline");
126static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::edx) == PROBE_CPU_EDX_OFFSET, "Probe::State::cpu.gprs[edx]'s offset matches ctiMasmProbeTrampoline");
127static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::ebx) == PROBE_CPU_EBX_OFFSET, "Probe::State::cpu.gprs[ebx]'s offset matches ctiMasmProbeTrampoline");
128static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::esp) == PROBE_CPU_ESP_OFFSET, "Probe::State::cpu.gprs[esp]'s offset matches ctiMasmProbeTrampoline");
129static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::ebp) == PROBE_CPU_EBP_OFFSET, "Probe::State::cpu.gprs[ebp]'s offset matches ctiMasmProbeTrampoline");
130static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::esi) == PROBE_CPU_ESI_OFFSET, "Probe::State::cpu.gprs[esi]'s offset matches ctiMasmProbeTrampoline");
131static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::edi) == PROBE_CPU_EDI_OFFSET, "Probe::State::cpu.gprs[edi]'s offset matches ctiMasmProbeTrampoline");
132static_assert(PROBE_OFFSETOF_REG(cpu.sprs, X86Registers::eip) == PROBE_CPU_EIP_OFFSET, "Probe::State::cpu.gprs[eip]'s offset matches ctiMasmProbeTrampoline");
133static_assert(PROBE_OFFSETOF_REG(cpu.sprs, X86Registers::eflags) == PROBE_CPU_EFLAGS_OFFSET, "Probe::State::cpu.sprs[eflags]'s offset matches ctiMasmProbeTrampoline");
134
135#if CPU(X86_64)
136static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r8) == PROBE_CPU_R8_OFFSET, "Probe::State::cpu.gprs[r8]'s offset matches ctiMasmProbeTrampoline");
137static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r9) == PROBE_CPU_R9_OFFSET, "Probe::State::cpu.gprs[r9]'s offset matches ctiMasmProbeTrampoline");
138static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r10) == PROBE_CPU_R10_OFFSET, "Probe::State::cpu.gprs[r10]'s offset matches ctiMasmProbeTrampoline");
139static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r11) == PROBE_CPU_R11_OFFSET, "Probe::State::cpu.gprs[r11]'s offset matches ctiMasmProbeTrampoline");
140static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r12) == PROBE_CPU_R12_OFFSET, "Probe::State::cpu.gprs[r12]'s offset matches ctiMasmProbeTrampoline");
141static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r13) == PROBE_CPU_R13_OFFSET, "Probe::State::cpu.gprs[r13]'s offset matches ctiMasmProbeTrampoline");
142static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r14) == PROBE_CPU_R14_OFFSET, "Probe::State::cpu.gprs[r14]'s offset matches ctiMasmProbeTrampoline");
143static_assert(PROBE_OFFSETOF_REG(cpu.gprs, X86Registers::r15) == PROBE_CPU_R15_OFFSET, "Probe::State::cpu.gprs[r15]'s offset matches ctiMasmProbeTrampoline");
144#endif // CPU(X86_64)
145
146static_assert(!(PROBE_CPU_XMM0_OFFSET & 0x7), "Probe::State::cpu.fprs[xmm0]'s offset should be 8 byte aligned");
147
148static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm0) == PROBE_CPU_XMM0_OFFSET, "Probe::State::cpu.fprs[xmm0]'s offset matches ctiMasmProbeTrampoline");
149static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm1) == PROBE_CPU_XMM1_OFFSET, "Probe::State::cpu.fprs[xmm1]'s offset matches ctiMasmProbeTrampoline");
150static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm2) == PROBE_CPU_XMM2_OFFSET, "Probe::State::cpu.fprs[xmm2]'s offset matches ctiMasmProbeTrampoline");
151static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm3) == PROBE_CPU_XMM3_OFFSET, "Probe::State::cpu.fprs[xmm3]'s offset matches ctiMasmProbeTrampoline");
152static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm4) == PROBE_CPU_XMM4_OFFSET, "Probe::State::cpu.fprs[xmm4]'s offset matches ctiMasmProbeTrampoline");
153static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm5) == PROBE_CPU_XMM5_OFFSET, "Probe::State::cpu.fprs[xmm5]'s offset matches ctiMasmProbeTrampoline");
154static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm6) == PROBE_CPU_XMM6_OFFSET, "Probe::State::cpu.fprs[xmm6]'s offset matches ctiMasmProbeTrampoline");
155static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm7) == PROBE_CPU_XMM7_OFFSET, "Probe::State::cpu.fprs[xmm7]'s offset matches ctiMasmProbeTrampoline");
156
157#if CPU(X86_64)
158static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm8) == PROBE_CPU_XMM8_OFFSET, "Probe::State::cpu.fprs[xmm8]'s offset matches ctiMasmProbeTrampoline");
159static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm9) == PROBE_CPU_XMM9_OFFSET, "Probe::State::cpu.fprs[xmm9]'s offset matches ctiMasmProbeTrampoline");
160static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm10) == PROBE_CPU_XMM10_OFFSET, "Probe::State::cpu.fprs[xmm10]'s offset matches ctiMasmProbeTrampoline");
161static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm11) == PROBE_CPU_XMM11_OFFSET, "Probe::State::cpu.fprs[xmm11]'s offset matches ctiMasmProbeTrampoline");
162static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm12) == PROBE_CPU_XMM12_OFFSET, "Probe::State::cpu.fprs[xmm12]'s offset matches ctiMasmProbeTrampoline");
163static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm13) == PROBE_CPU_XMM13_OFFSET, "Probe::State::cpu.fprs[xmm13]'s offset matches ctiMasmProbeTrampoline");
164static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm14) == PROBE_CPU_XMM14_OFFSET, "Probe::State::cpu.fprs[xmm14]'s offset matches ctiMasmProbeTrampoline");
165static_assert(PROBE_OFFSETOF_REG(cpu.fprs, X86Registers::xmm15) == PROBE_CPU_XMM15_OFFSET, "Probe::State::cpu.fprs[xmm15]'s offset matches ctiMasmProbeTrampoline");
166#endif // CPU(X86_64)
167
168static_assert(sizeof(Probe::State) == PROBE_SIZE, "Probe::State::size's matches ctiMasmProbeTrampoline");
169static_assert((PROBE_EXECUTOR_OFFSET + PTR_SIZE) <= (PROBE_SIZE + OUT_SIZE), "Must have room after ProbeContext to stash the probe handler");
170
171#if CPU(X86)
172// SSE2 is a hard requirement on x86.
173static_assert(isSSE2Present(), "SSE2 support is required in JavaScriptCore");
174#endif
175
176#undef PROBE_OFFSETOF
177
178#if CPU(X86)
179#if COMPILER(GCC_COMPATIBLE)
180asm (
181 ".globl " SYMBOL_STRING(ctiMasmProbeTrampoline) "\n"
182 HIDE_SYMBOL(ctiMasmProbeTrampoline) "\n"
183 SYMBOL_STRING(ctiMasmProbeTrampoline) ":" "\n"
184
185 "pushfl" "\n"
186
187 // MacroAssemblerX86Common::probe() has already generated code to store some values.
188 // Together with the eflags pushed above, the top of stack now looks like
189 // this:
190 // esp[0 * ptrSize]: eflags
191 // esp[1 * ptrSize]: return address / saved eip
192 // esp[2 * ptrSize]: saved ebx
193 // esp[3 * ptrSize]: saved edx
194 // esp[4 * ptrSize]: saved ecx
195 // esp[5 * ptrSize]: saved eax
196 //
197 // Incoming registers contain:
198 // ecx: Probe::executeProbe
199 // edx: probe function
200 // ebx: probe arg
201 // eax: scratch (was ctiMasmProbeTrampoline)
202
203 "movl %esp, %eax" "\n"
204 "subl $" STRINGIZE_VALUE_OF(PROBE_SIZE + OUT_SIZE) ", %esp" "\n"
205
206 // The X86_64 ABI specifies that the worse case stack alignment requirement is 32 bytes.
207 "andl $~0x1f, %esp" "\n"
208
209 "movl %ebp, " STRINGIZE_VALUE_OF(PROBE_CPU_EBP_OFFSET) "(%esp)" "\n"
210 "movl %esp, %ebp" "\n" // Save the Probe::State*.
211
212 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_EXECUTOR_OFFSET) "(%ebp)" "\n"
213 "movl %edx, " STRINGIZE_VALUE_OF(PROBE_PROBE_FUNCTION_OFFSET) "(%ebp)" "\n"
214 "movl %ebx, " STRINGIZE_VALUE_OF(PROBE_ARG_OFFSET) "(%ebp)" "\n"
215 "movl %esi, " STRINGIZE_VALUE_OF(PROBE_CPU_ESI_OFFSET) "(%ebp)" "\n"
216 "movl %edi, " STRINGIZE_VALUE_OF(PROBE_CPU_EDI_OFFSET) "(%ebp)" "\n"
217
218 "movl 0 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
219 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_EFLAGS_OFFSET) "(%ebp)" "\n"
220 "movl 1 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
221 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_EIP_OFFSET) "(%ebp)" "\n"
222 "movl 2 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
223 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_EBX_OFFSET) "(%ebp)" "\n"
224 "movl 3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
225 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_EDX_OFFSET) "(%ebp)" "\n"
226 "movl 4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
227 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_ECX_OFFSET) "(%ebp)" "\n"
228 "movl 5 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%eax), %ecx" "\n"
229 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_EAX_OFFSET) "(%ebp)" "\n"
230
231 "movl %eax, %ecx" "\n"
232 "addl $" STRINGIZE_VALUE_OF(6 * PTR_SIZE) ", %ecx" "\n"
233 "movl %ecx, " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%ebp)" "\n"
234
235 "movq %xmm0, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM0_OFFSET) "(%ebp)" "\n"
236 "movq %xmm1, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM1_OFFSET) "(%ebp)" "\n"
237 "movq %xmm2, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM2_OFFSET) "(%ebp)" "\n"
238 "movq %xmm3, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM3_OFFSET) "(%ebp)" "\n"
239 "movq %xmm4, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM4_OFFSET) "(%ebp)" "\n"
240 "movq %xmm5, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM5_OFFSET) "(%ebp)" "\n"
241 "movq %xmm6, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM6_OFFSET) "(%ebp)" "\n"
242 "movq %xmm7, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM7_OFFSET) "(%ebp)" "\n"
243
244 // Reserve stack space for the arg while maintaining the required stack
245 // pointer 32 byte alignment:
246 "subl $0x20, %esp" "\n"
247 "movl %ebp, 0(%esp)" "\n" // the Probe::State* arg.
248
249 "call *" STRINGIZE_VALUE_OF(PROBE_EXECUTOR_OFFSET) "(%ebp)" "\n"
250
251 // Make sure the Probe::State is entirely below the result stack pointer so
252 // that register values are still preserved when we call the initializeStack
253 // function.
254 "movl $" STRINGIZE_VALUE_OF(PROBE_SIZE + OUT_SIZE) ", %ecx" "\n"
255 "movl %ebp, %eax" "\n"
256 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%ebp), %edx" "\n"
257 "addl %ecx, %eax" "\n"
258 "cmpl %eax, %edx" "\n"
259 "jge " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) "\n"
260
261 // Allocate a safe place on the stack below the result stack pointer to stash the Probe::State.
262 "subl %ecx, %edx" "\n"
263 "andl $~0x1f, %edx" "\n" // Keep the stack pointer 32 bytes aligned.
264 "xorl %eax, %eax" "\n"
265 "movl %edx, %esp" "\n"
266
267 "movl $" STRINGIZE_VALUE_OF(PROBE_SIZE) ", %ecx" "\n"
268
269 // Copy the Probe::State to the safe place.
270 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) ":" "\n"
271 "movl (%ebp, %eax), %edx" "\n"
272 "movl %edx, (%esp, %eax)" "\n"
273 "addl $" STRINGIZE_VALUE_OF(PTR_SIZE) ", %eax" "\n"
274 "cmpl %eax, %ecx" "\n"
275 "jg " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) "\n"
276
277 "movl %esp, %ebp" "\n"
278
279 // Call initializeStackFunction if present.
280 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) ":" "\n"
281 "xorl %ecx, %ecx" "\n"
282 "addl " STRINGIZE_VALUE_OF(PROBE_INIT_STACK_FUNCTION_OFFSET) "(%ebp), %ecx" "\n"
283 "je " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) "\n"
284
285 // Reserve stack space for the arg while maintaining the required stack
286 // pointer 32 byte alignment:
287 "subl $0x20, %esp" "\n"
288 "movl %ebp, 0(%esp)" "\n" // the Probe::State* arg.
289 "call *%ecx" "\n"
290
291 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) ":" "\n"
292
293 // To enable probes to modify register state, we copy all registers
294 // out of the Probe::State before returning.
295
296 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EDX_OFFSET) "(%ebp), %edx" "\n"
297 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EBX_OFFSET) "(%ebp), %ebx" "\n"
298 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_ESI_OFFSET) "(%ebp), %esi" "\n"
299 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EDI_OFFSET) "(%ebp), %edi" "\n"
300
301 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM0_OFFSET) "(%ebp), %xmm0" "\n"
302 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM1_OFFSET) "(%ebp), %xmm1" "\n"
303 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM2_OFFSET) "(%ebp), %xmm2" "\n"
304 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM3_OFFSET) "(%ebp), %xmm3" "\n"
305 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM4_OFFSET) "(%ebp), %xmm4" "\n"
306 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM5_OFFSET) "(%ebp), %xmm5" "\n"
307 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM6_OFFSET) "(%ebp), %xmm6" "\n"
308 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM7_OFFSET) "(%ebp), %xmm7" "\n"
309
310 // There are 6 more registers left to restore:
311 // eax, ecx, ebp, esp, eip, and eflags.
312
313 // The restoration process at ctiMasmProbeTrampolineEnd below works by popping
314 // 5 words off the stack into eflags, eax, ecx, ebp, and eip. These 5 words need
315 // to be pushed on top of the final esp value so that just by popping the 5 words,
316 // we'll get the esp that the probe wants to set. Let's call this area (for storing
317 // these 5 words) the restore area.
318 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%ebp), %ecx" "\n"
319 "subl $5 * " STRINGIZE_VALUE_OF(PTR_SIZE) ", %ecx" "\n"
320
321 // ecx now points to the restore area.
322
323 // Copy remaining restore values from the Probe::State to the restore area.
324 // Note: We already ensured above that the Probe::State is in a safe location before
325 // calling the initializeStackFunction. The initializeStackFunction is not allowed to
326 // change the stack pointer again.
327 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EFLAGS_OFFSET) "(%ebp), %eax" "\n"
328 "movl %eax, 0 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%ecx)" "\n"
329 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EAX_OFFSET) "(%ebp), %eax" "\n"
330 "movl %eax, 1 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%ecx)" "\n"
331 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_ECX_OFFSET) "(%ebp), %eax" "\n"
332 "movl %eax, 2 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%ecx)" "\n"
333 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EBP_OFFSET) "(%ebp), %eax" "\n"
334 "movl %eax, 3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%ecx)" "\n"
335 "movl " STRINGIZE_VALUE_OF(PROBE_CPU_EIP_OFFSET) "(%ebp), %eax" "\n"
336 "movl %eax, 4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%ecx)" "\n"
337 "movl %ecx, %esp" "\n"
338
339 // Do the remaining restoration by popping off the restore area.
340 "popfl" "\n"
341 "popl %eax" "\n"
342 "popl %ecx" "\n"
343 "popl %ebp" "\n"
344 "ret" "\n"
345);
346#endif
347
348#if COMPILER(MSVC)
349extern "C" __declspec(naked) void ctiMasmProbeTrampoline()
350{
351 __asm {
352 pushfd;
353
354 // MacroAssemblerX86Common::probe() has already generated code to store some values.
355 // Together with the eflags pushed above, the top of stack now looks like
356 // this:
357 // esp[0 * ptrSize]: eflags
358 // esp[1 * ptrSize]: return address / saved eip
359 // esp[2 * ptrSize]: saved ebx
360 // esp[3 * ptrSize]: saved edx
361 // esp[4 * ptrSize]: saved ecx
362 // esp[5 * ptrSize]: saved eax
363 //
364 // Incoming registers contain:
365 // ecx: Probe::executeProbe
366 // edx: probe function
367 // ebx: probe arg
368 // eax: scratch (was ctiMasmProbeTrampoline)
369
370 mov eax, esp
371 sub esp, PROBE_SIZE + OUT_SIZE
372
373 // The X86_64 ABI specifies that the worse case stack alignment requirement is 32 bytes.
374 and esp, ~0x1f
375
376 mov [PROBE_CPU_EBP_OFFSET + esp], ebp
377 mov ebp, esp // Save the ProbeContext*.
378
379 mov [PROBE_EXECUTOR_OFFSET + ebp], ecx
380 mov [PROBE_PROBE_FUNCTION_OFFSET + ebp], edx
381 mov [PROBE_ARG_OFFSET + ebp], ebx
382 mov [PROBE_CPU_ESI_OFFSET + ebp], esi
383 mov [PROBE_CPU_EDI_OFFSET + ebp], edi
384
385 mov ecx, [0 * PTR_SIZE + eax]
386 mov [PROBE_CPU_EFLAGS_OFFSET + ebp], ecx
387 mov ecx, [1 * PTR_SIZE + eax]
388 mov [PROBE_CPU_EIP_OFFSET + ebp], ecx
389 mov ecx, [2 * PTR_SIZE + eax]
390 mov [PROBE_CPU_EBX_OFFSET + ebp], ecx
391 mov ecx, [3 * PTR_SIZE + eax]
392 mov [PROBE_CPU_EDX_OFFSET + ebp], ecx
393 mov ecx, [4 * PTR_SIZE + eax]
394 mov [PROBE_CPU_ECX_OFFSET + ebp], ecx
395 mov ecx, [5 * PTR_SIZE + eax]
396 mov [PROBE_CPU_EAX_OFFSET + ebp], ecx
397
398 mov ecx, eax
399 add ecx, 6 * PTR_SIZE
400 mov [PROBE_CPU_ESP_OFFSET + ebp], ecx
401
402 movq qword ptr[PROBE_CPU_XMM0_OFFSET + ebp], xmm0
403 movq qword ptr[PROBE_CPU_XMM1_OFFSET + ebp], xmm1
404 movq qword ptr[PROBE_CPU_XMM2_OFFSET + ebp], xmm2
405 movq qword ptr[PROBE_CPU_XMM3_OFFSET + ebp], xmm3
406 movq qword ptr[PROBE_CPU_XMM4_OFFSET + ebp], xmm4
407 movq qword ptr[PROBE_CPU_XMM5_OFFSET + ebp], xmm5
408 movq qword ptr[PROBE_CPU_XMM6_OFFSET + ebp], xmm6
409 movq qword ptr[PROBE_CPU_XMM7_OFFSET + ebp], xmm7
410
411 // Reserve stack space for the arg while maintaining the required stack
412 // pointer 32 byte alignment:
413 sub esp, 0x20
414 mov [0 + esp], ebp // the ProbeContext* arg.
415
416 call [PROBE_EXECUTOR_OFFSET + ebp]
417
418 // Make sure the ProbeContext is entirely below the result stack pointer so
419 // that register values are still preserved when we call the initializeStack
420 // function.
421 mov ecx, PROBE_SIZE + OUT_SIZE
422 mov eax, ebp
423 mov edx, [PROBE_CPU_ESP_OFFSET + ebp]
424 add eax, ecx
425 cmp edx, eax
426 jge ctiMasmProbeTrampolineProbeContextIsSafe
427
428 // Allocate a safe place on the stack below the result stack pointer to stash the ProbeContext.
429 sub edx, ecx
430 and edx, ~0x1f // Keep the stack pointer 32 bytes aligned.
431 xor eax, eax
432 mov esp, edx
433
434 mov ecx, PROBE_SIZE
435
436 // Copy the ProbeContext to the safe place.
437 ctiMasmProbeTrampolineCopyLoop :
438 mov edx, [ebp + eax]
439 mov [esp + eax], edx
440 add eax, PTR_SIZE
441 cmp ecx, eax
442 jg ctiMasmProbeTrampolineCopyLoop
443
444 mov ebp, esp
445
446 // Call initializeStackFunction if present.
447 ctiMasmProbeTrampolineProbeContextIsSafe :
448 xor ecx, ecx
449 add ecx, [PROBE_INIT_STACK_FUNCTION_OFFSET + ebp]
450 je ctiMasmProbeTrampolineRestoreRegisters
451
452 // Reserve stack space for the arg while maintaining the required stack
453 // pointer 32 byte alignment:
454 sub esp, 0x20
455 mov [0 + esp], ebp // the ProbeContext* arg.
456 call ecx
457
458 ctiMasmProbeTrampolineRestoreRegisters :
459
460 // To enable probes to modify register state, we copy all registers
461 // out of the ProbeContext before returning.
462
463 mov edx, [PROBE_CPU_EDX_OFFSET + ebp]
464 mov ebx, [PROBE_CPU_EBX_OFFSET + ebp]
465 mov esi, [PROBE_CPU_ESI_OFFSET + ebp]
466 mov edi, [PROBE_CPU_EDI_OFFSET + ebp]
467
468 movq xmm0, qword ptr[PROBE_CPU_XMM0_OFFSET + ebp]
469 movq xmm1, qword ptr[PROBE_CPU_XMM1_OFFSET + ebp]
470 movq xmm2, qword ptr[PROBE_CPU_XMM2_OFFSET + ebp]
471 movq xmm3, qword ptr[PROBE_CPU_XMM3_OFFSET + ebp]
472 movq xmm4, qword ptr[PROBE_CPU_XMM4_OFFSET + ebp]
473 movq xmm5, qword ptr[PROBE_CPU_XMM5_OFFSET + ebp]
474 movq xmm6, qword ptr[PROBE_CPU_XMM6_OFFSET + ebp]
475 movq xmm7, qword ptr[PROBE_CPU_XMM7_OFFSET + ebp]
476
477 // There are 6 more registers left to restore:
478 // eax, ecx, ebp, esp, eip, and eflags.
479
480 // The restoration process at ctiMasmProbeTrampolineEnd below works by popping
481 // 5 words off the stack into eflags, eax, ecx, ebp, and eip. These 5 words need
482 // to be pushed on top of the final esp value so that just by popping the 5 words,
483 // we'll get the esp that the probe wants to set. Let's call this area (for storing
484 // these 5 words) the restore area.
485 mov ecx, [PROBE_CPU_ESP_OFFSET + ebp]
486 sub ecx, 5 * PTR_SIZE
487
488 // ecx now points to the restore area.
489
490 // Copy remaining restore values from the ProbeContext to the restore area.
491 // Note: We already ensured above that the ProbeContext is in a safe location before
492 // calling the initializeStackFunction. The initializeStackFunction is not allowed to
493 // change the stack pointer again.
494 mov eax, [PROBE_CPU_EFLAGS_OFFSET + ebp]
495 mov [0 * PTR_SIZE + ecx], eax
496 mov eax, [PROBE_CPU_EAX_OFFSET + ebp]
497 mov [1 * PTR_SIZE + ecx], eax
498 mov eax, [PROBE_CPU_ECX_OFFSET + ebp]
499 mov [2 * PTR_SIZE + ecx], eax
500 mov eax, [PROBE_CPU_EBP_OFFSET + ebp]
501 mov [3 * PTR_SIZE + ecx], eax
502 mov eax, [PROBE_CPU_EIP_OFFSET + ebp]
503 mov [4 * PTR_SIZE + ecx], eax
504 mov esp, ecx
505
506 // Do the remaining restoration by popping off the restore area.
507 popfd
508 pop eax
509 pop ecx
510 pop ebp
511 ret
512 }
513}
514#endif // COMPILER(MSVC)
515
516#endif // CPU(X86)
517
518#if CPU(X86_64)
519#if COMPILER(GCC_COMPATIBLE)
520asm (
521 ".globl " SYMBOL_STRING(ctiMasmProbeTrampoline) "\n"
522 HIDE_SYMBOL(ctiMasmProbeTrampoline) "\n"
523 SYMBOL_STRING(ctiMasmProbeTrampoline) ":" "\n"
524
525 "pushfq" "\n"
526
527 // MacroAssemblerX86Common::probe() has already generated code to store some values.
528 // Together with the rflags pushed above, the top of stack now looks like this:
529 // rsp[0 * ptrSize]: rflags
530 // rsp[1 * ptrSize]: return address / saved rip
531 // rsp[2 * ptrSize]: saved rbx
532 // rsp[3 * ptrSize]: saved rdx
533 // rsp[4 * ptrSize]: saved rcx
534 // rsp[5 * ptrSize]: saved rax
535 //
536 // Incoming registers contain:
537 // rcx: Probe::executeProbe
538 // rdx: probe function
539 // rbx: probe arg
540 // rax: scratch (was ctiMasmProbeTrampoline)
541
542 "movq %rsp, %rax" "\n"
543 "subq $" STRINGIZE_VALUE_OF(PROBE_SIZE + OUT_SIZE) ", %rsp" "\n"
544
545 // The X86_64 ABI specifies that the worse case stack alignment requirement is 32 bytes.
546 "andq $~0x1f, %rsp" "\n"
547 // Since sp points to the Probe::State, we've ensured that it's protected from interrupts before we initialize it.
548
549 "movq %rbp, " STRINGIZE_VALUE_OF(PROBE_CPU_EBP_OFFSET) "(%rsp)" "\n"
550 "movq %rsp, %rbp" "\n" // Save the Probe::State*.
551
552 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_EXECUTOR_OFFSET) "(%rbp)" "\n"
553 "movq %rdx, " STRINGIZE_VALUE_OF(PROBE_PROBE_FUNCTION_OFFSET) "(%rbp)" "\n"
554 "movq %rbx, " STRINGIZE_VALUE_OF(PROBE_ARG_OFFSET) "(%rbp)" "\n"
555 "movq %rsi, " STRINGIZE_VALUE_OF(PROBE_CPU_ESI_OFFSET) "(%rbp)" "\n"
556 "movq %rdi, " STRINGIZE_VALUE_OF(PROBE_CPU_EDI_OFFSET) "(%rbp)" "\n"
557
558 "movq 0 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
559 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_EFLAGS_OFFSET) "(%rbp)" "\n"
560 "movq 1 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
561 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_EIP_OFFSET) "(%rbp)" "\n"
562 "movq 2 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
563 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_EBX_OFFSET) "(%rbp)" "\n"
564 "movq 3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
565 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_EDX_OFFSET) "(%rbp)" "\n"
566 "movq 4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
567 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_ECX_OFFSET) "(%rbp)" "\n"
568 "movq 5 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rax), %rcx" "\n"
569 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_EAX_OFFSET) "(%rbp)" "\n"
570
571 "movq %rax, %rcx" "\n"
572 "addq $" STRINGIZE_VALUE_OF(6 * PTR_SIZE) ", %rcx" "\n"
573 "movq %rcx, " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%rbp)" "\n"
574
575 "movq %r8, " STRINGIZE_VALUE_OF(PROBE_CPU_R8_OFFSET) "(%rbp)" "\n"
576 "movq %r9, " STRINGIZE_VALUE_OF(PROBE_CPU_R9_OFFSET) "(%rbp)" "\n"
577 "movq %r10, " STRINGIZE_VALUE_OF(PROBE_CPU_R10_OFFSET) "(%rbp)" "\n"
578 "movq %r11, " STRINGIZE_VALUE_OF(PROBE_CPU_R11_OFFSET) "(%rbp)" "\n"
579 "movq %r12, " STRINGIZE_VALUE_OF(PROBE_CPU_R12_OFFSET) "(%rbp)" "\n"
580 "movq %r13, " STRINGIZE_VALUE_OF(PROBE_CPU_R13_OFFSET) "(%rbp)" "\n"
581 "movq %r14, " STRINGIZE_VALUE_OF(PROBE_CPU_R14_OFFSET) "(%rbp)" "\n"
582 "movq %r15, " STRINGIZE_VALUE_OF(PROBE_CPU_R15_OFFSET) "(%rbp)" "\n"
583
584 "movq %xmm0, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM0_OFFSET) "(%rbp)" "\n"
585 "movq %xmm1, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM1_OFFSET) "(%rbp)" "\n"
586 "movq %xmm2, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM2_OFFSET) "(%rbp)" "\n"
587 "movq %xmm3, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM3_OFFSET) "(%rbp)" "\n"
588 "movq %xmm4, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM4_OFFSET) "(%rbp)" "\n"
589 "movq %xmm5, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM5_OFFSET) "(%rbp)" "\n"
590 "movq %xmm6, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM6_OFFSET) "(%rbp)" "\n"
591 "movq %xmm7, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM7_OFFSET) "(%rbp)" "\n"
592 "movq %xmm8, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM8_OFFSET) "(%rbp)" "\n"
593 "movq %xmm9, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM9_OFFSET) "(%rbp)" "\n"
594 "movq %xmm10, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM10_OFFSET) "(%rbp)" "\n"
595 "movq %xmm11, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM11_OFFSET) "(%rbp)" "\n"
596 "movq %xmm12, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM12_OFFSET) "(%rbp)" "\n"
597 "movq %xmm13, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM13_OFFSET) "(%rbp)" "\n"
598 "movq %xmm14, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM14_OFFSET) "(%rbp)" "\n"
599 "movq %xmm15, " STRINGIZE_VALUE_OF(PROBE_CPU_XMM15_OFFSET) "(%rbp)" "\n"
600
601 "movq %rbp, %rdi" "\n" // the Probe::State* arg.
602 "call *" STRINGIZE_VALUE_OF(PROBE_EXECUTOR_OFFSET) "(%rbp)" "\n"
603
604 // Make sure the Probe::State is entirely below the result stack pointer so
605 // that register values are still preserved when we call the initializeStack
606 // function.
607 "movq $" STRINGIZE_VALUE_OF(PROBE_SIZE + OUT_SIZE) ", %rcx" "\n"
608 "movq %rbp, %rax" "\n"
609 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%rbp), %rdx" "\n"
610 "addq %rcx, %rax" "\n"
611 "cmpq %rax, %rdx" "\n"
612 "jge " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) "\n"
613
614 // Allocate a safe place on the stack below the result stack pointer to stash the Probe::State.
615 "subq %rcx, %rdx" "\n"
616 "andq $~0x1f, %rdx" "\n" // Keep the stack pointer 32 bytes aligned.
617 "xorq %rax, %rax" "\n"
618 "movq %rdx, %rsp" "\n"
619
620 "movq $" STRINGIZE_VALUE_OF(PROBE_SIZE) ", %rcx" "\n"
621
622 // Copy the Probe::State to the safe place.
623 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) ":" "\n"
624 "movq (%rbp, %rax), %rdx" "\n"
625 "movq %rdx, (%rsp, %rax)" "\n"
626 "addq $" STRINGIZE_VALUE_OF(PTR_SIZE) ", %rax" "\n"
627 "cmpq %rax, %rcx" "\n"
628 "jg " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineCopyLoop) "\n"
629
630 "movq %rsp, %rbp" "\n"
631
632 // Call initializeStackFunction if present.
633 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineProbeStateIsSafe) ":" "\n"
634 "xorq %rcx, %rcx" "\n"
635 "addq " STRINGIZE_VALUE_OF(PROBE_INIT_STACK_FUNCTION_OFFSET) "(%rbp), %rcx" "\n"
636 "je " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) "\n"
637
638 "movq %rbp, %rdi" "\n" // the Probe::State* arg.
639 "call *%rcx" "\n"
640
641 LOCAL_LABEL_STRING(ctiMasmProbeTrampolineRestoreRegisters) ":" "\n"
642
643 // To enable probes to modify register state, we copy all registers
644 // out of the Probe::State before returning.
645
646 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EDX_OFFSET) "(%rbp), %rdx" "\n"
647 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EBX_OFFSET) "(%rbp), %rbx" "\n"
648 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_ESI_OFFSET) "(%rbp), %rsi" "\n"
649 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EDI_OFFSET) "(%rbp), %rdi" "\n"
650
651 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R8_OFFSET) "(%rbp), %r8" "\n"
652 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R9_OFFSET) "(%rbp), %r9" "\n"
653 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R10_OFFSET) "(%rbp), %r10" "\n"
654 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R11_OFFSET) "(%rbp), %r11" "\n"
655 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R12_OFFSET) "(%rbp), %r12" "\n"
656 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R13_OFFSET) "(%rbp), %r13" "\n"
657 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R14_OFFSET) "(%rbp), %r14" "\n"
658 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_R15_OFFSET) "(%rbp), %r15" "\n"
659
660 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM0_OFFSET) "(%rbp), %xmm0" "\n"
661 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM1_OFFSET) "(%rbp), %xmm1" "\n"
662 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM2_OFFSET) "(%rbp), %xmm2" "\n"
663 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM3_OFFSET) "(%rbp), %xmm3" "\n"
664 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM4_OFFSET) "(%rbp), %xmm4" "\n"
665 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM5_OFFSET) "(%rbp), %xmm5" "\n"
666 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM6_OFFSET) "(%rbp), %xmm6" "\n"
667 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM7_OFFSET) "(%rbp), %xmm7" "\n"
668 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM8_OFFSET) "(%rbp), %xmm8" "\n"
669 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM9_OFFSET) "(%rbp), %xmm9" "\n"
670 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM10_OFFSET) "(%rbp), %xmm10" "\n"
671 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM11_OFFSET) "(%rbp), %xmm11" "\n"
672 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM12_OFFSET) "(%rbp), %xmm12" "\n"
673 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM13_OFFSET) "(%rbp), %xmm13" "\n"
674 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM14_OFFSET) "(%rbp), %xmm14" "\n"
675 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_XMM15_OFFSET) "(%rbp), %xmm15" "\n"
676
677 // There are 6 more registers left to restore:
678 // rax, rcx, rbp, rsp, rip, and rflags.
679
680 // The restoration process at ctiMasmProbeTrampolineEnd below works by popping
681 // 5 words off the stack into rflags, rax, rcx, rbp, and rip. These 5 words need
682 // to be pushed on top of the final esp value so that just by popping the 5 words,
683 // we'll get the esp that the probe wants to set. Let's call this area (for storing
684 // these 5 words) the restore area.
685 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_ESP_OFFSET) "(%rbp), %rcx" "\n"
686 "subq $5 * " STRINGIZE_VALUE_OF(PTR_SIZE) ", %rcx" "\n"
687
688 // rcx now points to the restore area.
689
690 // Copy remaining restore values from the Probe::State to the restore area.
691 // Note: We already ensured above that the Probe::State is in a safe location before
692 // calling the initializeStackFunction. The initializeStackFunction is not allowed to
693 // change the stack pointer again.
694 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EFLAGS_OFFSET) "(%rbp), %rax" "\n"
695 "movq %rax, 0 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rcx)" "\n"
696 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EAX_OFFSET) "(%rbp), %rax" "\n"
697 "movq %rax, 1 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rcx)" "\n"
698 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_ECX_OFFSET) "(%rbp), %rax" "\n"
699 "movq %rax, 2 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rcx)" "\n"
700 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EBP_OFFSET) "(%rbp), %rax" "\n"
701 "movq %rax, 3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rcx)" "\n"
702 "movq " STRINGIZE_VALUE_OF(PROBE_CPU_EIP_OFFSET) "(%rbp), %rax" "\n"
703 "movq %rax, 4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "(%rcx)" "\n"
704 "movq %rcx, %rsp" "\n"
705
706 // Do the remaining restoration by popping off the restore area.
707 "popfq" "\n"
708 "popq %rax" "\n"
709 "popq %rcx" "\n"
710 "popq %rbp" "\n"
711 "ret" "\n"
712);
713#endif // COMPILER(GCC_COMPATIBLE)
714#endif // CPU(X86_64)
715
716// What code is emitted for the probe?
717// ==================================
718// We want to keep the size of the emitted probe invocation code as compact as
719// possible to minimize the perturbation to the JIT generated code. However,
720// we also need to preserve the CPU registers and set up the Probe::State to be
721// passed to the user probe function.
722//
723// Hence, we do only the minimum here to preserve a scratch register (i.e. rax
724// in this case) and the stack pointer (i.e. rsp), and pass the probe arguments.
725// We'll let the ctiMasmProbeTrampoline handle the rest of the probe invocation
726// work i.e. saving the CPUState (and setting up the Probe::State), calling the
727// user probe function, and restoring the CPUState before returning to JIT
728// generated code.
729//
730// What registers need to be saved?
731// ===============================
732// The registers are saved for 2 reasons:
733// 1. To preserve their state in the JITted code. This means that all registers
734// that are not callee saved needs to be saved. We also need to save the
735// condition code registers because the probe can be inserted between a test
736// and a branch.
737// 2. To allow the probe to inspect the values of the registers for debugging
738// purposes. This means all registers need to be saved.
739//
740// In summary, save everything. But for reasons stated above, we should do the
741// minimum here and let ctiMasmProbeTrampoline do the heavy lifting to save the
742// full set.
743//
744// What values are in the saved registers?
745// ======================================
746// Conceptually, the saved registers should contain values as if the probe
747// is not present in the JIT generated code. Hence, they should contain values
748// that are expected at the start of the instruction immediately following the
749// probe.
750//
751// Specifically, the saved stack pointer register will point to the stack
752// position before we push the Probe::State frame. The saved rip will point to
753// the address of the instruction immediately following the probe.
754
755void MacroAssembler::probe(Probe::Function function, void* arg)
756{
757 push(RegisterID::eax);
758 move(TrustedImmPtr(reinterpret_cast<void*>(ctiMasmProbeTrampoline)), RegisterID::eax);
759 push(RegisterID::ecx);
760 move(TrustedImmPtr(reinterpret_cast<void*>(Probe::executeProbe)), RegisterID::ecx);
761 push(RegisterID::edx);
762 move(TrustedImmPtr(reinterpret_cast<void*>(function)), RegisterID::edx);
763 push(RegisterID::ebx);
764 move(TrustedImmPtr(arg), RegisterID::ebx);
765 call(RegisterID::eax, CFunctionPtrTag);
766}
767#endif // ENABLE(MASM_PROBE)
768
769MacroAssemblerX86Common::CPUID MacroAssemblerX86Common::getCPUID(unsigned level)
770{
771 return getCPUIDEx(level, 0);
772}
773
774MacroAssemblerX86Common::CPUID MacroAssemblerX86Common::getCPUIDEx(unsigned level, unsigned count)
775{
776 CPUID result { };
777#if COMPILER(MSVC)
778 __cpuidex(bitwise_cast<int*>(result.data()), level, count);
779#else
780 __asm__ (
781 "cpuid\n"
782 : "=a"(result[0]), "=b"(result[1]), "=c"(result[2]), "=d"(result[3])
783 : "0"(level), "2"(count)
784 );
785#endif
786 return result;
787}
788
789void MacroAssemblerX86Common::collectCPUFeatures()
790{
791 static std::once_flag onceKey;
792 std::call_once(onceKey, [] {
793 {
794 CPUID cpuid = getCPUID(0x1);
795 s_sse2CheckState = (cpuid[3] & (1 << 26)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
796 s_sse4_1CheckState = (cpuid[2] & (1 << 19)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
797 s_sse4_2CheckState = (cpuid[2] & (1 << 20)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
798 s_popcntCheckState = (cpuid[2] & (1 << 23)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
799 s_avxCheckState = (cpuid[2] & (1 << 28)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
800 }
801 {
802 CPUID cpuid = getCPUID(0x7);
803 s_bmi1CheckState = (cpuid[2] & (1 << 3)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
804 }
805 {
806 CPUID cpuid = getCPUID(0x80000001);
807 s_lzcntCheckState = (cpuid[2] & (1 << 5)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear;
808 }
809 });
810}
811
812MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_sse2CheckState = CPUIDCheckState::NotChecked;
813MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_sse4_1CheckState = CPUIDCheckState::NotChecked;
814MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_sse4_2CheckState = CPUIDCheckState::NotChecked;
815MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_avxCheckState = CPUIDCheckState::NotChecked;
816MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_lzcntCheckState = CPUIDCheckState::NotChecked;
817MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_bmi1CheckState = CPUIDCheckState::NotChecked;
818MacroAssemblerX86Common::CPUIDCheckState MacroAssemblerX86Common::s_popcntCheckState = CPUIDCheckState::NotChecked;
819
820} // namespace JSC
821
822#endif // ENABLE(ASSEMBLER) && (CPU(X86) || CPU(X86_64))
823