Skip to content

GH-135904: Add tests for the JIT build process #136766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open

GH-135904: Add tests for the JIT build process #136766

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
7a6d819
Test the JIT stencils build process
brandtbucher Jul 15, 2025
a322ad4
Cleanup
brandtbucher Jul 16, 2025
e1eb85d
Add expected output for Windows
brandtbucher Jul 16, 2025
f4c05b3
Add expected output for macOS
brandtbucher Jul 16, 2025
00cd7e3
Is it BSS?
brandtbucher Jul 16, 2025
202fb6f
It *is* BSS!
brandtbucher Jul 17, 2025
4e5554c
Optimistically strip writable data
brandtbucher Jul 17, 2025
9db0563
Ditto for ELF
brandtbucher Jul 17, 2025
c4c3ccc
Ditto for Mach-O, and *way* simplify the parsing!
brandtbucher Jul 17, 2025
c6fc7bd
Rework tests
brandtbucher Jul 17, 2025
558c83b
Merge branch 'jit-tests' of https://github.com/brandtbucher/cpython i…
brandtbucher Jul 17, 2025
867a686
Cleanup for PR
brandtbucher Jul 18, 2025
7c9c3ff
Check for all of the tools
brandtbucher Jul 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add expected output for macOS
  • Loading branch information
@brandtbucher
brandtbucher committed Jul 16, 2025
commit f4c05b3d8346b701e82aeba634d35078fa492efd
4 changes: 4 additions & 0 deletions Lib/test/test_jit_stencils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,7 @@ def test_jit_stencils(self):
self._check_jit_stencils(expected, actual, test_jit_stencils_h)
# This is a local build. If the JIT is available, at least one test should run:
assert found or not sys._jit.is_available(), "No JIT stencils built!"


if __name__ == "__main__":
unittest.main()
11 changes: 5 additions & 6 deletions Tools/jit/_targets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,12 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
lines = output.splitlines()
started = False
for line in lines:
if not started:
if "_JIT_ENTRY" not in line:
continue
if line.lstrip().startswith("0:"):
started = True
cleaned = line.replace(long, short).expandtabs().strip()
if cleaned:
group.code.disassembly.append(cleaned)
if started:
cleaned = line.replace(long, short).expandtabs().strip()
if cleaned:
group.code.disassembly.append(cleaned)
args = [
"--elf-output-style=JSON",
"--expand-relocs",
Expand Down
149 changes: 149 additions & 0 deletions Tools/jit/test/test_jit_stencils-aarch64-apple-darwin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
void
emit_shim(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 6db63bef stp d15, d14, [sp, #-0xa0]!
// 4: 6d0133ed stp d13, d12, [sp, #0x10]
// 8: 6d022beb stp d11, d10, [sp, #0x20]
// c: 6d0323e9 stp d9, d8, [sp, #0x30]
// 10: a9046ffc stp x28, x27, [sp, #0x40]
// 14: a90567fa stp x26, x25, [sp, #0x50]
// 18: a9065ff8 stp x24, x23, [sp, #0x60]
// 1c: a90757f6 stp x22, x21, [sp, #0x70]
// 20: a9084ff4 stp x20, x19, [sp, #0x80]
// 24: a9097bfd stp x29, x30, [sp, #0x90]
// 28: 910243fd add x29, sp, #0x90
// 2c: aa0003f4 mov x20, x0
// 30: aa0103f5 mov x21, x1
// 34: aa0203f6 mov x22, x2
// 38: 9400000c bl 0x68 <ltmp0+0x68>
// 3c: a9497bfd ldp x29, x30, [sp, #0x90]
// 40: a9484ff4 ldp x20, x19, [sp, #0x80]
// 44: a94757f6 ldp x22, x21, [sp, #0x70]
// 48: a9465ff8 ldp x24, x23, [sp, #0x60]
// 4c: a94567fa ldp x26, x25, [sp, #0x50]
// 50: a9446ffc ldp x28, x27, [sp, #0x40]
// 54: 6d4323e9 ldp d9, d8, [sp, #0x30]
// 58: 6d422beb ldp d11, d10, [sp, #0x20]
// 5c: 6d4133ed ldp d13, d12, [sp, #0x10]
// 60: 6cca3bef ldp d15, d14, [sp], #0xa0
// 64: d65f03c0 ret
const unsigned char code_body[104] = {
0xef, 0x3b, 0xb6, 0x6d, 0xed, 0x33, 0x01, 0x6d,
0xeb, 0x2b, 0x02, 0x6d, 0xe9, 0x23, 0x03, 0x6d,
0xfc, 0x6f, 0x04, 0xa9, 0xfa, 0x67, 0x05, 0xa9,
0xf8, 0x5f, 0x06, 0xa9, 0xf6, 0x57, 0x07, 0xa9,
0xf4, 0x4f, 0x08, 0xa9, 0xfd, 0x7b, 0x09, 0xa9,
0xfd, 0x43, 0x02, 0x91, 0xf4, 0x03, 0x00, 0xaa,
0xf5, 0x03, 0x01, 0xaa, 0xf6, 0x03, 0x02, 0xaa,
0x0c, 0x00, 0x00, 0x94, 0xfd, 0x7b, 0x49, 0xa9,
0xf4, 0x4f, 0x48, 0xa9, 0xf6, 0x57, 0x47, 0xa9,
0xf8, 0x5f, 0x46, 0xa9, 0xfa, 0x67, 0x45, 0xa9,
0xfc, 0x6f, 0x44, 0xa9, 0xe9, 0x23, 0x43, 0x6d,
0xeb, 0x2b, 0x42, 0x6d, 0xed, 0x33, 0x41, 0x6d,
0xef, 0x3b, 0xca, 0x6c, 0xc0, 0x03, 0x5f, 0xd6,
};
memcpy(code, code_body, sizeof(code_body));
}

void
emit_0(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
}

void
emit_1(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: a9bf7bfd stp x29, x30, [sp, #-0x10]!
// 4: 910003fd mov x29, sp
// 8: 90000008 adrp x8, 0x0 <ltmp0>
// 0000000000000008: ARM64_RELOC_GOT_LOAD_PAGE21 _sausage
// c: f9400108 ldr x8, [x8]
// 000000000000000c: ARM64_RELOC_GOT_LOAD_PAGEOFF12 _sausage
// 10: 39400108 ldrb w8, [x8]
// 14: 36000068 tbz w8, #0x0, 0x20 <ltmp0+0x20>
// 18: 94000000 bl 0x18 <ltmp0+0x18>
// 0000000000000018: ARM64_RELOC_BRANCH26 _order_eggs_sausage_and_bacon
// 1c: 14000002 b 0x24 <ltmp0+0x24>
// 20: 94000000 bl 0x20 <ltmp0+0x20>
// 0000000000000020: ARM64_RELOC_BRANCH26 _order_eggs_and_bacon
// 24: 90000008 adrp x8, 0x0 <ltmp0>
// 0000000000000024: ARM64_RELOC_GOT_LOAD_PAGE21 _spammed
// 28: f9400108 ldr x8, [x8]
// 0000000000000028: ARM64_RELOC_GOT_LOAD_PAGEOFF12 _spammed
// 2c: 3900011f strb wzr, [x8]
// 30: a8c17bfd ldp x29, x30, [sp], #0x10
const unsigned char code_body[52] = {
0xfd, 0x7b, 0xbf, 0xa9, 0xfd, 0x03, 0x00, 0x91,
0x08, 0x00, 0x00, 0x90, 0x08, 0x01, 0x40, 0xf9,
0x08, 0x01, 0x40, 0x39, 0x68, 0x00, 0x00, 0x36,
0x00, 0x00, 0x00, 0x94, 0x02, 0x00, 0x00, 0x14,
0x00, 0x00, 0x00, 0x94, 0x08, 0x00, 0x00, 0x90,
0x08, 0x01, 0x40, 0xf9, 0x1f, 0x01, 0x00, 0x39,
0xfd, 0x7b, 0xc1, 0xa8,
};
// 0: &spammed+0x0
// 8: &sausage+0x0
patch_64(data + 0x0, (uintptr_t)&spammed);
patch_64(data + 0x8, (uintptr_t)&sausage);
memcpy(code, code_body, sizeof(code_body));
patch_aarch64_33rx(code + 0x8, (uintptr_t)data + 0x8);
patch_aarch64_trampoline(code + 0x18, 0x1, state);
patch_aarch64_trampoline(code + 0x20, 0x0, state);
patch_aarch64_33rx(code + 0x24, (uintptr_t)data);
}

void
emit_2(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 90000008 adrp x8, 0x0 <ltmp0>
// 0000000000000000: ARM64_RELOC_GOT_LOAD_PAGE21 _spam
// 4: f9400108 ldr x8, [x8]
// 0000000000000004: ARM64_RELOC_GOT_LOAD_PAGEOFF12 _spam
// 8: 39400108 ldrb w8, [x8]
// c: 7100051f cmp w8, #0x1
// 10: 54000041 b.ne 0x18 <ltmp0+0x18>
// 14: 14000000 b 0x14 <ltmp0+0x14>
// 0000000000000014: ARM64_RELOC_BRANCH26 __JIT_ERROR_TARGET
const unsigned char code_body[24] = {
0x08, 0x00, 0x00, 0x90, 0x08, 0x01, 0x40, 0xf9,
0x08, 0x01, 0x40, 0x39, 0x1f, 0x05, 0x00, 0x71,
0x41, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x14,
};
// 0: &spam+0x0
patch_64(data + 0x0, (uintptr_t)&spam);
memcpy(code, code_body, sizeof(code_body));
patch_aarch64_33rx(code + 0x0, (uintptr_t)data);
patch_aarch64_26r(code + 0x14, state->instruction_starts[instruction->error_target]);
}

static_assert(SYMBOL_MASK_WORDS >= 1, "SYMBOL_MASK_WORDS too small");

typedef struct {
void (*emit)(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state);
size_t code_size;
size_t data_size;
symbol_mask trampoline_mask;
} StencilGroup;

static const StencilGroup shim = {emit_shim, 104, 0, {0}};

static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {
[0] = {emit_0, 0, 0, {0}},
[1] = {emit_1, 52, 16, {0x03}},
[2] = {emit_2, 24, 8, {0}},
};

static const void * const symbols_map[2] = {
[0] = &order_eggs_and_bacon,
[1] = &order_eggs_sausage_and_bacon,
};
3 changes: 0 additions & 3 deletions Tools/jit/test/test_jit_stencils-aarch64-pc-windows-msvc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ emit_shim(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: 6db63bef stp d15, d14, [sp, #-0xa0]!
// 4: a90857f6 stp x22, x21, [sp, #0x80]
// 8: aa0103f5 mov x21, x1
Expand Down Expand Up @@ -59,7 +58,6 @@ emit_1(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: f81f0ffe str x30, [sp, #-0x10]!
// 4: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
// 0000000000000004: IMAGE_REL_ARM64_PAGEBASE_REL21 __imp_sausage
Expand Down Expand Up @@ -113,7 +111,6 @@ emit_2(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
// 0000000000000000: IMAGE_REL_ARM64_PAGEBASE_REL21 __imp_spam
// 4: f9400108 ldr x8, [x8]
Expand Down
3 changes: 0 additions & 3 deletions Tools/jit/test/test_jit_stencils-aarch64-unknown-linux-gnu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ emit_shim(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: 6db63bef stp d15, d14, [sp, #-0xa0]!
// 4: a90857f6 stp x22, x21, [sp, #0x80]
// 8: aa0103f5 mov x21, x1
Expand Down Expand Up @@ -60,7 +59,6 @@ emit_1(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: a9bf7bfd stp x29, x30, [sp, #-0x10]!
// 4: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
// 0000000000000004: R_AARCH64_ADR_GOT_PAGE sausage
Expand Down Expand Up @@ -116,7 +114,6 @@ emit_2(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0000000000000000 <_JIT_ENTRY>:
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
// 0000000000000000: R_AARCH64_ADR_GOT_PAGE spam
// 4: f9400108 ldr x8, [x8]
Expand Down
3 changes: 0 additions & 3 deletions Tools/jit/test/test_jit_stencils-i686-pc-windows-msvc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ emit_0(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 00000000 <__JIT_ENTRY>:
// 0: 8b 44 24 0c movl 0xc(%esp), %eax
// 4: 8b 4c 24 08 movl 0x8(%esp), %ecx
// 8: 8b 54 24 04 movl 0x4(%esp), %edx
Expand All @@ -30,7 +29,6 @@ emit_1(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 00000000 <__JIT_ENTRY>:
// 0: 53 pushl %ebx
// 1: 57 pushl %edi
// 2: 56 pushl %esi
Expand Down Expand Up @@ -75,7 +73,6 @@ emit_2(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 00000000 <__JIT_ENTRY>:
// 0: 8b 54 24 0c movl 0xc(%esp), %edx
// 4: 8b 4c 24 08 movl 0x8(%esp), %ecx
// 8: 8b 44 24 04 movl 0x4(%esp), %eax
Expand Down
142 changes: 142 additions & 0 deletions Tools/jit/test/test_jit_stencils-x86_64-apple-darwin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
void
emit_shim(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 55 pushq %rbp
// 1: 48 89 e5 movq %rsp, %rbp
// 4: 41 57 pushq %r15
// 6: 41 56 pushq %r14
// 8: 41 55 pushq %r13
// a: 41 54 pushq %r12
// c: 53 pushq %rbx
// d: 50 pushq %rax
// e: 49 89 fc movq %rdi, %r12
// 11: 49 89 f5 movq %rsi, %r13
// 14: 49 89 d6 movq %rdx, %r14
// 17: e8 0f 00 00 00 callq 0x2b <__JIT_ENTRY+0x2b>
// 1c: 48 83 c4 08 addq $0x8, %rsp
// 20: 5b popq %rbx
// 21: 41 5c popq %r12
// 23: 41 5d popq %r13
// 25: 41 5e popq %r14
// 27: 41 5f popq %r15
// 29: 5d popq %rbp
// 2a: c3 retq
const unsigned char code_body[43] = {
0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56,
0x41, 0x55, 0x41, 0x54, 0x53, 0x50, 0x49, 0x89,
0xfc, 0x49, 0x89, 0xf5, 0x49, 0x89, 0xd6, 0xe8,
0x0f, 0x00, 0x00, 0x00, 0x48, 0x83, 0xc4, 0x08,
0x5b, 0x41, 0x5c, 0x41, 0x5d, 0x41, 0x5e, 0x41,
0x5f, 0x5d, 0xc3,
};
memcpy(code, code_body, sizeof(code_body));
}

void
emit_0(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 55 pushq %rbp
// 1: 48 89 e5 movq %rsp, %rbp
// 4: 5d popq %rbp
const unsigned char code_body[5] = {
0x55, 0x48, 0x89, 0xe5, 0x5d,
};
memcpy(code, code_body, sizeof(code_body));
}

void
emit_1(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 55 pushq %rbp
// 1: 48 89 e5 movq %rsp, %rbp
// 4: 48 8b 05 00 00 00 00 movq (%rip), %rax ## 0xb <__JIT_ENTRY+0xb>
// 0000000000000007: X86_64_RELOC_GOT_LOAD _sausage@GOTPCREL
// b: 80 38 00 cmpb $0x0, (%rax)
// e: 74 08 je 0x18 <__JIT_ENTRY+0x18>
// 10: ff 15 00 00 00 00 callq *(%rip) ## 0x16 <__JIT_ENTRY+0x16>
// 0000000000000012: X86_64_RELOC_GOT _order_eggs_sausage_and_bacon@GOTPCREL
// 16: eb 06 jmp 0x1e <__JIT_ENTRY+0x1e>
// 18: ff 15 00 00 00 00 callq *(%rip) ## 0x1e <__JIT_ENTRY+0x1e>
// 000000000000001a: X86_64_RELOC_GOT _order_eggs_and_bacon@GOTPCREL
// 1e: 48 8b 05 00 00 00 00 movq (%rip), %rax ## 0x25 <__JIT_ENTRY+0x25>
// 0000000000000021: X86_64_RELOC_GOT_LOAD _spammed@GOTPCREL
// 25: c6 00 00 movb $0x0, (%rax)
// 28: 5d popq %rbp
const unsigned char code_body[41] = {
0x55, 0x48, 0x89, 0xe5, 0x48, 0x8b, 0x05, 0x00,
0x00, 0x00, 0x00, 0x80, 0x38, 0x00, 0x74, 0x08,
0xff, 0x15, 0x00, 0x00, 0x00, 0x00, 0xeb, 0x06,
0xff, 0x15, 0x00, 0x00, 0x00, 0x00, 0x48, 0x8b,
0x05, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00,
0x5d,
};
// 0: &spammed+0x0
// 8: &order_eggs_and_bacon+0x0
// 10: &order_eggs_sausage_and_bacon+0x0
// 18: &sausage+0x0
patch_64(data + 0x0, (uintptr_t)&spammed);
patch_64(data + 0x8, (uintptr_t)&order_eggs_and_bacon);
patch_64(data + 0x10, (uintptr_t)&order_eggs_sausage_and_bacon);
patch_64(data + 0x18, (uintptr_t)&sausage);
memcpy(code, code_body, sizeof(code_body));
patch_x86_64_32rx(code + 0x7, (uintptr_t)data + 0x14);
patch_x86_64_32rx(code + 0x12, (uintptr_t)data + 0xc);
patch_x86_64_32rx(code + 0x1a, (uintptr_t)data + 0x4);
patch_x86_64_32rx(code + 0x21, (uintptr_t)data + -0x4);
}

void
emit_2(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state)
{
// 0: 55 pushq %rbp
// 1: 48 89 e5 movq %rsp, %rbp
// 4: 48 8b 05 00 00 00 00 movq (%rip), %rax ## 0xb <__JIT_ENTRY+0xb>
// 0000000000000007: X86_64_RELOC_GOT_LOAD _spam@GOTPCREL
// b: 80 38 01 cmpb $0x1, (%rax)
// e: 75 06 jne 0x16 <__JIT_ENTRY+0x16>
// 10: 5d popq %rbp
// 11: e9 00 00 00 00 jmp 0x16 <__JIT_ENTRY+0x16>
// 0000000000000012: X86_64_RELOC_BRANCH __JIT_ERROR_TARGET
// 16: 5d popq %rbp
const unsigned char code_body[23] = {
0x55, 0x48, 0x89, 0xe5, 0x48, 0x8b, 0x05, 0x00,
0x00, 0x00, 0x00, 0x80, 0x38, 0x01, 0x75, 0x06,
0x5d, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x5d,
};
// 0: &spam+0x0
patch_64(data + 0x0, (uintptr_t)&spam);
memcpy(code, code_body, sizeof(code_body));
patch_x86_64_32rx(code + 0x7, (uintptr_t)data + -0x4);
patch_32r(code + 0x12, state->instruction_starts[instruction->error_target] + -0x4);
}

static_assert(SYMBOL_MASK_WORDS >= 1, "SYMBOL_MASK_WORDS too small");

typedef struct {
void (*emit)(
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
const _PyUOpInstruction *instruction, jit_state *state);
size_t code_size;
size_t data_size;
symbol_mask trampoline_mask;
} StencilGroup;

static const StencilGroup shim = {emit_shim, 43, 0, {0}};

static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {
[0] = {emit_0, 5, 0, {0}},
[1] = {emit_1, 41, 32, {0}},
[2] = {emit_2, 23, 8, {0}},
};

static const void * const symbols_map[1] = {
0
};
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy