diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 00da84e47c56..17fa4f11ef34 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -56,7 +56,6 @@ import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; -import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -87,7 +86,6 @@ import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; -import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.jfr.JfrGCWhen; import com.oracle.svm.core.jfr.JfrTicks; @@ -797,14 +795,7 @@ private static void walkStack(IsolateThread thread, JavaStackWalk walk, ObjectRe CodeInfo codeInfo = CodeInfoAccess.unsafeConvert(frame.getIPCodeInfo()); if (JavaFrames.isInterpreterLeaveStub(frame)) { - /* - * Variable frame size is packed into the first stack slot used for argument - * passing (re-use of deopt slot). - */ - long varStackSize = DeoptimizationSlotPacking.decodeVariableFrameSizeFromDeoptSlot(sp.readLong(0)); - Pointer actualSP = sp.add(Word.unsigned(varStackSize)); - - InterpreterSupport.walkInterpreterLeaveStubFrame(visitor, actualSP, sp); + /* nothing to scan */ } else { NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); long referenceMapIndex = frame.getReferenceMapIndex(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java index b5b0503db823..3576cbf05378 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java @@ -34,13 +34,10 @@ import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.CodeInfoTable; -import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; import com.oracle.svm.core.deopt.DeoptimizedFrame; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.RestrictHeapAccess; -import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.stack.JavaFrames; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackFrameVisitor; import com.oracle.svm.core.thread.VMThreads; @@ -96,14 +93,7 @@ public void initialize(IsolateThread thread) { @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while verifying the stack.") public boolean visitRegularFrame(Pointer currentSP, CodePointer currentIP, CodeInfo codeInfo) { verifyFrameReferencesVisitor.initialize(thread, currentSP, currentIP); - if (JavaFrames.isInterpreterLeaveStub(currentIP)) { - // This should probably be handled in JavaStackWalker GR-71827 - long varStackSize = DeoptimizationSlotPacking.decodeVariableFrameSizeFromDeoptSlot(currentSP.readLong(0)); - Pointer actualSP = currentSP.add(Word.unsigned(varStackSize)); - InterpreterSupport.walkInterpreterLeaveStubFrame(verifyFrameReferencesVisitor, actualSP, currentSP); - } else { - CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, verifyFrameReferencesVisitor); - } + CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, verifyFrameReferencesVisitor); result &= verifyFrameReferencesVisitor.result; return true; } diff --git a/substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/AArch64InterpreterStubs.java b/substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/AArch64InterpreterStubs.java index c95b7fd2dc48..fdf599fe8035 100644 --- a/substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/AArch64InterpreterStubs.java +++ b/substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/AArch64InterpreterStubs.java @@ -24,33 +24,11 @@ */ package com.oracle.svm.core.graal.aarch64; -import com.oracle.svm.core.c.struct.OffsetOf; -import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; -import com.oracle.svm.core.graal.code.InterpreterAccessStubData; -import com.oracle.svm.core.meta.SharedMethod; -import com.oracle.svm.core.util.VMError; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.asm.Label; -import jdk.graal.compiler.asm.aarch64.AArch64Address; -import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; -import jdk.graal.compiler.core.common.NumUtil; -import jdk.graal.compiler.lir.asm.CompilationResultBuilder; -import jdk.graal.compiler.word.Word; -import jdk.vm.ci.aarch64.AArch64; -import jdk.vm.ci.code.Register; -import jdk.vm.ci.code.RegisterValue; -import jdk.vm.ci.code.StackSlot; -import jdk.vm.ci.meta.AllocatableValue; -import org.graalvm.nativeimage.c.struct.RawField; -import org.graalvm.nativeimage.c.struct.RawStructure; -import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; - +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static jdk.graal.compiler.asm.aarch64.AArch64Address.createImmediateAddress; import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_POST_INDEXED; import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED; import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED; -import static jdk.graal.compiler.asm.aarch64.AArch64Address.createImmediateAddress; import static jdk.vm.ci.aarch64.AArch64.r0; import static jdk.vm.ci.aarch64.AArch64.r1; import static jdk.vm.ci.aarch64.AArch64.r2; @@ -69,6 +47,30 @@ import static jdk.vm.ci.aarch64.AArch64.v6; import static jdk.vm.ci.aarch64.AArch64.v7; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.struct.OffsetOf; +import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; +import com.oracle.svm.core.graal.code.InterpreterAccessStubData; +import com.oracle.svm.core.graal.code.PreparedArgumentType; +import com.oracle.svm.core.meta.SharedMethod; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.word.Word; +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.Register; + public class AArch64InterpreterStubs { public static final Register TRAMPOLINE_ARGUMENT = AArch64.r12; @@ -150,16 +152,12 @@ public void enter(CompilationResultBuilder crb) { super.enter(crb); AArch64MacroAssembler masm = (AArch64MacroAssembler) crb.asm; - /* sp points to four reserved stack slots for this stub */ + /* sp points to two reserved stack slots for this stub */ /* Pointer to InterpreterData struct */ masm.str(64, r1, createImmediateAddress(64, IMMEDIATE_UNSIGNED_SCALED, sp, 0)); /* Variable stack size */ masm.str(64, r2, createImmediateAddress(64, IMMEDIATE_UNSIGNED_SCALED, sp, 8)); - /* gcReferenceMap next */ - masm.str(64, r3, createImmediateAddress(64, IMMEDIATE_UNSIGNED_SCALED, sp, 16)); - - /* 4th slot is for stack alignment to 0x10 */ masm.sub(64, sp, sp, r2 /* variable stack size */); } @@ -260,11 +258,8 @@ public static int additionalFrameSizeEnterStub() { public static int additionalFrameSizeLeaveStub() { int wordSize = 8; - /* - * reserve four slots for: base address of outgoing stack args, variable stack size, - * gcReferenceMap, padding - */ - return 4 * wordSize; + // reserve two slots for: base address of outgoing stack args and variable stack size. + return 2 * wordSize; } @RawStructure @@ -509,7 +504,8 @@ public void setSp(Pointer data, int stackSize, Pointer stackBuffer) { } @Override - public long getGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + public long getGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos) { InterpreterDataAArch64 p = (InterpreterDataAArch64) data; return switch (pos) { case 0 -> p.getAbiGpArg0(); @@ -521,18 +517,19 @@ public long getGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { case 6 -> p.getAbiGpArg6(); case 7 -> p.getAbiGpArg7(); default -> { - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer spVal = Word.pointer(p.getAbiSpReg()); - yield spVal.readLong(stackSlot.getOffset(0)); + yield spVal.readLong(cArgType.getStackOffset()); } }; } @Override - public long setGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val) { + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + public void setGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val, boolean incoming) { InterpreterDataAArch64 p = (InterpreterDataAArch64) data; if (pos >= 0 && pos <= 7) { - VMError.guarantee(ccArg instanceof RegisterValue); + VMError.guarantee(cArgType.isRegister()); switch (pos) { case 0 -> p.setAbiGpArg0(val); case 1 -> p.setAbiGpArg1(val); @@ -543,24 +540,21 @@ public long setGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long case 6 -> p.setAbiGpArg6(val); case 7 -> p.setAbiGpArg7(val); } - /* no GC mask required */ - return 0; + return; } - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer spVal = Word.pointer(p.getAbiSpReg()); - int offset = stackSlot.getOffset(0); + int offset = cArgType.getStackOffset(); VMError.guarantee(spVal.isNonNull()); - VMError.guarantee(offset < p.getStackSize()); + VMError.guarantee(incoming || offset < p.getStackSize()); spVal.writeLong(offset, val); - - VMError.guarantee((pos - 8) < Long.SIZE, "more than 64 stack args are not supported"); - return 1L << (pos - 8); } @Override - public long getFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos) { InterpreterDataAArch64 p = (InterpreterDataAArch64) data; return switch (pos) { case 0 -> p.getAbiFpArg0(); @@ -572,15 +566,16 @@ public long getFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { case 6 -> p.getAbiFpArg6(); case 7 -> p.getAbiFpArg7(); default -> { - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer spVal = Word.pointer(p.getAbiSpReg()); - yield spVal.readLong(stackSlot.getOffset(0)); + yield spVal.readLong(cArgType.getStackOffset()); } }; } @Override - public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void setFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val) { InterpreterDataAArch64 p = (InterpreterDataAArch64) data; switch (pos) { case 0 -> p.setAbiFpArg0(val); @@ -592,10 +587,10 @@ public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long case 6 -> p.setAbiFpArg6(val); case 7 -> p.setAbiFpArg7(val); default -> { - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer spVal = Word.pointer(p.getAbiSpReg()); - int offset = stackSlot.getOffset(0); + int offset = cArgType.getStackOffset(); VMError.guarantee(spVal.isNonNull()); VMError.guarantee(offset < p.getStackSize()); @@ -606,21 +601,25 @@ public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long } @Override + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) public long getGpReturn(Pointer data) { return ((InterpreterDataAArch64) data).getAbiGpRet(); } @Override + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) public void setGpReturn(Pointer data, long gpReturn) { ((InterpreterDataAArch64) data).setAbiGpRet(gpReturn); } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getFpReturn(Pointer data) { return ((InterpreterDataAArch64) data).getAbiFpRet(); } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void setFpReturn(Pointer data, long fpReturn) { ((InterpreterDataAArch64) data).setAbiFpRet(fpReturn); } diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64InterpreterStubs.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64InterpreterStubs.java index f2d7d68d7a83..47c0b41a09ba 100644 --- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64InterpreterStubs.java +++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64InterpreterStubs.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.graal.amd64; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static jdk.vm.ci.amd64.AMD64.rax; import static jdk.vm.ci.amd64.AMD64.rsp; import static jdk.vm.ci.amd64.AMD64.xmm0; @@ -39,10 +40,12 @@ import org.graalvm.word.PointerBase; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.struct.OffsetOf; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.deopt.DeoptimizationSlotPacking; import com.oracle.svm.core.graal.code.InterpreterAccessStubData; +import com.oracle.svm.core.graal.code.PreparedArgumentType; import com.oracle.svm.core.graal.meta.SubstrateRegisterConfig; import com.oracle.svm.core.meta.SharedMethod; import com.oracle.svm.core.util.VMError; @@ -58,9 +61,6 @@ import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.code.CallingConvention; import jdk.vm.ci.code.Register; -import jdk.vm.ci.code.RegisterValue; -import jdk.vm.ci.code.StackSlot; -import jdk.vm.ci.meta.AllocatableValue; public class AMD64InterpreterStubs { @@ -168,10 +168,6 @@ public void enter(CompilationResultBuilder crb) { masm.movq(new AMD64Address(rsp, 0), gps.get(1)); /* arg2: Variable stack size */ masm.movq(new AMD64Address(rsp, 8), gps.get(2)); - /* arg3: gcReferenceMap next */ - masm.movq(new AMD64Address(rsp, 16), gps.get(3)); - - /* 4th slot is for stack alignment to 0x10 */ masm.subq(rsp, gps.get(2) /* variable stack size */); } @@ -280,11 +276,8 @@ public static int additionalFrameSizeEnterStub() { public static int additionalFrameSizeLeaveStub() { int wordSize = 8; - /* - * reserve four slots for: base address of outgoing stack args, variable stack size, - * gcReferenceMap, padding - */ - return 4 * wordSize; + // reserve two slots for: base address of outgoing stack args and variable stack size. + return 2 * wordSize; } @RawStructure @@ -488,6 +481,13 @@ public static int offsetAbiFpRet() { public static class AMD64InterpreterAccessStubData implements InterpreterAccessStubData { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static int spAdjustOnCall(int offset) { + // offset is relative caller sp, undo side-effect of call instruction + int spAdjustmentOnCall = ConfigurationValues.getTarget().wordSize; + return offset + spAdjustmentOnCall; + } + @Override public void setSp(Pointer data, int stackSize, Pointer stackBuffer) { VMError.guarantee(stackBuffer.isNonNull()); @@ -509,7 +509,8 @@ public void setSp(Pointer data, int stackSize, Pointer stackBuffer) { } @Override - public long getGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + public long getGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos) { InterpreterDataAMD64 p = (InterpreterDataAMD64) data; return switch (pos) { case 0 -> p.getAbiGp0(); @@ -519,20 +520,20 @@ public long getGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { case 4 -> p.getAbiGp4(); case 5 -> p.getAbiGp5(); default -> { - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer sp = Word.pointer(p.getAbiSpReg()); - int spAdjustmentOnCall = ConfigurationValues.getTarget().wordSize; - int offset = stackSlot.getOffset(0) + spAdjustmentOnCall; - yield sp.readLong(offset); + + yield sp.readLong(spAdjustOnCall(cArgType.getStackOffset())); } }; } @Override - public long setGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val) { + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + public void setGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val, boolean incoming) { InterpreterDataAMD64 p = (InterpreterDataAMD64) data; if (pos >= 0 && pos <= 5) { - VMError.guarantee(ccArg instanceof RegisterValue); + VMError.guarantee(cArgType.isRegister()); switch (pos) { case 0 -> p.setAbiGp0(val); case 1 -> p.setAbiGp1(val); @@ -541,32 +542,34 @@ public long setGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long case 4 -> p.setAbiGp4(val); case 5 -> p.setAbiGp5(val); } - /* no GC mask required */ - return 0; + return; } - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer sp = Word.pointer(p.getAbiSpReg()); - int offset = stackSlot.getOffset(0); + int offset = cArgType.getStackOffset(); + if (incoming) { + offset = spAdjustOnCall(offset); + } + VMError.guarantee(sp.isNonNull()); - VMError.guarantee(offset < p.getStackSize()); + VMError.guarantee(incoming || offset < p.getStackSize()); sp.writeLong(offset, val); - - VMError.guarantee((pos - 6) < Long.SIZE, "more than 64 stack args are not supported"); - return 1L << (pos - 6); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static int upperFpEnd() { /* only 4 floating point regs on Windows, 8 otherwise */ return Platform.includedIn(InternalPlatform.WINDOWS_BASE.class) ? 3 : 7; } @Override - public long getFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos) { InterpreterDataAMD64 p = (InterpreterDataAMD64) data; if (pos >= 0 && pos <= upperFpEnd()) { - VMError.guarantee(ccArg instanceof RegisterValue); + VMError.guarantee(cArgType.isRegister()); switch (pos) { case 0: return p.getAbiFpArg0(); @@ -586,20 +589,18 @@ public long getFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos) { return p.getAbiFpArg7(); } } - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer sp = Word.pointer(p.getAbiSpReg()); - int spAdjustmentOnCall = ConfigurationValues.getTarget().wordSize; - int offset = stackSlot.getOffset(0) + spAdjustmentOnCall; - - return sp.readLong(offset); + return sp.readLong(spAdjustOnCall(cArgType.getStackOffset())); } @Override - public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void setFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val) { InterpreterDataAMD64 p = (InterpreterDataAMD64) data; if (pos >= 0 && pos <= upperFpEnd()) { - VMError.guarantee(ccArg instanceof RegisterValue); + VMError.guarantee(cArgType.isRegister()); switch (pos) { case 0 -> p.setAbiFpArg0(val); case 1 -> p.setAbiFpArg1(val); @@ -611,10 +612,10 @@ public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long case 7 -> p.setAbiFpArg7(val); } } else { - StackSlot stackSlot = (StackSlot) ccArg; + VMError.guarantee(cArgType.isStackSlot()); Pointer sp = Word.pointer(p.getAbiSpReg()); - int offset = stackSlot.getOffset(0); + int offset = cArgType.getStackOffset(); VMError.guarantee(sp.isNonNull()); VMError.guarantee(offset < p.getStackSize()); @@ -624,21 +625,25 @@ public void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long } @Override + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) public long getGpReturn(Pointer data) { return ((InterpreterDataAMD64) data).getAbiGpRet(); } @Override + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) public void setGpReturn(Pointer data, long gpReturn) { ((InterpreterDataAMD64) data).setAbiGpRet(gpReturn); } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public long getFpReturn(Pointer data) { return ((InterpreterDataAMD64) data).getAbiFpRet(); } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void setFpReturn(Pointer data, long fpReturn) { ((InterpreterDataAMD64) data).setAbiFpRet(fpReturn); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/InterpreterAccessStubData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/InterpreterAccessStubData.java index 8d0b2d5a8e9c..d1f43028bff8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/InterpreterAccessStubData.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/InterpreterAccessStubData.java @@ -24,27 +24,50 @@ */ package com.oracle.svm.core.graal.code; -import jdk.vm.ci.meta.AllocatableValue; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.word.Pointer; +import com.oracle.svm.core.Uninterruptible; + /* Helper class to set ABI specific data */ public interface InterpreterAccessStubData { + String REASON_RAW_POINTER = "raw pointer to object"; + void setSp(Pointer data, int stackSize, Pointer stackBuffer); - long getGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos); + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + long getGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos); + + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + default void setGpArgumentAtOutgoing(PreparedArgumentType cArgType, Pointer data, int pos, long val) { + setGpArgumentAt(cArgType, data, pos, val, false); + } + + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + default void setGpArgumentAtIncoming(PreparedArgumentType cArgType, Pointer data, int pos, long val) { + setGpArgumentAt(cArgType, data, pos, val, true); + } - long setGpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val); + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) + void setGpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val, boolean incoming); - long getFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + long getFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos); - void setFpArgumentAt(AllocatableValue ccArg, Pointer data, int pos, long val); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void setFpArgumentAt(PreparedArgumentType cArgType, Pointer data, int pos, long val); + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) long getGpReturn(Pointer data); + @Uninterruptible(reason = REASON_RAW_POINTER, callerMustBe = true) void setGpReturn(Pointer data, long gpReturn); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) long getFpReturn(Pointer data); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setFpReturn(Pointer data, long fpReturn); int allocateStubDataSize(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedArgumentType.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedArgumentType.java new file mode 100644 index 000000000000..70c8937018e5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedArgumentType.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.graal.code; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.Uninterruptible; + +import jdk.vm.ci.meta.JavaKind; + +public final class PreparedArgumentType { + private final JavaKind kind; + private final int value; + private final boolean isRegister; + + public PreparedArgumentType(JavaKind kind, int value, boolean isRegister) { + this.kind = kind; + this.value = value; + this.isRegister = isRegister; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isRegister() { + return isRegister; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isStackSlot() { + return !isRegister; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getRegister() { + assert isRegister; + return value; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getStackOffset() { + assert !isRegister; + return value; + } + + @Platforms(Platform.HOSTED_ONLY.class) + public int getOffsetForSerialization() { + return value; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public JavaKind getKind() { + return kind; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedSignature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedSignature.java new file mode 100644 index 000000000000..60ed9ecf79f7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/code/PreparedSignature.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.graal.code; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import com.oracle.svm.core.Uninterruptible; + +import jdk.vm.ci.meta.JavaKind; + +public final class PreparedSignature { + private final JavaKind returnKind; + private final PreparedArgumentType[] preparedArgumentTypes; + private final int stackSize; + + public PreparedSignature(JavaKind returnKind, PreparedArgumentType[] preparedArgumentTypes, int stackSize) { + this.returnKind = returnKind; + this.preparedArgumentTypes = preparedArgumentTypes; + this.stackSize = stackSize; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public PreparedArgumentType[] getPreparedArgumentTypes() { + return preparedArgumentTypes; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getCount() { + return preparedArgumentTypes.length; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public JavaKind getReturnKind() { + return returnKind; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getStackSize() { + return stackSize; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ThreadLocalHandles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ThreadLocalHandles.java index 0b5487093923..cf4d34317f9e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ThreadLocalHandles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/handles/ThreadLocalHandles.java @@ -24,14 +24,18 @@ */ package com.oracle.svm.core.handles; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.util.Arrays; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.word.SignedWord; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; /** * Implementation of local object handles, which are bound to a specific thread and can be created @@ -69,6 +73,7 @@ private static int toIndex(T handle) { return (int) handle.rawValue(); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public int getHandleCount() { return top - MIN_VALUE; } @@ -83,6 +88,18 @@ public int pushFrame(int capacity) { return frameCount; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int pushFrameUninterruptible(int capacity) { + VMError.guarantee(frameCount < frameStack.length, "cannot grow"); + frameStack[frameCount] = top; + frameCount++; + + int minLength = top + capacity; + VMError.guarantee(minLength < objects.length, "cannot ensure capacity"); + + return frameCount; + } + @NeverInline("Decrease code size of JNI entry points by not inlining allocations") private void growFrameStack() { frameStack = Arrays.copyOf(frameStack, frameStack.length * 2); @@ -112,6 +129,18 @@ public T tryCreateNonNull(Object obj) { return Word.signed(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T tryCreateNonNullUninterruptible(Object obj) { + assert obj != null; + if (top >= objects.length) { + return nullHandle(); + } + int index = top; + objects[index] = obj; + top++; + return Word.signed(index); + } + @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public U getObject(T handle) { @@ -125,19 +154,24 @@ public boolean delete(T handle) { return previous != null; } - public void popFrame() { - popFramesIncluding(frameCount); + public int popFrame() { + return popFramesIncluding(frameCount); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void popFramesIncluding(int frame) { + public int popFramesIncluding(int frame) { assert frame > 0 && frame <= frameCount; + + int currentFrameCount = frameCount; int previousTop = top; + frameCount = frame - 1; top = frameStack[frameCount]; for (int i = top; i < previousTop; i++) { objects[i] = null; // so objects can be garbage collected } + + return currentFrameCount; } public void ensureCapacity(int capacity) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index f813771946b5..8c0e323bd35e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -903,6 +903,7 @@ public SharedType getMetaType() { return companion.metaType; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public ResolvedJavaType getInterpreterType() { return companion.interpreterType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java index 7e4b98553511..3f55b28e266f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java @@ -40,6 +40,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.code.FrameInfoQueryResult; import com.oracle.svm.core.code.FrameSourceInfo; +import com.oracle.svm.core.graal.code.PreparedSignature; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.UnknownPrimitiveField; @@ -89,6 +90,8 @@ public static InterpreterSupport singleton() { @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Used for crash log") public abstract void logInterpreterFrame(Log log, FrameInfoQueryResult frameInfo, Pointer sp); + public abstract PreparedSignature prepareSignature(ResolvedJavaMethod method); + @Platforms(Platform.HOSTED_ONLY.class) public static void setLeaveStubPointer(CFunctionPointer leaveStubPointer, int length) { assert singleton().leaveStubPointer == null : "multiple leave stub methods registered"; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java index 3ccadedb0ef5..db9f73da9d53 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java @@ -157,6 +157,34 @@ private static Enum valueOf(Class> enumType, String name) { public native int ordinal(); } +@TargetClass(java.lang.Byte.class) +final class Target_java_lang_Byte { + @AnnotateOriginal + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + native long longValue(); +} + +@TargetClass(java.lang.Short.class) +final class Target_java_lang_Short { + @AnnotateOriginal + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + native long longValue(); +} + +@TargetClass(java.lang.Character.class) +final class Target_java_lang_Character { + @AnnotateOriginal + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + native char charValue(); +} + +@TargetClass(java.lang.Integer.class) +final class Target_java_lang_Integer { + @AnnotateOriginal + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + native long longValue(); +} + @TargetClass(java.lang.String.class) final class Target_java_lang_String { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java index 848222b8884c..9c3405477f90 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaFrames.java @@ -62,14 +62,6 @@ public static boolean isInterpreterLeaveStub(JavaFrame frame) { return InterpreterSupport.isInInterpreterLeaveStub(frame.getIP()); } - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static boolean isInterpreterLeaveStub(CodePointer ip) { - if (!InterpreterSupport.isEnabled()) { - return false; - } - return InterpreterSupport.isInInterpreterLeaveStub(ip); - } - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static UnsignedWord getTotalFrameSize(JavaFrame frame) { long size = CodeInfoQueryResult.getTotalFrameSize(frame.getEncodedFrameSize()); diff --git a/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/CremaResolvedJavaMethodImpl.java b/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/CremaResolvedJavaMethodImpl.java index cbf327deb9ea..26fbd18942db 100644 --- a/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/CremaResolvedJavaMethodImpl.java +++ b/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/CremaResolvedJavaMethodImpl.java @@ -28,6 +28,7 @@ import com.oracle.svm.core.hub.crema.CremaResolvedJavaMethod; import com.oracle.svm.core.hub.registry.SVMSymbols; +import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.reflect.CremaConstructorAccessor; import com.oracle.svm.core.reflect.CremaMethodAccessor; import com.oracle.svm.core.util.VMError; @@ -54,7 +55,9 @@ private CremaResolvedJavaMethodImpl(InterpreterResolvedObjectType declaringClass } public static InterpreterResolvedJavaMethod create(InterpreterResolvedObjectType declaringClass, ParserMethod m, int vtableIndex) { - return new CremaResolvedJavaMethodImpl(declaringClass, m, vtableIndex); + InterpreterResolvedJavaMethod interpreterMethod = new CremaResolvedJavaMethodImpl(declaringClass, m, vtableIndex); + interpreterMethod.setPreparedSignature(InterpreterSupport.singleton().prepareSignature(interpreterMethod)); + return interpreterMethod; } @Override diff --git a/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/InterpreterResolvedJavaMethod.java b/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/InterpreterResolvedJavaMethod.java index a01ef1ff6b89..89276d8c8733 100644 --- a/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/InterpreterResolvedJavaMethod.java +++ b/substratevm/src/com.oracle.svm.interpreter.metadata/src/com/oracle/svm/interpreter/metadata/InterpreterResolvedJavaMethod.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.interpreter.metadata; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static com.oracle.svm.espresso.classfile.Constants.ACC_CALLER_SENSITIVE; import static com.oracle.svm.espresso.classfile.Constants.ACC_FINAL; import static com.oracle.svm.espresso.classfile.Constants.ACC_NATIVE; @@ -47,8 +48,12 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.FunctionPointerHolder; import com.oracle.svm.core.SubstrateMetadata; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.graal.code.PreparedSignature; +import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.hub.RuntimeClassLoading; import com.oracle.svm.core.hub.crema.CremaSupport; import com.oracle.svm.core.hub.registry.SymbolsSupport; @@ -141,6 +146,9 @@ public class InterpreterResolvedJavaMethod extends InterpreterAnnotated implemen private static final AtomicReferenceFieldUpdater RISTRETTO_METHOD_UPDATER = AtomicReferenceFieldUpdater .newUpdater(InterpreterResolvedJavaMethod.class, ResolvedJavaMethod.class, "ristrettoMethod"); + @UnknownObjectField(availability = BuildPhaseProvider.ReadyForCompilation.class) // + private PreparedSignature preparedSignature; + public static class InlinedBy { public InterpreterResolvedJavaMethod holder; public Set inliners; @@ -180,7 +188,7 @@ public InlinedBy(InterpreterResolvedJavaMethod holder, Set name, int maxLocals, int maxStackSize, int flags, - InterpreterResolvedObjectType declaringClass, InterpreterUnresolvedSignature signature, Symbol signatureSymbol, + InterpreterResolvedObjectType declaringClass, InterpreterUnresolvedSignature signature, PreparedSignature preparedSignature, Symbol signatureSymbol, byte[] code, ExceptionHandler[] exceptionHandlers, LineNumberTable lineNumberTable, LocalVariableTable localVariableTable, ReferenceConstant nativeEntryPoint, int vtableIndex, int gotOffset, int enterStubOffset, int methodId) { this.name = MetadataUtil.requireNonNull(name); @@ -190,6 +198,7 @@ private InterpreterResolvedJavaMethod(ResolvedJavaMethod originalMethod, Symbol< this.declaringClass = MetadataUtil.requireNonNull(declaringClass); this.signature = MetadataUtil.requireNonNull(signature); this.signatureSymbol = MetadataUtil.requireNonNull(signatureSymbol); + this.preparedSignature = preparedSignature; this.interpretedCode = code; this.exceptionHandlers = exceptionHandlers; this.lineNumberTable = lineNumberTable; @@ -207,7 +216,7 @@ private InterpreterResolvedJavaMethod(ResolvedJavaMethod originalMethod, Symbol< // Used at run-time for deserialization private InterpreterResolvedJavaMethod(Symbol name, int maxLocals, int maxStackSize, int flags, - InterpreterResolvedObjectType declaringClass, InterpreterUnresolvedSignature signature, Symbol signatureSymbol, + InterpreterResolvedObjectType declaringClass, InterpreterUnresolvedSignature signature, PreparedSignature preparedSignature, Symbol signatureSymbol, byte[] code, ExceptionHandler[] exceptionHandlers, LineNumberTable lineNumberTable, LocalVariableTable localVariableTable, ReferenceConstant nativeEntryPoint, int vtableIndex, int gotOffset, int enterStubOffset, int methodId) { this.name = MetadataUtil.requireNonNull(name); @@ -216,6 +225,7 @@ private InterpreterResolvedJavaMethod(Symbol name, int maxLocals, int maxS this.flags = flags; this.declaringClass = MetadataUtil.requireNonNull(declaringClass); this.signature = MetadataUtil.requireNonNull(signature); + this.preparedSignature = preparedSignature; this.signatureSymbol = MetadataUtil.requireNonNull(signatureSymbol); this.interpretedCode = code; this.exceptionHandlers = exceptionHandlers; @@ -289,12 +299,12 @@ protected InterpreterResolvedJavaMethod(InterpreterResolvedObjectType declaringC @VisibleForSerialization public static InterpreterResolvedJavaMethod createForDeserialization(String name, int maxLocals, int maxStackSize, int flags, InterpreterResolvedObjectType declaringClass, - InterpreterUnresolvedSignature signature, + InterpreterUnresolvedSignature signature, PreparedSignature preparedSignature, byte[] code, ExceptionHandler[] exceptionHandlers, LineNumberTable lineNumberTable, LocalVariableTable localVariableTable, ReferenceConstant nativeEntryPoint, int vtableIndex, int gotOffset, int enterStubOffset, int methodId) { Symbol nameSymbol = SymbolsSupport.getNames().getOrCreate(name); Symbol signatureSymbol = toSymbol(signature); - return new InterpreterResolvedJavaMethod(nameSymbol, maxLocals, maxStackSize, flags, declaringClass, signature, signatureSymbol, code, + return new InterpreterResolvedJavaMethod(nameSymbol, maxLocals, maxStackSize, flags, declaringClass, signature, preparedSignature, signatureSymbol, code, exceptionHandlers, lineNumberTable, localVariableTable, nativeEntryPoint, vtableIndex, gotOffset, enterStubOffset, methodId); } @@ -308,7 +318,8 @@ public static InterpreterResolvedJavaMethod createAtBuildTime(ResolvedJavaMethod Symbol nameSymbol = SymbolsSupport.getNames().getOrCreate(name); Symbol signatureSymbol = toSymbol(signature); int flags = createFlags(modifiers, declaringClass, signatureSymbol, isSubstitutedNative, originalMethod); - return new InterpreterResolvedJavaMethod(originalMethod, nameSymbol, maxLocals, maxStackSize, flags, declaringClass, signature, signatureSymbol, code, + PreparedSignature preparedSignature = null; + return new InterpreterResolvedJavaMethod(originalMethod, nameSymbol, maxLocals, maxStackSize, flags, declaringClass, signature, preparedSignature, signatureSymbol, code, exceptionHandlers, lineNumberTable, localVariableTable, nativeEntryPoint, vtableIndex, gotOffset, enterStubOffset, methodId); } @@ -500,11 +511,13 @@ public final InterpreterUnresolvedSignature getSignature() { } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final int getMaxLocals() { return maxLocals; } @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final int getMaxStackSize() { return maxStackSize; } @@ -752,6 +765,15 @@ public final PartialMethod [method...] .collect(Collectors.groupingBy(InterpreterResolvedJavaMethod::getDeclaringClass))); - private final Lazy methodESTOffsetTable = Lazy.of(() -> createMethodTable(getMethods())); + private final InterpreterResolvedJavaMethod[] methodESTOffsetTable; private final Lazy> methodInverseTable = Lazy.of(() -> createInverseTable(getMethods())); private final Lazy> typeInverseTable = Lazy.of(() -> createInverseTable(getTypes())); @@ -103,6 +105,8 @@ public InterpreterUniverseImpl( this.types = List.copyOf(types); this.fields = List.copyOf(fields); this.methods = List.copyOf(methods); + + this.methodESTOffsetTable = createMethodTable(this.methods); } private static void consumeMagic(DataInput in) throws IOException { @@ -360,8 +364,9 @@ private static InterpreterResolvedJavaMethod[] createMethodTable(Collection> newReferenceConstantSerializ context.writerFor(InterpreterResolvedJavaMethod[].class).write(context, out, value.vtable); }); + static final ValueSerializer PREPARED_ARGUMENT_TYPE = createSerializer( + (context, in) -> { + JavaKind kind = context.readReference(in); + int value = LEB128.readUnsignedInt(in); + boolean isRegister = in.readBoolean(); + return new PreparedArgumentType(kind, value, isRegister); + }, + (context, out, value) -> { + context.writeReference(out, value.getKind()); + LEB128.writeUnsignedInt(out, value.getOffsetForSerialization()); + out.writeBoolean(value.isRegister()); + }); + + static final ValueSerializer PREPARED_SIGNATURE = createSerializer( + (context, in) -> { + JavaKind returnKind = context.readReference(in); + PreparedArgumentType[] preparedArgumentTypes = context.readerFor(PreparedArgumentType[].class).read(context, in); + int stackSize = in.readInt(); + return new PreparedSignature(returnKind, preparedArgumentTypes, stackSize); + }, + (context, out, value) -> { + context.writeReference(out, value.getReturnKind()); + context.writerFor(PreparedArgumentType[].class).write(context, out, value.getPreparedArgumentTypes()); + out.writeInt(value.getStackSize()); + }); + static final ValueSerializer RESOLVED_METHOD = createSerializer( (context, in) -> { String name = context.readReference(in); @@ -643,6 +671,7 @@ public static ValueSerializer> newReferenceConstantSerializ int flags = LEB128.readUnsignedInt(in); InterpreterResolvedObjectType declaringClass = context.readReference(in); InterpreterUnresolvedSignature signature = context.readReference(in); + PreparedSignature preparedSignature = context.readReference(in); byte[] code = context.readReference(in); ExceptionHandler[] exceptionHandlers = context.readReference(in); LineNumberTable lineNumberTable = context.readReference(in); @@ -654,8 +683,8 @@ public static ValueSerializer> newReferenceConstantSerializ int enterStubOffset = LEB128.readUnsignedInt(in); int methodId = LEB128.readUnsignedInt(in); - return InterpreterResolvedJavaMethod.createForDeserialization(name, maxLocals, maxStackSize, flags, declaringClass, signature, code, exceptionHandlers, lineNumberTable, - localVariableTable, nativeEntryPoint, vtableIndex, gotOffset, enterStubOffset, methodId); + return InterpreterResolvedJavaMethod.createForDeserialization(name, maxLocals, maxStackSize, flags, declaringClass, signature, preparedSignature, code, exceptionHandlers, + lineNumberTable, localVariableTable, nativeEntryPoint, vtableIndex, gotOffset, enterStubOffset, methodId); }, (context, out, value) -> { String name = value.getName(); @@ -664,6 +693,8 @@ public static ValueSerializer> newReferenceConstantSerializ int flags = value.getFlags(); InterpreterResolvedObjectType declaringClass = value.getDeclaringClass(); InterpreterUnresolvedSignature signature = value.getSignature(); + PreparedSignature preparedSignature = value.getPreparedSignature(); + assert preparedSignature != null : "no prepared signature for " + value; byte[] code = value.getInterpretedCode(); ExceptionHandler[] exceptionHandlers = value.getExceptionHandlers(); LineNumberTable lineNumberTable = value.getLineNumberTable(); @@ -685,6 +716,7 @@ public static ValueSerializer> newReferenceConstantSerializ LEB128.writeUnsignedInt(out, flags); context.writeReference(out, declaringClass); context.writeReference(out, signature); + context.writeReference(out, preparedSignature); context.writeReference(out, code); context.writeReference(out, exceptionHandlers); context.writeReference(out, lineNumberTable); @@ -744,6 +776,9 @@ public static ValueSerializer> newReferenceConstantSerializ InterpreterResolvedPrimitiveType.class, InterpreterResolvedObjectType.class, InterpreterResolvedObjectType.VTableHolder.class, + PreparedSignature.class, + PreparedArgumentType.class, + PreparedArgumentType[].class, InterpreterResolvedJavaField.class, FunctionPointerHolder.class, InterpreterResolvedJavaMethod.class, @@ -779,6 +814,9 @@ public static SerializationContext.Builder newBuilderForInterpreterMetadata() { .registerSerializer(InterpreterResolvedPrimitiveType.class, PRIMITIVE_TYPE) .registerSerializer(InterpreterResolvedObjectType.class, OBJECT_TYPE) .registerSerializer(InterpreterResolvedObjectType.VTableHolder.class, VTABLE_HOLDER) + .registerSerializer(PreparedSignature.class, PREPARED_SIGNATURE) + .registerSerializer(PreparedArgumentType[].class, ofReferenceArray(PreparedArgumentType[]::new)) + .registerSerializer(PreparedArgumentType.class, PREPARED_ARGUMENT_TYPE) .registerSerializer(InterpreterResolvedJavaField.class, RESOLVED_FIELD) .registerSerializer(FunctionPointerHolder.class, asReferenceConstant()) .registerSerializer(InterpreterResolvedJavaMethod.class, RESOLVED_METHOD) diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/CremaFeature.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/CremaFeature.java index 78d17a4cee5e..688eb75ce876 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/CremaFeature.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/CremaFeature.java @@ -139,6 +139,8 @@ public void beforeCompilation(BeforeCompilationAccess access) { for (HostedType hType : hUniverse.getTypes()) { iUniverse.mirrorSVMVTable(hType, objectType -> accessImpl.getHeapScanner().rescanField(objectType, vtableHolderField, reason)); } + + InterpreterFeature.prepareSignatures(); } @Override diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerFeature.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerFeature.java index 74f64609303c..bc3cc55a5b74 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerFeature.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerFeature.java @@ -508,6 +508,8 @@ public void beforeCompilation(BeforeCompilationAccess access) { AnalysisMethod arraycopy = (AnalysisMethod) JVMCIReflectionUtil.getUniqueDeclaredMethod(aMetaAccess, systemClass.getWrapped(), "arraycopy", Object.class, int.class, Object.class, int.class, int.class); SubstrateCompilationDirectives.singleton().registerForcedCompilation(arraycopy); + + InterpreterFeature.prepareSignatures(); } @Override diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerSupport.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerSupport.java index b2418e8ad55f..f4a4ab41f0a9 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerSupport.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/DebuggerSupport.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.interpreter; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import static com.oracle.svm.interpreter.InterpreterUtil.traceInterpreter; import java.io.IOException; @@ -46,6 +47,7 @@ import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.FunctionPointerHolder; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.CGlobalData; import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.heap.UnknownObjectField; @@ -164,6 +166,11 @@ public InterpreterUniverse getUniverse() { return universe.get(); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public InterpreterUniverse getUniverseOrNull() { + return universe.getOrNull(); + } + @Platforms(Platform.HOSTED_ONLY.class) void trimForcedReferencesInImageHeap() { Set unique = Collections.newSetFromMap(new IdentityHashMap<>()); diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/EspressoFrame.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/EspressoFrame.java index 06f6b621dd38..176666cff8ab 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/EspressoFrame.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/EspressoFrame.java @@ -25,6 +25,9 @@ package com.oracle.svm.interpreter; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.util.VMError; import com.oracle.svm.interpreter.metadata.InterpreterUnresolvedSignature; @@ -202,6 +205,7 @@ public static void clearLocal(InterpreterFrame frame, int localSlot) { clear(frame, localSlot); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void setLocalObject(InterpreterFrame frame, int localSlot, Object value) { assert !(value instanceof ReturnAddress); frame.setObjectStatic(localSlot, value); @@ -211,18 +215,22 @@ static void setLocalObjectOrReturnAddress(InterpreterFrame frame, int localSlot, frame.setObjectStatic(localSlot, value); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void setLocalInt(InterpreterFrame frame, int localSlot, int value) { frame.setIntStatic(localSlot, value); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void setLocalFloat(InterpreterFrame frame, int localSlot, float value) { frame.setFloatStatic(localSlot, value); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void setLocalLong(InterpreterFrame frame, int localSlot, long value) { frame.setLongStatic(localSlot, value); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void setLocalDouble(InterpreterFrame frame, int localSlot, double value) { frame.setDoubleStatic(localSlot, value); } diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterDirectivesSupportImpl.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterDirectivesSupportImpl.java index f4c5f8ca806f..41cb05f7b44c 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterDirectivesSupportImpl.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterDirectivesSupportImpl.java @@ -177,7 +177,7 @@ public Object callIntoInterpreter(Object method, Object... args) { public Object callIntoUnknown(Object method, Object... args) { InterpreterResolvedJavaMethod interpreterMethod = getInterpreterMethod(method); MethodPointer calleeFtnPtr = interpreterMethod.getNativeEntryPoint(); - return InterpreterStubSection.leaveInterpreter(calleeFtnPtr, interpreterMethod, interpreterMethod.getDeclaringClass(), args); + return InterpreterStubSection.leaveInterpreter(calleeFtnPtr, interpreterMethod, args); } private static String getDescriptor(Class returnType, Class... parameterTypes) { diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFeature.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFeature.java index 8726891edb16..a9629bae5919 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFeature.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFeature.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.NoSuchElementException; +import com.oracle.svm.core.graal.code.StubCallingConvention; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -54,6 +55,7 @@ import com.oracle.svm.core.graal.code.InterpreterAccessStubData; import com.oracle.svm.core.interpreter.InterpreterSupport; import com.oracle.svm.core.meta.MethodPointer; +import com.oracle.svm.core.thread.ThreadListenerSupport; import com.oracle.svm.core.util.VMError; import com.oracle.svm.espresso.shared.meta.SignaturePolymorphicIntrinsic; import com.oracle.svm.hosted.FeatureImpl; @@ -108,6 +110,13 @@ static boolean executableByInterpreter(AnalysisMethod m) { return false; } } + if (StubCallingConvention.Utils.hasStubCallingConvention(m)) { + /* + * enterstub can only deal with the internal Java calling convention of SVM. If ever + * needed, the enterstub can be adapted. + */ + return false; + } return true; } @@ -176,6 +185,11 @@ public void lower(LoweringTool tool) { } } + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ThreadListenerSupport.get().register(new ThreadListenerThreadLocalHandlesAllocator()); + } + @Override public void duringSetup(DuringSetupAccess access) { if (Platform.includedIn(Platform.AARCH64.class)) { @@ -232,7 +246,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { SubstrateCompilationDirectives.singleton().registerFrameInformationRequired(interpreterRoot); SubstrateCompilationDirectives.singleton().registerFrameInformationRequired(intrinsicRoot); - Method leaveMethod = ReflectionUtil.lookupMethod(InterpreterStubSection.class, "leaveInterpreterStub", CFunctionPointer.class, Pointer.class, long.class, long.class); + Method leaveMethod = ReflectionUtil.lookupMethod(InterpreterStubSection.class, "leaveInterpreterStub", CFunctionPointer.class, Pointer.class, long.class); leaveStub = metaAccess.lookupJavaMethod(leaveMethod); accessImpl.registerAsRoot(leaveStub, true, "low level entry point"); } @@ -249,8 +263,28 @@ public void beforeCompilation(BeforeCompilationAccess access) { InterpreterMethodPointerHolder.setMethodNotCompiledHandler(new MethodPointer(methodNotCompiledHandler)); } + /** + * Must be called by all features that depend on InterpreterFeature in + * {@link #beforeCompilation(BeforeCompilationAccess)}. + */ + public static void prepareSignatures() { + InterpreterSupport interpreterSingleton = InterpreterSupport.singleton(); + for (InterpreterResolvedJavaMethod interpreterMethod : BuildTimeInterpreterUniverse.singleton().getMethods()) { + interpreterMethod.setPreparedSignature(interpreterSingleton.prepareSignature(interpreterMethod)); + } + } + + private static boolean verifyPreparedSignatures() { + for (InterpreterResolvedJavaMethod interpreterMethod : BuildTimeInterpreterUniverse.singleton().getMethods()) { + assert interpreterMethod.getPreparedSignature() != null; + } + return true; + } + @Override public void afterCompilation(AfterCompilationAccess access) { + assert verifyPreparedSignatures(); + FeatureImpl.AfterCompilationAccessImpl accessImpl = (FeatureImpl.AfterCompilationAccessImpl) access; HostedMethod hLeaveStub = accessImpl.getUniverse().lookup(leaveStub); diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFrame.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFrame.java index 37af8235dd29..ff236a91fbdf 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFrame.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterFrame.java @@ -24,8 +24,11 @@ */ package com.oracle.svm.interpreter; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.util.Arrays; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.monitor.MonitorSupport; import com.oracle.svm.core.util.VMError; @@ -75,22 +78,27 @@ long getLongStatic(int slot) { return Double.longBitsToDouble(primitives[slot]); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setObjectStatic(int slot, Object value) { references[slot] = value; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setIntStatic(int slot, int value) { primitives[slot] = value; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setFloatStatic(int slot, float value) { primitives[slot] = Float.floatToRawIntBits(value); } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setLongStatic(int slot, long value) { primitives[slot] = value; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) void setDoubleStatic(int slot, double value) { primitives[slot] = Double.doubleToRawLongBits(value); } diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterOptions.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterOptions.java index 6fb516d7f80f..6748755ffc3d 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterOptions.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterOptions.java @@ -63,4 +63,7 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o public static boolean interpreterEnabled() { return DebuggerWithInterpreter.getValue() || RuntimeClassLoading.isSupported(); } + + @Option(help = "Enables backdoors in interpreter for testing", type = OptionType.Debug)// + public static final HostedOptionKey InterpreterBackdoor = new HostedOptionKey<>(false); } diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterStubSection.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterStubSection.java index 0b66d29f3be5..1451648a747c 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterStubSection.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterStubSection.java @@ -25,16 +25,20 @@ package com.oracle.svm.interpreter; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.ObjectHandle; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import com.oracle.objectfile.BasicProgbitsSectionImpl; import com.oracle.objectfile.ObjectFile; @@ -43,17 +47,23 @@ import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateTargetDescription; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.CGlobalData; import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.graal.code.InterpreterAccessStubData; -import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; -import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; +import com.oracle.svm.core.graal.code.PreparedArgumentType; +import com.oracle.svm.core.graal.code.PreparedSignature; +import com.oracle.svm.core.handles.ThreadLocalHandles; +import com.oracle.svm.core.heap.GCCause; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.jdk.InternalVMMethod; import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.image.AbstractImage; import com.oracle.svm.hosted.image.NativeImage; @@ -61,19 +71,15 @@ import com.oracle.svm.hosted.meta.HostedMethod; import com.oracle.svm.interpreter.metadata.InterpreterResolvedJavaMethod; import com.oracle.svm.interpreter.metadata.InterpreterResolvedObjectType; -import com.oracle.svm.interpreter.metadata.InterpreterUnresolvedSignature; +import com.oracle.svm.interpreter.metadata.InterpreterUniverse; import com.oracle.svm.interpreter.ristretto.meta.RistrettoMethod; import jdk.graal.compiler.core.common.LIRKind; import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.word.Word; -import jdk.vm.ci.code.CallingConvention; import jdk.vm.ci.code.RegisterConfig; import jdk.vm.ci.code.ValueKindFactory; -import jdk.vm.ci.meta.AllocatableValue; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.JavaType; -import jdk.vm.ci.meta.ResolvedJavaType; @InternalVMMethod public abstract class InterpreterStubSection { @@ -82,6 +88,8 @@ public abstract class InterpreterStubSection { private static final CGlobalData BASE = CGlobalDataFactory.forSymbol(nameForVTableIndex(0)); + private static final String REASON_REFERENCES_ON_STACK = "stack frame might contain object references that are not known to the GC"; + /* '-3' to reduce padding due to alignment in .svm_interp section */ static final int MAX_VTABLE_STUBS = 2 * 1024 - 3; @@ -179,12 +187,34 @@ public void markEnterStubPatch(HostedMethod enterStub) { @Platforms(Platform.HOSTED_ONLY.class) protected abstract void markEnterStubPatch(ObjectFile.ProgbitsSectionImpl pltBuffer, HostedMethod enterStub); + interface ThreadLocalInterpreterHandle extends ObjectHandle, PointerBase { + } + + @SuppressWarnings("rawtypes") // + public static final FastThreadLocalObject TL_HANDLES = FastThreadLocalFactory.createObject(ThreadLocalHandles.class, "Interpreter handles for enter stub"); + + /* + * Maximum number of parameters that can be passed according to 4.3.3 in the JVM spec. Could be + * optimized, see GR-71907. + */ + public static final int MAX_ARGUMENT_HANDLES = 255; + + @SuppressWarnings("unchecked") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static ThreadLocalHandles tlsHandles() { + return TL_HANDLES.get(); + } + @Deoptimizer.DeoptStub(stubType = Deoptimizer.StubType.InterpreterEnterStub) @NeverInline("needs ABI boundary") + @Uninterruptible(reason = REASON_REFERENCES_ON_STACK) public static Pointer enterMethodInterpreterStub(int interpreterMethodESTOffset, Pointer enterData) { - DebuggerSupport interpreterSupport = ImageSingletons.lookup(DebuggerSupport.class); + DebuggerSupport debuggerSupport = ImageSingletons.lookup(DebuggerSupport.class); - InterpreterResolvedJavaMethod interpreterMethod = (InterpreterResolvedJavaMethod) interpreterSupport.getUniverse().getMethodForESTOffset(interpreterMethodESTOffset); + InterpreterUniverse interpreterUniverse = debuggerSupport.getUniverseOrNull(); + VMError.guarantee(interpreterUniverse != null); + + InterpreterResolvedJavaMethod interpreterMethod = (InterpreterResolvedJavaMethod) interpreterUniverse.getMethodForESTOffset(interpreterMethodESTOffset); VMError.guarantee(interpreterMethod != null); return enterHelper(interpreterMethod, enterData); @@ -192,6 +222,7 @@ public static Pointer enterMethodInterpreterStub(int interpreterMethodESTOffset, @Deoptimizer.DeoptStub(stubType = Deoptimizer.StubType.InterpreterEnterStub) @NeverInline("needs ABI boundary") + @Uninterruptible(reason = REASON_REFERENCES_ON_STACK) public static Pointer enterVTableInterpreterStub(int vTableIndex, Pointer enterData) { InterpreterAccessStubData accessHelper = ImageSingletons.lookup(InterpreterAccessStubData.class); @@ -205,81 +236,74 @@ public static Pointer enterVTableInterpreterStub(int vTableIndex, Pointer enterD return enterHelper(interpreterMethod, enterData); } - @AlwaysInline("helper") + /** + * The "enter stub" pretends to be like a compiled method, with the advantage that the caller + * does not need to know where the call ends up. Therefore, it has to look like a compiled + * entrypoint. + * + * The low-level stubs calling this helper spill native ABI arguments to the stack frame, see + * {@link com.oracle.svm.core.graal.amd64.AMD64InterpreterStubs.InterpreterEnterStubContext} and + * {@link com.oracle.svm.core.graal.aarch64.AArch64InterpreterStubs.InterpreterEnterStubContext}. + * Its layout is defined in + * {@link com.oracle.svm.core.graal.amd64.AMD64InterpreterStubs.InterpreterDataAMD64} and + * {@link com.oracle.svm.core.graal.aarch64.AArch64InterpreterStubs.InterpreterDataAArch64}. + * + * The ABI arguments can contain references which the GC is not aware of, so until they are + * moved to a known location, a safepoint must be avoided. ThreadLocalHandles are used for that. + * + * @param interpreterMethod method that should run in the interpreter. + * @param enterData pointer to struct that contains ABI arguments. + * @return pointer to enterData, used by the low-level caller stub. + */ + @AlwaysInline("Performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static Pointer enterHelper(InterpreterResolvedJavaMethod interpreterMethod, Pointer enterData) { InterpreterAccessStubData accessHelper = ImageSingletons.lookup(InterpreterAccessStubData.class); - InterpreterStubSection stubSection = ImageSingletons.lookup(InterpreterStubSection.class); - - InterpreterUnresolvedSignature signature = interpreterMethod.getSignature(); - VMError.guarantee(signature != null); - - int count = signature.getParameterCount(false); - ResolvedJavaType accessingClass = interpreterMethod.getDeclaringClass(); + PreparedSignature compiledSignature = interpreterMethod.getPreparedSignature(); + VMError.guarantee(compiledSignature != null); - JavaType thisType = interpreterMethod.hasReceiver() ? accessingClass : null; - JavaType returnType = signature.getReturnType(accessingClass); + ThreadLocalHandles handles = tlsHandles(); + VMError.guarantee(handles.getHandleCount() == 0); + int handleFrameId = handles.pushFrameUninterruptible(MAX_ARGUMENT_HANDLES - 1); - var kind = SubstrateCallingConventionKind.Java.toType(true); - CallingConvention callingConvention = stubSection.registerConfig.getCallingConvention(kind, returnType, signature.toParameterTypes(thisType), stubSection.valueKindFactory); - - Object[] args = new Object[count + (interpreterMethod.hasReceiver() ? 1 : 0)]; - - int interpSlot = 0; int gpIdx = 0; - int fpIdx = 0; - if (interpreterMethod.hasReceiver()) { - Object receiver = ((Pointer) Word.pointer(accessHelper.getGpArgumentAt(callingConvention.getArgument(gpIdx), enterData, gpIdx))).toObject(); - args[interpSlot] = receiver; - gpIdx++; - interpSlot++; - } + int handleCount = 0; + for (int i = 0; i < compiledSignature.getCount(); i++) { + PreparedArgumentType cArgType = compiledSignature.getPreparedArgumentTypes()[i]; + if (cArgType.getKind() == JavaKind.Object) { + /* + * The GC is not aware of references in enterData, therefore they are replaced with + * object handles before allowing safepoints again. + */ + long rawAddr = accessHelper.getGpArgumentAt(cArgType, enterData, gpIdx); + Object obj = ((Pointer) Word.pointer(rawAddr)).toObject(); + if (obj == null) { + accessHelper.setGpArgumentAtIncoming(cArgType, enterData, gpIdx, 0L); + } else { + ThreadLocalInterpreterHandle threadLocalInterpreterHandle = handles.tryCreateNonNullUninterruptible(obj); + VMError.guarantee(threadLocalInterpreterHandle.isNonNull()); + accessHelper.setGpArgumentAtIncoming(cArgType, enterData, gpIdx, threadLocalInterpreterHandle.rawValue()); + handleCount++; + } + } - for (int i = 0; i < count; i++) { - JavaKind argKind = signature.getParameterKind(i); - long arg; - AllocatableValue ccArg = callingConvention.getArgument(gpIdx + fpIdx); - switch (argKind) { + switch (cArgType.getKind()) { case Float: case Double: - arg = accessHelper.getFpArgumentAt(ccArg, enterData, fpIdx); - fpIdx++; break; + case Void: + case Illegal: + throw VMError.shouldNotReachHereAtRuntime(); default: - arg = accessHelper.getGpArgumentAt(ccArg, enterData, gpIdx); gpIdx++; break; } - - switch (argKind) { - // @formatter:off - case Boolean: args[interpSlot] = (arg & 0xff) != 0; break; - case Byte: args[interpSlot] = (byte) arg; break; - case Short: args[interpSlot] = (short) arg; break; - case Char: args[interpSlot] = (char) arg; break; - case Int: args[interpSlot] = (int) arg; break; - case Long: args[interpSlot] = arg; break; - case Float: args[interpSlot] = Float.intBitsToFloat((int) arg); break; - case Double: args[interpSlot] = Double.longBitsToDouble(arg); break; - case Object: args[interpSlot] = ((Pointer) Word.pointer(arg)).toObject(); break; - // @formatter:on - default: - throw VMError.shouldNotReachHereAtRuntime(); - } - interpSlot++; } - Object retVal; - RistrettoMethod rMethod = (com.oracle.svm.interpreter.ristretto.meta.RistrettoMethod) interpreterMethod.getRistrettoMethod(); - if (rMethod != null && rMethod.installedCode != null && rMethod.installedCode.isValid()) { - /* A JIT compiled version is available, execute this one instead */ - CFunctionPointer entryPoint = Word.pointer(rMethod.installedCode.getEntryPoint()); - retVal = leaveInterpreter(entryPoint, interpreterMethod, accessingClass, args); - } else { - retVal = Interpreter.execute(interpreterMethod, args); - } + Object retVal = enterInterpreterStub0(interpreterMethod, compiledSignature, enterData, handleCount, handleFrameId); - switch (returnType.getJavaKind()) { + switch (compiledSignature.getReturnKind()) { case Boolean: InterpreterUtil.assertion(retVal instanceof Boolean, "invalid return type"); accessHelper.setGpReturn(enterData, ((Boolean) retVal) ? 1 : 0); @@ -323,40 +347,119 @@ private static Pointer enterHelper(InterpreterResolvedJavaMethod interpreterMeth return enterData; } - /* - * reserve four slots for: 1. base address of outgoing stack args, 2. variable stack size, 3. - * gcReferenceMap, 4. stack padding to match alignment - */ + @Uninterruptible(reason = "allow allocation now ", calleeMustBe = false) + private static Object enterInterpreterStub0(InterpreterResolvedJavaMethod interpreterMethod, PreparedSignature compiledSignature, Pointer enterData, int handleCount, int handleFrameId) { + TestingBackdoor.stressEnterStub(); + return enterInterpreterStubCore(interpreterMethod, compiledSignature, enterData, handleCount, handleFrameId); + } + + private static Object enterInterpreterStubCore(InterpreterResolvedJavaMethod interpreterMethod, PreparedSignature compiledSignature, Pointer enterData, int handleCount, int handleFrameId) { + InterpreterAccessStubData accessHelper = ImageSingletons.lookup(InterpreterAccessStubData.class); + PreparedArgumentType[] cArgsType = compiledSignature.getPreparedArgumentTypes(); + ThreadLocalHandles handles = tlsHandles(); + int count = cArgsType.length; + + int interpSlot = 0; + int gpIdx = 0; + int fpIdx = 0; + + Object[] args = new Object[count + (interpreterMethod.hasReceiver() ? 1 : 0)]; + + for (int i = 0; i < count; i++) { + long arg = 0; + PreparedArgumentType cArgType = cArgsType[gpIdx + fpIdx]; + JavaKind argKind = cArgType.getKind(); + switch (argKind) { + case Float: + case Double: + arg = accessHelper.getFpArgumentAt(cArgType, enterData, fpIdx); + fpIdx++; + break; + case Object: + args[interpSlot] = popReferenceFromEnterData(accessHelper, cArgType, enterData, gpIdx); + gpIdx++; + break; + case Void: + case Illegal: + throw VMError.shouldNotReachHereAtRuntime(); + default: + arg = popPrimitiveFromEnterData(accessHelper, cArgType, enterData, gpIdx); + gpIdx++; + break; + } + + switch (argKind) { + // @formatter:off + case Boolean: args[interpSlot] = (arg & 0xff) != 0; break; + case Byte: args[interpSlot] = (byte) arg; break; + case Short: args[interpSlot] = (short) arg; break; + case Char: args[interpSlot] = (char) arg; break; + case Int: args[interpSlot] = (int) arg; break; + case Long: args[interpSlot] = arg; break; + case Float: args[interpSlot] = Float.intBitsToFloat((int) arg); break; + case Double: args[interpSlot] = Double.longBitsToDouble(arg); break; + case Object: /* already handled */ break; + // @formatter:on + default: + throw VMError.shouldNotReachHereAtRuntime(); + } + interpSlot++; + } + + VMError.guarantee(handles.getHandleCount() == handleCount); + VMError.guarantee(handleFrameId == handles.popFrame()); + + RistrettoMethod rMethod = (com.oracle.svm.interpreter.ristretto.meta.RistrettoMethod) interpreterMethod.getRistrettoMethod(); + if (rMethod != null && rMethod.installedCode != null && rMethod.installedCode.isValid()) { + /* + * A JIT compiled version is available, execute this one instead. This could be more + * optimized, see GR-71160. + */ + + CFunctionPointer entryPoint = Word.pointer(rMethod.installedCode.getEntryPoint()); + return leaveInterpreter(entryPoint, interpreterMethod, args); + } else { + return Interpreter.execute(interpreterMethod, args); + } + } + + @Uninterruptible(reason = "Raw object pointer.") + private static Object popReferenceFromEnterData(InterpreterAccessStubData accessHelper, PreparedArgumentType cArgType, Pointer enterData, int gpIdx) { + long arg = accessHelper.getGpArgumentAt(cArgType, enterData, gpIdx); + + /* reference in `enterData` has been replaced with a handle */ + ThreadLocalInterpreterHandle handle = Word.pointer(arg); + + if (handle.rawValue() == 0L) { + return null; + } else { + return tlsHandles().getObject(handle); + } + } + + @Uninterruptible(reason = "Wrapping of getter, no raw object pointer involved in this case.") + private static long popPrimitiveFromEnterData(InterpreterAccessStubData accessHelper, PreparedArgumentType cArgType, Pointer enterData, int gpIdx) { + return accessHelper.getGpArgumentAt(cArgType, enterData, gpIdx); + } + + /* reserve two slots for: 1. base address of outgoing stack args, and 2. variable stack size. */ @Deoptimizer.DeoptStub(stubType = Deoptimizer.StubType.InterpreterLeaveStub) @NeverInline("needs ABI boundary") + @Uninterruptible(reason = REASON_REFERENCES_ON_STACK) @SuppressWarnings("unused") - public static Pointer leaveInterpreterStub(CFunctionPointer entryPoint, Pointer leaveData, long stackSize, long gcReferenceMap) { + public static Pointer leaveInterpreterStub(CFunctionPointer entryPoint, Pointer leaveData, long stackSize) { return (Pointer) entryPoint; } - public static Object leaveInterpreter(CFunctionPointer compiledEntryPoint, InterpreterResolvedJavaMethod seedMethod, ResolvedJavaType accessingClass, Object[] args) { - InterpreterUnresolvedSignature targetSignature = seedMethod.getSignature(); + public static Object leaveInterpreter(CFunctionPointer compiledEntryPoint, InterpreterResolvedJavaMethod seedMethod, Object[] args) { + PreparedSignature compiledSignature = seedMethod.getPreparedSignature(); + VMError.guarantee(compiledSignature != null); InterpreterStubSection stubSection = ImageSingletons.lookup(InterpreterStubSection.class); - JavaType thisType = seedMethod.hasReceiver() ? seedMethod.getDeclaringClass() : null; - SubstrateCallingConventionType kind = SubstrateCallingConventionKind.Java.toType(true); - JavaType returnType = targetSignature.getReturnType(accessingClass); - - CallingConvention callingConvention = stubSection.registerConfig.getCallingConvention(kind, returnType, targetSignature.toParameterTypes(thisType), stubSection.valueKindFactory); - InterpreterAccessStubData accessHelper = ImageSingletons.lookup(InterpreterAccessStubData.class); - Pointer leaveData = StackValue.get(1, accessHelper.allocateStubDataSize()); - - /* GR-54726: Reference map is currently limited to 64 arguments */ - long gcReferenceMap = 0; - int gpIdx = 0; - int fpIdx = 0; - if (seedMethod.hasReceiver()) { - gcReferenceMap |= accessHelper.setGpArgumentAt(callingConvention.getArgument(gpIdx), leaveData, gpIdx, Word.objectToTrackedPointer(args[0]).rawValue()); - gpIdx++; - } + Pointer leaveData = StackValue.get(accessHelper.allocateStubDataSize()); - int stackSize = NumUtil.roundUp(callingConvention.getStackSize(), stubSection.target.stackAlignment); + int stackSize = NumUtil.roundUp(compiledSignature.getStackSize(), stubSection.target.stackAlignment); Pointer stackBuffer = Word.nullPointer(); if (stackSize > 0) { @@ -364,48 +467,66 @@ public static Object leaveInterpreter(CFunctionPointer compiledEntryPoint, Inter accessHelper.setSp(leaveData, stackSize, stackBuffer); } - int argCount = targetSignature.getParameterCount(false); - for (int i = 0; i < argCount; i++) { - Object arg = args[i + (seedMethod.hasReceiver() ? 1 : 0)]; + try { + // GR-55022: Stack overflow check should be done here + return leaveInterpreter0(compiledEntryPoint, args, compiledSignature, accessHelper, leaveData, stackSize); + } catch (Throwable e) { + // native code threw exception, wrap it + throw SemanticJavaException.raise(e); + } finally { + if (stackSize > 0) { + VMError.guarantee(stackBuffer.isNonNull()); + NativeMemory.free(stackBuffer); + } + } + } - AllocatableValue ccArg = callingConvention.getArgument(gpIdx + fpIdx); - JavaType type = targetSignature.getParameterType(i, accessingClass); - switch (type.getJavaKind()) { + @Uninterruptible(reason = "References are put on the stack which the GC is unaware of.") + private static Object leaveInterpreter0(CFunctionPointer compiledEntryPoint, Object[] args, PreparedSignature compiledSignature, InterpreterAccessStubData accessHelper, Pointer leaveData, + int stackSize) { + int gpIdx = 0; + int fpIdx = 0; + + int argCount = compiledSignature.getCount(); + for (int i = 0; i < argCount; i++) { + Object arg = args[i]; + PreparedArgumentType cArgType = compiledSignature.getPreparedArgumentTypes()[gpIdx + fpIdx]; + switch (cArgType.getKind()) { case Boolean: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (boolean) arg ? 1 : 0); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (boolean) arg ? 1 : 0); gpIdx++; break; case Byte: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (byte) arg); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (byte) arg); gpIdx++; break; case Short: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (short) arg); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (short) arg); gpIdx++; break; case Char: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (char) arg); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (char) arg); gpIdx++; break; case Int: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (int) arg); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (int) arg); gpIdx++; break; case Long: - accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, (long) arg); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, (long) arg); gpIdx++; break; case Object: - gcReferenceMap |= accessHelper.setGpArgumentAt(ccArg, leaveData, gpIdx, Word.objectToTrackedPointer(arg).rawValue()); + accessHelper.setGpArgumentAtOutgoing(cArgType, leaveData, gpIdx, Word.objectToTrackedPointer(arg).rawValue()); gpIdx++; break; case Float: - accessHelper.setFpArgumentAt(ccArg, leaveData, fpIdx, Float.floatToRawIntBits((float) arg)); + accessHelper.setFpArgumentAt(cArgType, leaveData, fpIdx, Float.floatToRawIntBits((float) arg)); fpIdx++; break; case Double: - accessHelper.setFpArgumentAt(ccArg, leaveData, fpIdx, Double.doubleToRawLongBits((double) arg)); + accessHelper.setFpArgumentAt(cArgType, leaveData, fpIdx, Double.doubleToRawLongBits((double) arg)); fpIdx++; break; @@ -415,22 +536,10 @@ public static Object leaveInterpreter(CFunctionPointer compiledEntryPoint, Inter } VMError.guarantee(compiledEntryPoint.isNonNull()); - - try { - // GR-55022: Stack overflow check should be done here - leaveInterpreterStub(compiledEntryPoint, leaveData, stackSize, gcReferenceMap); - } catch (Throwable e) { - // native code threw exception, wrap it - throw SemanticJavaException.raise(e); - } finally { - if (stackSize > 0) { - VMError.guarantee(stackBuffer.isNonNull()); - NativeMemory.free(stackBuffer); - } - } + leaveInterpreterStub(compiledEntryPoint, leaveData, stackSize); // @formatter:off - return switch (returnType.getJavaKind()) { + return switch (compiledSignature.getReturnKind()) { case Boolean -> (accessHelper.getGpReturn(leaveData) & 0xff) != 0; case Byte -> (byte) accessHelper.getGpReturn(leaveData); case Short -> (short) accessHelper.getGpReturn(leaveData); @@ -445,4 +554,24 @@ public static Object leaveInterpreter(CFunctionPointer compiledEntryPoint, Inter }; // @formatter:on } + + public static class TestingBackdoor { + private static boolean stressEnterStub = false; + + public static void enableStressEnterStub() { + VMError.guarantee(InterpreterOptions.InterpreterBackdoor.getValue()); + stressEnterStub = true; + } + + public static void disableStressEnterStub() { + VMError.guarantee(InterpreterOptions.InterpreterBackdoor.getValue()); + stressEnterStub = false; + } + + public static void stressEnterStub() { + if (InterpreterOptions.InterpreterBackdoor.getValue() && stressEnterStub) { + Heap.getHeap().getGC().collectCompletely(GCCause.UnitTest); + } + } + } } diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterSupportImpl.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterSupportImpl.java index f4e78bcf43f1..65c740d7560d 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterSupportImpl.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterSupportImpl.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.code.FrameSourceInfo.LINENUMBER_NATIVE; +import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -35,6 +36,9 @@ import com.oracle.svm.core.code.FrameInfoQueryResult; import com.oracle.svm.core.code.FrameSourceInfo; +import com.oracle.svm.core.graal.code.PreparedArgumentType; +import com.oracle.svm.core.graal.code.PreparedSignature; +import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.interpreter.InterpreterFrameSourceInfo; @@ -45,10 +49,17 @@ import com.oracle.svm.espresso.classfile.descriptors.Name; import com.oracle.svm.espresso.classfile.descriptors.Symbol; import com.oracle.svm.interpreter.metadata.InterpreterResolvedJavaMethod; +import com.oracle.svm.interpreter.metadata.InterpreterUnresolvedSignature; import jdk.graal.compiler.word.Word; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.StackSlot; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.JavaType; import jdk.vm.ci.meta.LineNumberTable; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; public final class InterpreterSupportImpl extends InterpreterSupport { private static final int MAX_SYMBOL_LOG_LENGTH = 255; @@ -67,6 +78,41 @@ public final class InterpreterSupportImpl extends InterpreterSupport { this.intrinsicFrameSlot = intrinsicFrameSlot; } + @Override + public PreparedSignature prepareSignature(ResolvedJavaMethod method) { + InterpreterResolvedJavaMethod interpreterMethod = (InterpreterResolvedJavaMethod) method; + + InterpreterUnresolvedSignature signature = interpreterMethod.getSignature(); + boolean hasReceiver = interpreterMethod.hasReceiver(); + InterpreterStubSection stubSection = ImageSingletons.lookup(InterpreterStubSection.class); + int count = signature.getParameterCount(false); + PreparedArgumentType[] argumentTypes = new PreparedArgumentType[count + (hasReceiver ? 1 : 0)]; + + // The calling convention is always used with a caller perspective, i.e. sp is unmodified. + SubstrateCallingConventionType callingConventionType = SubstrateCallingConventionKind.Java.toType(true); + ResolvedJavaType accessingClass = interpreterMethod.getDeclaringClass(); + JavaType thisType = interpreterMethod.hasReceiver() ? accessingClass : null; + JavaType returnType = signature.getReturnType(accessingClass); + CallingConvention callingConvention = stubSection.registerConfig.getCallingConvention(callingConventionType, returnType, signature.toParameterTypes(thisType), stubSection.valueKindFactory); + + if (hasReceiver) { + argumentTypes[0] = new PreparedArgumentType(JavaKind.Object, 0, true); + } + for (int i = 0; i < count; i++) { + int index = i + (hasReceiver ? 1 : 0); + AllocatableValue allocatableValue = callingConvention.getArgument(index); + JavaKind argKind = signature.getParameterKind(i); + int value = 0; + if (allocatableValue instanceof StackSlot stackSlot) { + // Both, in the enter- and leavestub we want the "outgoing semantics". + value = stackSlot.getOffset(0); + } + boolean isRegister = !(allocatableValue instanceof StackSlot); + argumentTypes[index] = new PreparedArgumentType(argKind, value, isRegister); + } + return new PreparedSignature(signature.getReturnKind(), argumentTypes, callingConvention.getStackSize()); + } + @Override public boolean isInterpreterRoot(FrameInfoQueryResult frameInfo) { return isInterpreterBytecodeRoot(frameInfo) || isInterpreterIntrinsicRoot(frameInfo); diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterToVM.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterToVM.java index 5a01d53a2aab..c7813341185a 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterToVM.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterToVM.java @@ -915,7 +915,7 @@ public static Object dispatchInvocation(InterpreterResolvedJavaMethod seedMethod } // wrapping of exceptions is done in leaveInterpreter - retObj = InterpreterStubSection.leaveInterpreter(calleeFtnPtr, targetMethod, targetMethod.getDeclaringClass(), calleeArgs); + retObj = InterpreterStubSection.leaveInterpreter(calleeFtnPtr, targetMethod, calleeArgs); } else { try { retObj = Interpreter.execute(targetMethod, calleeArgs, forceStayInInterpreter); diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterUtil.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterUtil.java index c8dfbc966545..4e4d0dc7c9de 100644 --- a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterUtil.java +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/InterpreterUtil.java @@ -24,9 +24,12 @@ */ package com.oracle.svm.interpreter; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.util.VMError; import com.oracle.svm.interpreter.metadata.MetadataUtil; @@ -49,6 +52,7 @@ public static void guarantee(boolean condition, String simpleFormat, Object arg1 } } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public static void assertion(boolean condition, String message) { if (assertionsEnabled && !condition) { VMError.guarantee(condition, message); diff --git a/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/ThreadListenerThreadLocalHandlesAllocator.java b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/ThreadListenerThreadLocalHandlesAllocator.java new file mode 100644 index 000000000000..63319b1f012d --- /dev/null +++ b/substratevm/src/com.oracle.svm.interpreter/src/com/oracle/svm/interpreter/ThreadListenerThreadLocalHandlesAllocator.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.interpreter; + +import com.oracle.svm.core.handles.ThreadLocalHandles; +import com.oracle.svm.core.thread.ThreadListener; + +class ThreadListenerThreadLocalHandlesAllocator implements ThreadListener { + @Override + public void beforeThreadRun() { + InterpreterStubSection.TL_HANDLES.set(new ThreadLocalHandles<>(InterpreterStubSection.MAX_ARGUMENT_HANDLES)); + ThreadListener.super.beforeThreadRun(); + } +}