From 771739e01448896a6910814fabc3cbcbbdbacd33 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 5 Jun 2025 10:06:06 +0200 Subject: [PATCH 1/2] Update TLAB logic to JDK 25+25. Fix gcWaste computation. --- .../genscavenge/ThreadLocalAllocation.java | 28 ++++------ .../svm/core/genscavenge/TlabSupport.java | 54 +++++++++---------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index 9ba1b4a1396c..3a12a919b2a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -25,16 +25,11 @@ package com.oracle.svm.core.genscavenge; import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; -import static com.oracle.svm.core.genscavenge.TlabSupport.computeMinSizeOfNewTlab; -import static com.oracle.svm.core.genscavenge.TlabSupport.computeSizeOfNewTlab; -import static com.oracle.svm.core.genscavenge.TlabSupport.fillTlab; -import static com.oracle.svm.core.genscavenge.TlabSupport.recordSlowAllocation; -import static com.oracle.svm.core.genscavenge.TlabSupport.retireTlabBeforeAllocation; -import static com.oracle.svm.core.genscavenge.TlabSupport.shouldRetainTlab; import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_END_IDENTITY; import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_START_IDENTITY; import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_TOP_IDENTITY; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -371,6 +366,7 @@ private static Pointer allocateRawMemory(UnsignedWord size, BooleanPointer alloc return allocateRawMemoryOutsideTlab(size, allocatedOutsideTlab); } + // TEMP (chaeubl): move this to TlabSupport and make most TlabSupport methods private @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L257-L329") @Uninterruptible(reason = "Holds uninitialized memory.") private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { @@ -380,19 +376,18 @@ private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { * Retain tlab and allocate object as an heap allocation if the amount free in the tlab is * too large to discard. */ - if (shouldRetainTlab(tlab)) { - recordSlowAllocation(); + if (TlabSupport.shouldRetainTlab(tlab)) { + TlabSupport.recordSlowAllocation(); return Word.nullPointer(); } - /* - * Discard tlab and allocate a new one. To minimize fragmentation, the last tlab may be - * smaller than the rest. - */ - UnsignedWord newTlabSize = computeSizeOfNewTlab(size); + /* Discard tlab and allocate a new one. */ - retireTlabBeforeAllocation(); + TlabSupport.recordRefillWaste(); + TlabSupport.retireTlab(CurrentIsolate.getCurrentThread(), false); + /* To minimize fragmentation, the last tlab may be smaller than the rest. */ + UnsignedWord newTlabSize = TlabSupport.computeSizeOfNewTlab(size); if (newTlabSize.equal(0)) { return Word.nullPointer(); } @@ -401,8 +396,7 @@ private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { * Allocate a new TLAB requesting newTlabSize. Any size between minimal and newTlabSize is * accepted. */ - - UnsignedWord computedMinSize = computeMinSizeOfNewTlab(size); + UnsignedWord computedMinSize = TlabSupport.computeMinSizeOfNewTlab(size); WordPointer allocatedTlabSize = StackValue.get(WordPointer.class); Pointer memory = YoungGeneration.getHeapAllocation().allocateNewTlab(computedMinSize, newTlabSize, allocatedTlabSize); @@ -412,7 +406,7 @@ private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { } assert Word.unsigned(0).notEqual(allocatedTlabSize.read()) : "Allocation succeeded but actual size not updated."; - fillTlab(memory, memory.add(size), allocatedTlabSize); + TlabSupport.fillTlab(memory, memory.add(size), allocatedTlabSize); return memory; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java index a9aa2835bf41..2065e67706dd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java @@ -70,7 +70,7 @@ * and another thread allocates nothing. Between GC 23 and GC 24 the allocation behaviour of these * two threads switches. The allocation average and the TLAB size adapt to the new allocation * behavior. - * + * * 
  * +-----+---------------------------------------++---------------------------------------+
  * | #GC | Thread 1                              || Thread 2                              |
@@ -87,7 +87,7 @@
  * |  29 |           0B |   270,44kB |    5,41kB ||       3,55MB |     3,28MB |   67,14kB |
  * +-----+--------------+------------+-----------++--------------+------------+-----------+
  *
- * + *

* A thread allocating a very large amount of memory will also have a high * {@link #allocatedBytesAvg}. If such a thread later changes its allocation behaviour and only * allocates a small amount of memory the {@link #allocatedBytesAvg} starts decreasing with the next @@ -109,25 +109,23 @@ public class TlabSupport { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/tlab_globals.hpp#L82-L85")// private static final long TLAB_WASTE_INCREMENT = 4; - // The desired size of the TLAB, including the reserve for filling the unused memory. + /* The desired size of the TLAB, including the reserve for filling the unused memory. */ private static final FastThreadLocalWord desiredSize = FastThreadLocalFactory.createWord("TlabSupport.desiredSize"); - private static final FastThreadLocalWord tlabAllocatedAlignedBytesBeforeLastGC = FastThreadLocalFactory.createWord("TlabSupport.tlabAllocatedAlignedBytesBeforeLastGC"); - private static final FastThreadLocalInt numberOfRefills = FastThreadLocalFactory.createInt("TlabSupport.numberOfRefills"); private static final FastThreadLocalInt refillWaste = FastThreadLocalFactory.createInt("TlabSupport.refillWaste"); private static final FastThreadLocalInt gcWaste = FastThreadLocalFactory.createInt("TlabSupport.gcWaste"); - // Average of allocated bytes in TLABs of this thread. + /* Average of allocated bytes in TLABs of this thread. */ private static final FastThreadLocalBytes allocatedBytesAvg = FastThreadLocalFactory .createBytes(() -> SizeOf.get(AdaptiveWeightedAverageStruct.Data.class), "TlabSupport.allocatedBytesAvg"); - // Hold onto the TLAB if availableTlabMemory() is larger than this. + /* Hold onto the TLAB if availableTlabMemory() is larger than this. */ private static final FastThreadLocalWord refillWasteLimit = FastThreadLocalFactory.createWord("TlabSupport.refillWasteLimit"); private static final FastThreadLocalInt slowAllocations = FastThreadLocalFactory.createInt("TlabSupport.slowAllocations"); - // Expected number of refills between GCs. + /* Expected number of refills between GCs. */ private static UnsignedWord targetRefills = Word.unsigned(1); private static boolean initialized; @@ -160,9 +158,11 @@ public static void initialize(IsolateThread thread) { resetStatistics(thread); } + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L168-L174") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L183-L195") @Uninterruptible(reason = "Accesses TLAB") static void fillTlab(Pointer start, Pointer top, WordPointer newSize) { + /* Fill the TLAB. */ numberOfRefills.set(numberOfRefills.get() + 1); Pointer hardEnd = start.add(newSize.read()); @@ -172,39 +172,38 @@ static void fillTlab(Pointer start, Pointer top, WordPointer newSize) { initialize(getTlab(), start, top, end); - // Reset amount of internal fragmentation + /* Reset amount of internal fragmentation. */ refillWasteLimit.set(initialRefillWasteLimit()); } @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L143-L145") @Uninterruptible(reason = "Accesses TLAB") - static void retireTlabBeforeAllocation() { + static void recordRefillWaste() { long availableTlabMemory = availableTlabMemory(getTlab()).rawValue(); refillWaste.set(refillWaste.get() + UninterruptibleUtils.NumUtil.safeToInt(availableTlabMemory)); - retireCurrentTlab(CurrentIsolate.getCurrentThread(), false); } + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L157-L166") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L131-L141") @Uninterruptible(reason = "Accesses TLAB") - private static void retireCurrentTlab(IsolateThread thread, boolean calculateStats) { + static void retireTlab(IsolateThread thread, boolean calculateStats) { + /* Sampling and serviceability support. */ ThreadLocalAllocation.Descriptor tlab = getTlab(thread); - if (tlab.getAllocationEnd(TLAB_END_IDENTITY).isNonNull()) { - assert checkInvariants(tlab); - - UnsignedWord usedTlabSize = getUsedTlabSize(tlab); - allocatedAlignedBytes.set(thread, allocatedAlignedBytes.get(thread).add(usedTlabSize)); - insertFiller(tlab); - initialize(tlab, Word.nullPointer(), Word.nullPointer(), Word.nullPointer()); + UnsignedWord usedBytes = getUsedTlabSize(tlab); + allocatedAlignedBytes.set(thread, allocatedAlignedBytes.get(thread).add(usedBytes)); } - /* - * Collect statistics after the TLAB has been retired. Otherwise, the current TLAB is - * excluded from the statistics. - */ + /* Retire the TLAB. */ if (calculateStats) { accumulateAndResetStatistics(thread); } + + if (tlab.getAllocationEnd(TLAB_END_IDENTITY).isNonNull()) { + assert checkInvariants(tlab); + insertFiller(tlab); + initialize(tlab, Word.nullPointer(), Word.nullPointer(), Word.nullPointer()); + } } @Uninterruptible(reason = "Accesses TLAB") @@ -238,7 +237,7 @@ private static boolean checkInvariants(Descriptor tlab) { @Uninterruptible(reason = "Accesses TLAB") static void suspendAllocationInCurrentThread() { /* The statistics for this thread will be updated later. */ - retireCurrentTlab(CurrentIsolate.getCurrentThread(), false); + retireTlab(CurrentIsolate.getCurrentThread(), false); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -268,7 +267,7 @@ static void disableAndFlushForThread(IsolateThread vmThread) { private static void retireTlabToEden(IsolateThread thread) { VMThreads.guaranteeOwnsThreadMutex("Otherwise, we wouldn't be allowed to access the space.", true); - retireCurrentTlab(thread, true); + retireTlab(thread, true); Descriptor tlab = getTlab(thread); UnalignedHeapChunk.UnalignedHeader unalignedChunk = tlab.getUnalignedChunk(); @@ -436,7 +435,9 @@ static UnsignedWord maxSize() { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L76-L117") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) private static void accumulateAndResetStatistics(IsolateThread thread) { - gcWaste.set(thread, gcWaste.get() + UninterruptibleUtils.NumUtil.safeToInt(availableTlabMemory(getTlab(thread)).rawValue())); + UnsignedWord remaining = availableTlabMemory(getTlab()); + gcWaste.set(thread, gcWaste.get() + UnsignedUtils.safeToInt(remaining)); + UnsignedWord totalAlignedAllocated = ThreadLocalAllocation.getAlignedAllocatedBytes(thread); UnsignedWord allocatedAlignedSinceLastGC = totalAlignedAllocated.subtract(tlabAllocatedAlignedBytesBeforeLastGC.get(thread)); tlabAllocatedAlignedBytesBeforeLastGC.set(thread, totalAlignedAllocated); @@ -444,7 +445,6 @@ private static void accumulateAndResetStatistics(IsolateThread thread) { AdaptiveWeightedAverageStruct.sample(allocatedBytesAvg.getAddress(thread), allocatedAlignedSinceLastGC.rawValue()); printStats(thread, allocatedAlignedSinceLastGC); - resetStatistics(thread); } From 089e691d2d36af0b15e0f49268663baec9ee2728 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 5 Jun 2025 10:14:54 +0200 Subject: [PATCH 2/2] Refactorings to improve the encapsulation of TlabSupport. --- .../genscavenge/ThreadLocalAllocation.java | 48 +-------------- .../svm/core/genscavenge/TlabSupport.java | 61 ++++++++++++++++--- 2 files changed, 52 insertions(+), 57 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index 3a12a919b2a8..24a72f72e0d8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -29,7 +29,6 @@ import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_START_IDENTITY; import static com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets.TLAB_TOP_IDENTITY; -import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -39,7 +38,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity; -import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -359,57 +357,13 @@ private static Object allocateArraySlow(DynamicHub hub, int length, UnsignedWord @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/memAllocator.cpp#L333-L341") @Uninterruptible(reason = "Holds uninitialized memory.") private static Pointer allocateRawMemory(UnsignedWord size, BooleanPointer allocatedOutsideTlab) { - Pointer memory = allocateRawMemoryInTlabSlow(size); + Pointer memory = TlabSupport.allocateRawMemoryInTlabSlow(size); if (memory.isNonNull()) { return memory; } return allocateRawMemoryOutsideTlab(size, allocatedOutsideTlab); } - // TEMP (chaeubl): move this to TlabSupport and make most TlabSupport methods private - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L257-L329") - @Uninterruptible(reason = "Holds uninitialized memory.") - private static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { - ThreadLocalAllocation.Descriptor tlab = getTlab(); - - /* - * Retain tlab and allocate object as an heap allocation if the amount free in the tlab is - * too large to discard. - */ - if (TlabSupport.shouldRetainTlab(tlab)) { - TlabSupport.recordSlowAllocation(); - return Word.nullPointer(); - } - - /* Discard tlab and allocate a new one. */ - - TlabSupport.recordRefillWaste(); - TlabSupport.retireTlab(CurrentIsolate.getCurrentThread(), false); - - /* To minimize fragmentation, the last tlab may be smaller than the rest. */ - UnsignedWord newTlabSize = TlabSupport.computeSizeOfNewTlab(size); - if (newTlabSize.equal(0)) { - return Word.nullPointer(); - } - - /* - * Allocate a new TLAB requesting newTlabSize. Any size between minimal and newTlabSize is - * accepted. - */ - UnsignedWord computedMinSize = TlabSupport.computeMinSizeOfNewTlab(size); - - WordPointer allocatedTlabSize = StackValue.get(WordPointer.class); - Pointer memory = YoungGeneration.getHeapAllocation().allocateNewTlab(computedMinSize, newTlabSize, allocatedTlabSize); - if (memory.isNull()) { - assert Word.unsigned(0).equal(allocatedTlabSize.read()) : "Allocation failed, but actual size was updated."; - return Word.nullPointer(); - } - assert Word.unsigned(0).notEqual(allocatedTlabSize.read()) : "Allocation succeeded but actual size not updated."; - - TlabSupport.fillTlab(memory, memory.add(size), allocatedTlabSize); - return memory; - } - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L239-L251") @Uninterruptible(reason = "Holds uninitialized memory.") private static Pointer allocateRawMemoryOutsideTlab(UnsignedWord size, BooleanPointer allocatedOutsideTlab) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java index 2065e67706dd..1c672aff3671 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java @@ -34,6 +34,7 @@ import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; @@ -158,10 +159,52 @@ public static void initialize(IsolateThread thread) { resetStatistics(thread); } + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/memAllocator.cpp#L257-L329") + @Uninterruptible(reason = "Holds uninitialized memory.") + static Pointer allocateRawMemoryInTlabSlow(UnsignedWord size) { + ThreadLocalAllocation.Descriptor tlab = getTlab(); + + /* + * Retain tlab and allocate object as an heap allocation if the amount free in the tlab is + * too large to discard. + */ + if (shouldRetainTlab(tlab)) { + recordSlowAllocation(); + return Word.nullPointer(); + } + + /* Discard tlab and allocate a new one. */ + recordRefillWaste(); + retireTlab(CurrentIsolate.getCurrentThread(), false); + + /* To minimize fragmentation, the last tlab may be smaller than the rest. */ + UnsignedWord newTlabSize = computeSizeOfNewTlab(size); + if (newTlabSize.equal(0)) { + return Word.nullPointer(); + } + + /* + * Allocate a new TLAB requesting newTlabSize. Any size between minimal and newTlabSize is + * accepted. + */ + UnsignedWord computedMinSize = computeMinSizeOfNewTlab(size); + + WordPointer allocatedTlabSize = StackValue.get(WordPointer.class); + Pointer memory = YoungGeneration.getHeapAllocation().allocateNewTlab(computedMinSize, newTlabSize, allocatedTlabSize); + if (memory.isNull()) { + assert Word.unsigned(0).equal(allocatedTlabSize.read()) : "Allocation failed, but actual size was updated."; + return Word.nullPointer(); + } + assert Word.unsigned(0).notEqual(allocatedTlabSize.read()) : "Allocation succeeded but actual size not updated."; + + fillTlab(memory, memory.add(size), allocatedTlabSize); + return memory; + } + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L168-L174") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L183-L195") @Uninterruptible(reason = "Accesses TLAB") - static void fillTlab(Pointer start, Pointer top, WordPointer newSize) { + private static void fillTlab(Pointer start, Pointer top, WordPointer newSize) { /* Fill the TLAB. */ numberOfRefills.set(numberOfRefills.get() + 1); @@ -178,7 +221,7 @@ static void fillTlab(Pointer start, Pointer top, WordPointer newSize) { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L143-L145") @Uninterruptible(reason = "Accesses TLAB") - static void recordRefillWaste() { + private static void recordRefillWaste() { long availableTlabMemory = availableTlabMemory(getTlab()).rawValue(); refillWaste.set(refillWaste.get() + UninterruptibleUtils.NumUtil.safeToInt(availableTlabMemory)); } @@ -186,7 +229,7 @@ static void recordRefillWaste() { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/runtime/thread.cpp#L157-L166") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+25/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L131-L141") @Uninterruptible(reason = "Accesses TLAB") - static void retireTlab(IsolateThread thread, boolean calculateStats) { + private static void retireTlab(IsolateThread thread, boolean calculateStats) { /* Sampling and serviceability support. */ ThreadLocalAllocation.Descriptor tlab = getTlab(thread); if (tlab.getAllocationEnd(TLAB_END_IDENTITY).isNonNull()) { @@ -284,7 +327,7 @@ private static void retireTlabToEden(IsolateThread thread) { } @Uninterruptible(reason = "Accesses TLAB") - static UnsignedWord availableTlabMemory(Descriptor tlab) { + private static UnsignedWord availableTlabMemory(Descriptor tlab) { Pointer top = tlab.getAllocationTop(TLAB_TOP_IDENTITY); Pointer end = tlab.getAllocationEnd(TLAB_END_IDENTITY); assert top.belowOrEqual(end); @@ -322,7 +365,6 @@ private static void insertFiller(ThreadLocalAllocation.Descriptor tlab) { if (top.belowThan(hardEnd)) { FillerObjectUtil.writeFillerObjectAt(top, size); } - } @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L175-L181") @@ -384,7 +426,7 @@ private static UnsignedWord initialRefillWasteLimit() { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+8/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L54-L71") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) { + private static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) { assert UnsignedUtils.isAMultiple(allocationSize, Word.unsigned(ConfigurationValues.getObjectLayout().getAlignment())); /* @@ -403,7 +445,7 @@ static UnsignedWord computeSizeOfNewTlab(UnsignedWord allocationSize) { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L73-L77") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) { + private static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) { UnsignedWord alignedSize = Word.unsigned(ConfigurationValues.getObjectLayout().alignUp(allocationSize.rawValue())); UnsignedWord sizeWithReserve = alignedSize.add(getFillerObjectSize()); long minTlabSize = TlabOptionCache.singleton().getMinTlabSize(); @@ -412,13 +454,13 @@ static UnsignedWord computeMinSizeOfNewTlab(UnsignedWord allocationSize) { } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - static boolean shouldRetainTlab(Descriptor tlab) { + private static boolean shouldRetainTlab(Descriptor tlab) { return availableTlabMemory(tlab).aboveThan(refillWasteLimit.get()); } @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+11/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp#L79-L94") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - static void recordSlowAllocation() { + private static void recordSlowAllocation() { /* * Raise size required to bypass TLAB next time. Else there's a risk that a thread that * repeatedly allocates objects of one size will get stuck on this slow path. @@ -469,7 +511,6 @@ static void logTlabChunks(Log log, IsolateThread thread, String shortSpaceName) ThreadLocalAllocation.Descriptor tlab = getTlabUnsafe(thread); // Aligned chunks are handled in HeapAllocation. - UnalignedHeapChunk.UnalignedHeader uChunk = tlab.getUnalignedChunk(); HeapChunkLogging.logChunks(log, uChunk, shortSpaceName, false); }