295 Zeilen
17 KiB
Diff
295 Zeilen
17 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Aikar <aikar@aikar.co>
|
|
Date: Wed, 8 Apr 2020 03:06:30 -0400
|
|
Subject: [PATCH] Optimize PlayerChunkMap memory use for visibleChunks
|
|
|
|
No longer clones visible chunks which is causing massive memory
|
|
allocation issues, likely the source of Humongous Objects on large servers.
|
|
|
|
Instead we just synchronize, clear and rebuild, reusing the same object buffers
|
|
as before with only 2 small objects created (FastIterator/MapEntry)
|
|
|
|
This should result in siginificant memory use reduction and improved GC behavior.
|
|
|
|
diff --git a/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..f6ff4d8132a95895680f5bc81f8f873e78f0bbdb
|
|
--- /dev/null
|
|
+++ b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
@@ -0,0 +1,39 @@
|
|
+package com.destroystokyo.paper.util.map;
|
|
+
|
|
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
|
+
|
|
+public class Long2ObjectLinkedOpenHashMapFastCopy<V> extends Long2ObjectLinkedOpenHashMap<V> {
|
|
+
|
|
+ public void copyFrom(Long2ObjectLinkedOpenHashMapFastCopy<V> map) {
|
|
+ if (key.length != map.key.length) {
|
|
+ key = null;
|
|
+ key = new long[map.key.length];
|
|
+ }
|
|
+ if (value.length != map.value.length) {
|
|
+ value = null;
|
|
+ //noinspection unchecked
|
|
+ value = (V[]) new Object[map.value.length];
|
|
+ }
|
|
+ if (link.length != map.link.length) {
|
|
+ link = null;
|
|
+ link = new long[map.link.length];
|
|
+ }
|
|
+ System.arraycopy(map.key, 0, this.key, 0, map.key.length);
|
|
+ System.arraycopy(map.value, 0, this.value, 0, map.value.length);
|
|
+ System.arraycopy(map.link, 0, this.link, 0, map.link.length);
|
|
+ this.size = map.size;
|
|
+ this.mask = map.mask;
|
|
+ this.first = map.first;
|
|
+ this.last = map.last;
|
|
+ this.n = map.n;
|
|
+ this.maxFill = map.maxFill;
|
|
+ this.containsNullKey = map.containsNullKey;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public Long2ObjectLinkedOpenHashMapFastCopy<V> clone() {
|
|
+ Long2ObjectLinkedOpenHashMapFastCopy<V> clone = (Long2ObjectLinkedOpenHashMapFastCopy<V>) super.clone();
|
|
+ clone.copyFrom(this);
|
|
+ return clone;
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
|
|
index 99c3337eec552ba47d3b8b2d8feaaa80acf2a86f..9abef8550a89df5e15ac28de1a5549d064f29122 100644
|
|
--- a/src/main/java/net/minecraft/server/MCUtil.java
|
|
+++ b/src/main/java/net/minecraft/server/MCUtil.java
|
|
@@ -616,7 +616,7 @@ public final class MCUtil {
|
|
|
|
ServerLevel world = ((org.bukkit.craftbukkit.CraftWorld)bukkitWorld).getHandle();
|
|
ChunkMap chunkMap = world.getChunkSource().chunkMap;
|
|
- Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.visibleChunkMap;
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.getVisibleChunks();
|
|
DistanceManager chunkMapDistance = chunkMap.distanceManager;
|
|
List<ChunkHolder> allChunks = new ArrayList<>(visibleChunks.values());
|
|
List<ServerPlayer> players = world.players;
|
|
diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
index e6eeca5834a164d87f5b0e564fe6237902edaa6a..99eee56d6e66b79dd48ecbd1eeebc08174003f4a 100644
|
|
--- a/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
+++ b/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
@@ -104,8 +104,33 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
|
|
private static final Logger LOGGER = LogManager.getLogger();
|
|
public static final int MAX_CHUNK_DISTANCE = 33 + ChunkStatus.maxDistance();
|
|
- public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new Long2ObjectLinkedOpenHashMap();
|
|
- public volatile Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap;
|
|
+ // Paper start - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap = new ProtectedVisibleChunksMap(); // Paper - faster copying
|
|
+
|
|
+ private class ProtectedVisibleChunksMap extends com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> {
|
|
+ @Override
|
|
+ public ChunkHolder put(long k, ChunkHolder playerChunk) {
|
|
+ throw new UnsupportedOperationException("Updating visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public ChunkHolder remove(long k) {
|
|
+ throw new UnsupportedOperationException("Removing visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public ChunkHolder get(long k) {
|
|
+ return ChunkMap.this.getVisibleChunkIfPresent(k);
|
|
+ }
|
|
+
|
|
+ public ChunkHolder safeGet(long k) {
|
|
+ return super.get(k);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> pendingVisibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>(); // Paper - this is used if the visible chunks is updated while iterating only
|
|
+ public transient com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> visibleChunksClone; // Paper - used for async access of visible chunks, clone and cache only when needed
|
|
private final Long2ObjectLinkedOpenHashMap<ChunkHolder> pendingUnloads;
|
|
public final LongSet entitiesInLevel; // Paper - private -> public
|
|
public final ServerLevel level;
|
|
@@ -178,7 +203,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
|
|
public ChunkMap(ServerLevel worldserver, LevelStorageSource.LevelStorageAccess convertable_conversionsession, DataFixer dataFixer, StructureManager definedstructuremanager, Executor workerExecutor, BlockableEventLoop<Runnable> mainThreadExecutor, LightChunkGetter chunkProvider, ChunkGenerator chunkGenerator, ChunkProgressListener worldGenerationProgressListener, Supplier<DimensionDataStorage> supplier, int i, boolean flag) {
|
|
super(new File(convertable_conversionsession.getDimensionPath(worldserver.dimension()), "region"), dataFixer, flag);
|
|
- this.visibleChunkMap = this.updatingChunkMap.clone();
|
|
+ //this.visibleChunks = this.updatingChunks.clone(); // Paper - no more cloning
|
|
this.pendingUnloads = new Long2ObjectLinkedOpenHashMap();
|
|
this.entitiesInLevel = new LongOpenHashSet();
|
|
this.toDrop = new LongOpenHashSet();
|
|
@@ -270,9 +295,52 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
return (ChunkHolder) this.updatingChunkMap.get(pos);
|
|
}
|
|
|
|
+ // Paper start - remove cloning of visible chunks unless accessed as a collection async
|
|
+ private static final boolean DEBUG_ASYNC_VISIBLE_CHUNKS = Boolean.getBoolean("paper.debug-async-visible-chunks");
|
|
+ private boolean isIterating = false;
|
|
+ private boolean hasPendingVisibleUpdate = false;
|
|
+ public void forEachVisibleChunk(java.util.function.Consumer<ChunkHolder> consumer) {
|
|
+ org.spigotmc.AsyncCatcher.catchOp("forEachVisibleChunk");
|
|
+ boolean prev = isIterating;
|
|
+ isIterating = true;
|
|
+ try {
|
|
+ for (ChunkHolder value : this.visibleChunkMap.values()) {
|
|
+ consumer.accept(value);
|
|
+ }
|
|
+ } finally {
|
|
+ this.isIterating = prev;
|
|
+ if (!this.isIterating && this.hasPendingVisibleUpdate) {
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom(this.pendingVisibleChunks);
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ this.hasPendingVisibleUpdate = false;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ public Long2ObjectLinkedOpenHashMap<ChunkHolder> getVisibleChunks() {
|
|
+ if (Thread.currentThread() == this.level.thread) {
|
|
+ return this.visibleChunkMap;
|
|
+ } else {
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ if (DEBUG_ASYNC_VISIBLE_CHUNKS) new Throwable("Async getVisibleChunks").printStackTrace();
|
|
+ if (this.visibleChunksClone == null) {
|
|
+ this.visibleChunksClone = this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.clone() : ((ProtectedVisibleChunksMap)this.visibleChunkMap).clone();
|
|
+ }
|
|
+ return this.visibleChunksClone;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
@Nullable
|
|
public ChunkHolder getVisibleChunkIfPresent(long pos) { // Paper - protected -> public
|
|
- return (ChunkHolder) this.visibleChunkMap.get(pos);
|
|
+ // Paper start - mt safe get
|
|
+ if (Thread.currentThread() != this.level.thread) {
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
|
|
+ }
|
|
+ }
|
|
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
|
|
+ // Paper end
|
|
}
|
|
|
|
protected IntSupplier getChunkQueueLevel(long pos) {
|
|
@@ -460,8 +528,9 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
// Paper end
|
|
|
|
protected void saveAllChunks(boolean flush) {
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = this.getVisibleChunks(); // Paper remove clone of visible Chunks unless saving off main thread (watchdog kill)
|
|
if (flush) {
|
|
- List<ChunkHolder> list = (List) this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList());
|
|
+ List<ChunkHolder> list = (List) visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList()); // Paper - remove cloning of visible chunks
|
|
MutableBoolean mutableboolean = new MutableBoolean();
|
|
|
|
do {
|
|
@@ -489,7 +558,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
// this.i(); // Paper - nuke IOWorker
|
|
ChunkMap.LOGGER.info("ThreadedAnvilChunkStorage ({}): All chunks are saved", this.storageFolder.getName());
|
|
} else {
|
|
- this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
|
|
+ visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
|
|
ChunkAccess ichunkaccess = (ChunkAccess) playerchunk.getChunkToSave().getNow(null); // CraftBukkit - decompile error
|
|
|
|
if (ichunkaccess instanceof ImposterProtoChunk || ichunkaccess instanceof LevelChunk) {
|
|
@@ -660,7 +729,20 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
if (!this.modified) {
|
|
return false;
|
|
} else {
|
|
- this.visibleChunkMap = this.updatingChunkMap.clone();
|
|
+ // Paper start - stop cloning visibleChunks
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ if (isIterating) {
|
|
+ hasPendingVisibleUpdate = true;
|
|
+ this.pendingVisibleChunks.copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
|
|
+ } else {
|
|
+ hasPendingVisibleUpdate = false;
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
|
|
+ this.visibleChunksClone = null;
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
this.modified = false;
|
|
return true;
|
|
}
|
|
@@ -1139,12 +1221,12 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
}
|
|
|
|
protected Iterable<ChunkHolder> getChunks() {
|
|
- return Iterables.unmodifiableIterable(this.visibleChunkMap.values());
|
|
+ return Iterables.unmodifiableIterable(this.getVisibleChunks().values()); // Paper
|
|
}
|
|
|
|
void dumpChunks(Writer writer) throws IOException {
|
|
CsvOutput csvwriter = CsvOutput.builder().addColumn("x").addColumn("z").addColumn("level").addColumn("in_memory").addColumn("status").addColumn("full_status").addColumn("accessible_ready").addColumn("ticking_ready").addColumn("entity_ticking_ready").addColumn("ticket").addColumn("spawning").addColumn("entity_count").addColumn("block_entity_count").build(writer);
|
|
- ObjectBidirectionalIterator objectbidirectionaliterator = this.visibleChunkMap.long2ObjectEntrySet().iterator();
|
|
+ ObjectBidirectionalIterator objectbidirectionaliterator = this.getVisibleChunks().long2ObjectEntrySet().iterator(); // Paper
|
|
|
|
while (objectbidirectionaliterator.hasNext()) {
|
|
Entry<ChunkHolder> entry = (Entry) objectbidirectionaliterator.next();
|
|
diff --git a/src/main/java/net/minecraft/server/level/ServerChunkCache.java b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
index e2b1541042bceac965411e3176d08c61f217c07f..f5de878020be9465739fba07fd7dea46b0a3ae34 100644
|
|
--- a/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
+++ b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
@@ -782,7 +782,7 @@ public class ServerChunkCache extends ChunkSource {
|
|
};
|
|
// Paper end
|
|
this.level.timings.chunkTicks.startTiming(); // Paper
|
|
- this.chunkMap.getChunks().forEach((playerchunk) -> { // Paper - no... just no...
|
|
+ this.chunkMap.forEachVisibleChunk((playerchunk) -> { // Paper - safe iterator incase chunk loads, also no wrapping
|
|
Optional<LevelChunk> optional = ((Either) playerchunk.getTickingChunkFuture().getNow(ChunkHolder.UNLOADED_LEVEL_CHUNK)).left();
|
|
|
|
if (optional.isPresent()) {
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
index fb74bdcf4c2935b56e92717cc5a1504fbc853d0a..1a839242e359fa32f32d0e571c6e918ac39642e9 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
@@ -276,6 +276,7 @@ public class CraftWorld implements World {
|
|
return ret;
|
|
}
|
|
public int getTileEntityCount() {
|
|
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
|
|
// We don't use the full world tile entity list, so we must iterate chunks
|
|
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
|
|
int size = 0;
|
|
@@ -287,11 +288,13 @@ public class CraftWorld implements World {
|
|
size += chunk.blockEntities.size();
|
|
}
|
|
return size;
|
|
+ });
|
|
}
|
|
public int getTickableTileEntityCount() {
|
|
return world.tickableBlockEntities.size();
|
|
}
|
|
public int getChunkCount() {
|
|
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
|
|
int ret = 0;
|
|
|
|
for (ChunkHolder chunkHolder : world.getChunkSource().chunkMap.visibleChunkMap.values()) {
|
|
@@ -300,7 +303,7 @@ public class CraftWorld implements World {
|
|
}
|
|
}
|
|
|
|
- return ret;
|
|
+ return ret; });
|
|
}
|
|
public int getPlayerCount() {
|
|
return world.players.size();
|
|
@@ -425,6 +428,14 @@ public class CraftWorld implements World {
|
|
|
|
@Override
|
|
public Chunk[] getLoadedChunks() {
|
|
+ // Paper start
|
|
+ if (Thread.currentThread() != world.getLevel().thread) {
|
|
+ synchronized (world.getChunkSource().chunkMap.visibleChunkMap) {
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
|
|
+ return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
|
|
return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
}
|