300 Zeilen
17 KiB
Diff
300 Zeilen
17 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Aikar <aikar@aikar.co>
|
|
Date: Wed, 8 Apr 2020 03:06:30 -0400
|
|
Subject: [PATCH] Optimize PlayerChunkMap memory use for visibleChunks
|
|
|
|
No longer clones visible chunks which is causing massive memory
|
|
allocation issues, likely the source of Humongous Objects on large servers.
|
|
|
|
Instead we just synchronize, clear and rebuild, reusing the same object buffers
|
|
as before with only 2 small objects created (FastIterator/MapEntry)
|
|
|
|
This should result in siginificant memory use reduction and improved GC behavior.
|
|
|
|
diff --git a/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..f6ff4d8132a95895680f5bc81f8f873e78f0bbdb
|
|
--- /dev/null
|
|
+++ b/src/main/java/com/destroystokyo/paper/util/map/Long2ObjectLinkedOpenHashMapFastCopy.java
|
|
@@ -0,0 +1,39 @@
|
|
+package com.destroystokyo.paper.util.map;
|
|
+
|
|
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
|
+
|
|
+public class Long2ObjectLinkedOpenHashMapFastCopy<V> extends Long2ObjectLinkedOpenHashMap<V> {
|
|
+
|
|
+ public void copyFrom(Long2ObjectLinkedOpenHashMapFastCopy<V> map) {
|
|
+ if (key.length != map.key.length) {
|
|
+ key = null;
|
|
+ key = new long[map.key.length];
|
|
+ }
|
|
+ if (value.length != map.value.length) {
|
|
+ value = null;
|
|
+ //noinspection unchecked
|
|
+ value = (V[]) new Object[map.value.length];
|
|
+ }
|
|
+ if (link.length != map.link.length) {
|
|
+ link = null;
|
|
+ link = new long[map.link.length];
|
|
+ }
|
|
+ System.arraycopy(map.key, 0, this.key, 0, map.key.length);
|
|
+ System.arraycopy(map.value, 0, this.value, 0, map.value.length);
|
|
+ System.arraycopy(map.link, 0, this.link, 0, map.link.length);
|
|
+ this.size = map.size;
|
|
+ this.mask = map.mask;
|
|
+ this.first = map.first;
|
|
+ this.last = map.last;
|
|
+ this.n = map.n;
|
|
+ this.maxFill = map.maxFill;
|
|
+ this.containsNullKey = map.containsNullKey;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public Long2ObjectLinkedOpenHashMapFastCopy<V> clone() {
|
|
+ Long2ObjectLinkedOpenHashMapFastCopy<V> clone = (Long2ObjectLinkedOpenHashMapFastCopy<V>) super.clone();
|
|
+ clone.copyFrom(this);
|
|
+ return clone;
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
|
|
index 4bc418f45c6f16a9a1c78b6688625268870a96ad..2d5b8e35d52b0dfd075af81a3a936d8a21053b31 100644
|
|
--- a/src/main/java/net/minecraft/server/MCUtil.java
|
|
+++ b/src/main/java/net/minecraft/server/MCUtil.java
|
|
@@ -614,7 +614,7 @@ public final class MCUtil {
|
|
|
|
ServerLevel world = ((org.bukkit.craftbukkit.CraftWorld)bukkitWorld).getHandle();
|
|
ChunkMap chunkMap = world.getChunkSource().chunkMap;
|
|
- Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.visibleChunkMap;
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = chunkMap.getVisibleChunks();
|
|
DistanceManager chunkMapDistance = chunkMap.distanceManager;
|
|
List<ChunkHolder> allChunks = new ArrayList<>(visibleChunks.values());
|
|
List<ServerPlayer> players = world.players;
|
|
diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
index 2b291296821dc6d6a8437bd977eeba517cdb5003..962028a58ee54b99be20905c1fd16cfe15c82175 100644
|
|
--- a/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
+++ b/src/main/java/net/minecraft/server/level/ChunkMap.java
|
|
@@ -112,9 +112,36 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
private static final int MIN_VIEW_DISTANCE = 3;
|
|
public static final int MAX_VIEW_DISTANCE = 33;
|
|
public static final int MAX_CHUNK_DISTANCE = 33 + ChunkStatus.maxDistance();
|
|
+ // Paper start - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<>(); // Paper - faster copying
|
|
+ public final Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap = new ProtectedVisibleChunksMap(); // Paper - faster copying
|
|
+
|
|
+ private class ProtectedVisibleChunksMap extends com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> {
|
|
+ @Override
|
|
+ public ChunkHolder put(long k, ChunkHolder playerChunk) {
|
|
+ throw new UnsupportedOperationException("Updating visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public ChunkHolder remove(long k) {
|
|
+ throw new UnsupportedOperationException("Removing visible Chunks");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public ChunkHolder get(long k) {
|
|
+ return ChunkMap.this.getVisibleChunkIfPresent(k);
|
|
+ }
|
|
+
|
|
+ public ChunkHolder safeGet(long k) {
|
|
+ return super.get(k);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+ public final com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> pendingVisibleChunks = new com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>(); // Paper - this is used if the visible chunks is updated while iterating only
|
|
+ public transient com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder> visibleChunksClone; // Paper - used for async access of visible chunks, clone and cache only when needed
|
|
public static final int FORCED_TICKET_LEVEL = 31;
|
|
- public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new Long2ObjectLinkedOpenHashMap();
|
|
- public volatile Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap;
|
|
+ // public final Long2ObjectLinkedOpenHashMap<ChunkHolder> updatingChunkMap = new Long2ObjectLinkedOpenHashMap(); // Paper - moved up
|
|
+ // public volatile Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunkMap; // Paper - moved up
|
|
private final Long2ObjectLinkedOpenHashMap<ChunkHolder> pendingUnloads;
|
|
public final LongSet entitiesInLevel;
|
|
public final ServerLevel level;
|
|
@@ -237,7 +264,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
|
|
public ChunkMap(ServerLevel world, LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, StructureManager structureManager, Executor executor, BlockableEventLoop<Runnable> mainThreadExecutor, LightChunkGetter chunkProvider, ChunkGenerator chunkGenerator, ChunkProgressListener worldGenerationProgressListener, ChunkStatusUpdateListener chunkStatusChangeListener, Supplier<DimensionDataStorage> persistentStateManagerFactory, int viewDistance, boolean dsync) {
|
|
super(new File(session.getDimensionPath(world.dimension()), "region"), dataFixer, dsync);
|
|
- this.visibleChunkMap = this.updatingChunkMap.clone();
|
|
+ //this.visibleChunks = this.updatingChunks.clone(); // Paper - no more cloning
|
|
this.pendingUnloads = new Long2ObjectLinkedOpenHashMap();
|
|
this.entitiesInLevel = new LongOpenHashSet();
|
|
this.toDrop = new LongOpenHashSet();
|
|
@@ -378,9 +405,52 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
return (ChunkHolder) this.updatingChunkMap.get(pos);
|
|
}
|
|
|
|
+ // Paper start - remove cloning of visible chunks unless accessed as a collection async
|
|
+ private static final boolean DEBUG_ASYNC_VISIBLE_CHUNKS = Boolean.getBoolean("paper.debug-async-visible-chunks");
|
|
+ private boolean isIterating = false;
|
|
+ private boolean hasPendingVisibleUpdate = false;
|
|
+ public void forEachVisibleChunk(java.util.function.Consumer<ChunkHolder> consumer) {
|
|
+ org.spigotmc.AsyncCatcher.catchOp("forEachVisibleChunk");
|
|
+ boolean prev = isIterating;
|
|
+ isIterating = true;
|
|
+ try {
|
|
+ for (ChunkHolder value : this.visibleChunkMap.values()) {
|
|
+ consumer.accept(value);
|
|
+ }
|
|
+ } finally {
|
|
+ this.isIterating = prev;
|
|
+ if (!this.isIterating && this.hasPendingVisibleUpdate) {
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom(this.pendingVisibleChunks);
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ this.hasPendingVisibleUpdate = false;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ public Long2ObjectLinkedOpenHashMap<ChunkHolder> getVisibleChunks() {
|
|
+ if (Thread.currentThread() == this.level.thread) {
|
|
+ return this.visibleChunkMap;
|
|
+ } else {
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ if (DEBUG_ASYNC_VISIBLE_CHUNKS) new Throwable("Async getVisibleChunks").printStackTrace();
|
|
+ if (this.visibleChunksClone == null) {
|
|
+ this.visibleChunksClone = this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.clone() : ((ProtectedVisibleChunksMap)this.visibleChunkMap).clone();
|
|
+ }
|
|
+ return this.visibleChunksClone;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
@Nullable
|
|
public ChunkHolder getVisibleChunkIfPresent(long pos) {
|
|
- return (ChunkHolder) this.visibleChunkMap.get(pos);
|
|
+ // Paper start - mt safe get
|
|
+ if (Thread.currentThread() != this.level.thread) {
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
|
|
+ }
|
|
+ }
|
|
+ return (ChunkHolder) (this.hasPendingVisibleUpdate ? this.pendingVisibleChunks.get(pos) : ((ProtectedVisibleChunksMap)this.visibleChunkMap).safeGet(pos));
|
|
+ // Paper end
|
|
}
|
|
|
|
protected IntSupplier getChunkQueueLevel(long pos) {
|
|
@@ -537,8 +607,9 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
}
|
|
|
|
protected void saveAllChunks(boolean flush) {
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> visibleChunks = this.getVisibleChunks(); // Paper remove clone of visible Chunks unless saving off main thread (watchdog kill)
|
|
if (flush) {
|
|
- List<ChunkHolder> list = (List) this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList());
|
|
+ List<ChunkHolder> list = (List) visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).peek(ChunkHolder::refreshAccessibility).collect(Collectors.toList()); // Paper - remove cloning of visible chunks
|
|
MutableBoolean mutableboolean = new MutableBoolean();
|
|
|
|
do {
|
|
@@ -568,7 +639,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
this.level.asyncChunkTaskManager.flush(); // Paper - flush to preserve behavior compat with pre-async behaviour
|
|
// this.i(); // Paper - nuke IOWorker
|
|
} else {
|
|
- this.visibleChunkMap.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
|
|
+ visibleChunks.values().stream().filter(ChunkHolder::wasAccessibleSinceLastSave).forEach((playerchunk) -> {
|
|
ChunkAccess ichunkaccess = (ChunkAccess) playerchunk.getChunkToSave().getNow(null); // CraftBukkit - decompile error
|
|
|
|
if (ichunkaccess instanceof ImposterProtoChunk || ichunkaccess instanceof LevelChunk) {
|
|
@@ -726,7 +797,20 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
if (!this.modified) {
|
|
return false;
|
|
} else {
|
|
- this.visibleChunkMap = this.updatingChunkMap.clone();
|
|
+ // Paper start - stop cloning visibleChunks
|
|
+ synchronized (this.visibleChunkMap) {
|
|
+ if (isIterating) {
|
|
+ hasPendingVisibleUpdate = true;
|
|
+ this.pendingVisibleChunks.copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
|
|
+ } else {
|
|
+ hasPendingVisibleUpdate = false;
|
|
+ this.pendingVisibleChunks.clear();
|
|
+ ((ProtectedVisibleChunksMap)this.visibleChunkMap).copyFrom((com.destroystokyo.paper.util.map.Long2ObjectLinkedOpenHashMapFastCopy<ChunkHolder>)this.updatingChunkMap);
|
|
+ this.visibleChunksClone = null;
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
+
|
|
this.modified = false;
|
|
return true;
|
|
}
|
|
@@ -1173,12 +1257,12 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
|
|
}
|
|
|
|
protected Iterable<ChunkHolder> getChunks() {
|
|
- return Iterables.unmodifiableIterable(this.visibleChunkMap.values());
|
|
+ return Iterables.unmodifiableIterable(this.getVisibleChunks().values()); // Paper
|
|
}
|
|
|
|
void dumpChunks(Writer writer) throws IOException {
|
|
CsvOutput csvwriter = CsvOutput.builder().addColumn("x").addColumn("z").addColumn("level").addColumn("in_memory").addColumn("status").addColumn("full_status").addColumn("accessible_ready").addColumn("ticking_ready").addColumn("entity_ticking_ready").addColumn("ticket").addColumn("spawning").addColumn("block_entity_count").build(writer);
|
|
- ObjectBidirectionalIterator objectbidirectionaliterator = this.visibleChunkMap.long2ObjectEntrySet().iterator();
|
|
+ ObjectBidirectionalIterator objectbidirectionaliterator = this.getVisibleChunks().long2ObjectEntrySet().iterator(); // Paper
|
|
|
|
while (objectbidirectionaliterator.hasNext()) {
|
|
Entry<ChunkHolder> entry = (Entry) objectbidirectionaliterator.next();
|
|
diff --git a/src/main/java/net/minecraft/server/level/ServerChunkCache.java b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
index b9516560183d37ef2917a3f9aaba60a53bcdfd72..b3c66e3f3ef47949f90efa023610ede55aabdf91 100644
|
|
--- a/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
+++ b/src/main/java/net/minecraft/server/level/ServerChunkCache.java
|
|
@@ -766,7 +766,7 @@ public class ServerChunkCache extends ChunkSource {
|
|
};
|
|
// Paper end
|
|
this.level.timings.chunkTicks.startTiming(); // Paper
|
|
- this.chunkMap.getChunks().forEach((playerchunk) -> { // Paper - no... just no...
|
|
+ this.chunkMap.forEachVisibleChunk((playerchunk) -> { // Paper - safe iterator incase chunk loads, also no wrapping
|
|
Optional<LevelChunk> optional = ((Either) playerchunk.getTickingChunkFuture().getNow(ChunkHolder.UNLOADED_LEVEL_CHUNK)).left();
|
|
|
|
if (optional.isPresent()) {
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
index b6134895d1b04d3ea7340e77f70efa23cff8b568..72c9ad9f75c20d6c1a6d54e2913e2f9918c11ffd 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
|
|
@@ -287,6 +287,7 @@ public class CraftWorld implements World {
|
|
|
|
@Override
|
|
public int getTileEntityCount() {
|
|
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
|
|
// We don't use the full world tile entity list, so we must iterate chunks
|
|
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
|
|
int size = 0;
|
|
@@ -298,6 +299,7 @@ public class CraftWorld implements World {
|
|
size += chunk.blockEntities.size();
|
|
}
|
|
return size;
|
|
+ });
|
|
}
|
|
|
|
@Override
|
|
@@ -307,6 +309,7 @@ public class CraftWorld implements World {
|
|
|
|
@Override
|
|
public int getChunkCount() {
|
|
+ return net.minecraft.server.MCUtil.ensureMain(() -> {
|
|
int ret = 0;
|
|
|
|
for (ChunkHolder chunkHolder : world.getChunkSource().chunkMap.visibleChunkMap.values()) {
|
|
@@ -315,7 +318,7 @@ public class CraftWorld implements World {
|
|
}
|
|
}
|
|
|
|
- return ret;
|
|
+ return ret; });
|
|
}
|
|
|
|
@Override
|
|
@@ -442,6 +445,14 @@ public class CraftWorld implements World {
|
|
|
|
@Override
|
|
public Chunk[] getLoadedChunks() {
|
|
+ // Paper start
|
|
+ if (Thread.currentThread() != world.getLevel().thread) {
|
|
+ synchronized (world.getChunkSource().chunkMap.visibleChunkMap) {
|
|
+ Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = world.getChunkSource().chunkMap.visibleChunkMap;
|
|
+ return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
+ }
|
|
+ }
|
|
+ // Paper end
|
|
Long2ObjectLinkedOpenHashMap<ChunkHolder> chunks = this.world.getChunkSource().chunkMap.visibleChunkMap;
|
|
return chunks.values().stream().map(ChunkHolder::getFullChunk).filter(Objects::nonNull).map(net.minecraft.world.level.chunk.LevelChunk::getBukkitChunk).toArray(Chunk[]::new);
|
|
}
|