Load chunks asynchronously for players.

When a player triggers a chunk load via walking around or teleporting there
is no need to stop everything and get this chunk on the main thread. The
client is used to having to wait some time for this chunk and the server
doesn't immediately do anything with it except send it to the player. At
the same time chunk loading is the last major source of file IO that still
runs on the main thread.

These two facts make it possible to offload chunks loaded for this reason
to another thread. However, not all parts of chunk loading can happen off
the main thread. For this we use the new AsynchronousExecutor system to
split chunk loading in to three pieces. The first is loading data from
disk, decompressing it, and parsing it in to an NBT structure.  The second
piece is creating entities and tile entities in the chunk and adding them
to the world, this is still done on the main thread. The third piece is
informing everyone who requested a chunk load that the load is finished.
For this we register callbacks and then run them on the main thread once
the previous two stages are finished.

There are still cases where a chunk is needed immediately and these will
still trigger chunk loading entirely on the main thread. The most obvious
case is plugins using the API to request a chunk load. We also must load
the chunk immediately when something in the world tries to access it. In
these cases we ignore any possibly pending or in progress chunk loading
that is happening asynchronously as we will have the chunk loaded by the
time they are finished.

The hope is that overall this system will result in less CPU time and
pauses due to blocking file IO on the main thread thus giving more
consistent performance. Testing so far has shown that this also speeds up
chunk loading client side although some of this is likely to be because
we are sending less chunks at once for the client to process.

Thanks for @ammaraskar for help with the implementation of this feature.

src/main/java/net/minecraft/server/ChunkProviderServer.java

@@ -11,6 +11,7 @@ import java.util.Set;
 import java.util.Random;
 
 import org.bukkit.Server;
+import org.bukkit.craftbukkit.chunkio.ChunkIOExecutor;
 import org.bukkit.craftbukkit.util.LongHash;
 import org.bukkit.craftbukkit.util.LongHashSet;
 import org.bukkit.craftbukkit.util.LongObjectHashMap;
@@ -79,11 +80,26 @@ public class ChunkProviderServer implements IChunkProvider {
         }
     }
 
+    // CraftBukkit start - add async variant, provide compatibility
     public Chunk getChunkAt(int i, int j) {
-        // CraftBukkit start
+        return getChunkAt(i, j, null);
+    }
+
+    public Chunk getChunkAt(int i, int j, Runnable runnable) {
         this.unloadQueue.remove(i, j);
         Chunk chunk = (Chunk) this.chunks.get(LongHash.toLong(i, j));
         boolean newChunk = false;
+        ChunkRegionLoader loader = null;
+
+        if (this.e instanceof ChunkRegionLoader) {
+            loader = (ChunkRegionLoader) this.e;
+        }
+
+        // If the chunk exists but isn't loaded do it async
+        if (chunk == null && runnable != null && loader != null && loader.chunkExists(this.world, i, j)) {
+            ChunkIOExecutor.queueChunkLoad(this.world, loader, this, i, j, runnable);
+            return null;
+        }
         // CraftBukkit end
 
         if (chunk == null) {
@@ -127,6 +143,12 @@ public class ChunkProviderServer implements IChunkProvider {
             chunk.a(this, this, i, j);
         }
 
+        // CraftBukkit start - If we didn't need to load the chunk run the callback now
+        if (runnable != null) {
+            runnable.run();
+        }
+        // CraftBukkit end
+
         return chunk;
     }
 

src/main/java/net/minecraft/server/ChunkRegionLoader.java

@@ -22,7 +22,39 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
         this.d = file1;
     }
 
+    // CraftBukkit start
+    public boolean chunkExists(World world, int i, int j) {
+        ChunkCoordIntPair chunkcoordintpair = new ChunkCoordIntPair(i, j);
+
+        synchronized (this.c) {
+            if (this.b.contains(chunkcoordintpair)) {
+                for (int k = 0; k < this.a.size(); ++k) {
+                    if (((PendingChunkToSave) this.a.get(k)).a.equals(chunkcoordintpair)) {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        return RegionFileCache.a(this.d, i, j).chunkExists(i & 31, j & 31);
+    }
+    // CraftBukkit end
+
+    // CraftBukkit start - add async variant, provide compatibility
     public Chunk a(World world, int i, int j) {
+        Object[] data = this.loadChunk(world, i, j);
+        if (data != null) {
+            Chunk chunk = (Chunk) data[0];
+            NBTTagCompound nbttagcompound = (NBTTagCompound) data[1];
+            this.loadEntities(chunk, nbttagcompound.getCompound("Level"), world);
+            return chunk;
+        }
+
+        return null;
+    }
+
+    public Object[] loadChunk(World world, int i, int j) {
+        // CraftBukkit end
         NBTTagCompound nbttagcompound = null;
         ChunkCoordIntPair chunkcoordintpair = new ChunkCoordIntPair(i, j);
         Object object = this.c;
@@ -51,7 +83,7 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
         return this.a(world, i, j, nbttagcompound);
     }
 
-    protected Chunk a(World world, int i, int j, NBTTagCompound nbttagcompound) {
+    protected Object[] a(World world, int i, int j, NBTTagCompound nbttagcompound) { // CraftBukkit - return Chunk -> Object[]
         if (!nbttagcompound.hasKey("Level")) {
             System.out.println("Chunk file at " + i + "," + j + " is missing level data, skipping");
             return null;
@@ -68,7 +100,12 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
                 chunk = this.a(world, nbttagcompound.getCompound("Level"));
             }
 
-            return chunk;
+            // CraftBukkit start
+            Object[] data = new Object[2];
+            data[0] = chunk;
+            data[1] = nbttagcompound;
+            return data;
+            // CraftBukkit end
         }
     }
 
@@ -271,6 +308,13 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
             chunk.a(nbttagcompound.getByteArray("Biomes"));
         }
 
+        // CraftBukkit start - end this method here and split off entity loading to another method
+        return chunk;
+    }
+
+    public void loadEntities(Chunk chunk, NBTTagCompound nbttagcompound, World world) {
+        // CraftBukkit end
+
         NBTTagList nbttaglist1 = nbttagcompound.getList("Entities");
 
         if (nbttaglist1 != null) {
@@ -310,6 +354,6 @@ public class ChunkRegionLoader implements IAsyncChunkSaver, IChunkLoader {
             }
         }
 
-        return chunk;
+        // return chunk; // CraftBukkit
     }
 }

src/main/java/net/minecraft/server/MinecraftServer.java

@@ -533,6 +533,8 @@ public abstract class MinecraftServer implements ICommandListener, Runnable, IMo
             processQueue.remove().run();
         }
 
+        org.bukkit.craftbukkit.chunkio.ChunkIOExecutor.tick();
+
         // Send timeupdates to everyone, it will get the right time from the world the player is in.
         if (this.ticks % 20 == 0) {
             for (int i = 0; i < this.getServerConfigurationManager().players.size(); ++i) {

src/main/java/net/minecraft/server/PlayerInstance.java

@@ -10,6 +10,7 @@ class PlayerInstance {
     private short[] dirtyBlocks;
     private int dirtyCount;
     private int f;
+    private boolean loaded = false; // CraftBukkit
 
     final PlayerManager playerManager;
 
@@ -19,15 +20,33 @@ class PlayerInstance {
         this.dirtyBlocks = new short[64];
         this.dirtyCount = 0;
         this.location = new ChunkCoordIntPair(i, j);
-        playermanager.a().chunkProviderServer.getChunkAt(i, j);
+        // CraftBukkit start
+        playermanager.a().chunkProviderServer.getChunkAt(i, j, new Runnable() {
+            public void run() {
+                PlayerInstance.this.loaded = true;
+            }
+        });
+        // CraftBukkit end
     }
 
-    public void a(EntityPlayer entityplayer) {
+    public void a(final EntityPlayer entityplayer) { // CraftBukkit - added final to argument
         if (this.b.contains(entityplayer)) {
             throw new IllegalStateException("Failed to add player. " + entityplayer + " already is in chunk " + this.location.x + ", " + this.location.z);
         } else {
             this.b.add(entityplayer);
+
+            // CraftBukkit start
+            if (this.loaded) {
                 entityplayer.chunkCoordIntPairQueue.add(this.location);
+            } else {
+                // Abuse getChunkAt to add another callback
+                this.playerManager.a().chunkProviderServer.getChunkAt(this.location.x, this.location.z, new Runnable() {
+                    public void run() {
+                        entityplayer.chunkCoordIntPairQueue.add(PlayerInstance.this.location);
+                    }
+                });
+            }
+            // CraftBukkit end
         }
     }
 

src/main/java/net/minecraft/server/PlayerManager.java

@@ -4,8 +4,10 @@ import java.util.ArrayList;
 import java.util.List;
 
 // CraftBukkit start
+import java.util.Collections;
 import java.util.Queue;
 import java.util.Iterator;
+import java.util.LinkedList;
 // CraftBukkit end
 
 public class PlayerManager {
@@ -98,12 +100,20 @@ public class PlayerManager {
         entityplayer.d = entityplayer.locX;
         entityplayer.e = entityplayer.locZ;
 
+        // CraftBukkit start - load nearby chunks first
+        List<ChunkCoordIntPair> chunkList = new LinkedList<ChunkCoordIntPair>();
         for (int k = i - this.e; k <= i + this.e; ++k) {
             for (int l = j - this.e; l <= j + this.e; ++l) {
-                this.a(k, l, true).a(entityplayer);
+                chunkList.add(new ChunkCoordIntPair(k, l));
             }
         }
 
+        Collections.sort(chunkList, new ChunkCoordComparator(entityplayer));
+        for (ChunkCoordIntPair pair : chunkList) {
+            this.a(pair.x, pair.z, true).a(entityplayer);
+        }
+        // CraftBukkit end
+
         this.managedPlayers.add(entityplayer);
         this.b(entityplayer);
     }
@@ -189,12 +199,13 @@ public class PlayerManager {
             int i1 = this.e;
             int j1 = i - k;
             int k1 = j - l;
+            List<ChunkCoordIntPair> chunksToLoad = new LinkedList<ChunkCoordIntPair>(); // CraftBukkit
 
             if (j1 != 0 || k1 != 0) {
                 for (int l1 = i - i1; l1 <= i + i1; ++l1) {
                     for (int i2 = j - i1; i2 <= j + i1; ++i2) {
                         if (!this.a(l1, i2, k, l, i1)) {
-                            this.a(l1, i2, true).a(entityplayer);
+                            chunksToLoad.add(new ChunkCoordIntPair(l1, i2)); // CraftBukkit
                         }
 
                         if (!this.a(l1 - j1, i2 - k1, i, j, i1)) {
@@ -212,16 +223,13 @@ public class PlayerManager {
                 entityplayer.e = entityplayer.locZ;
 
                 // CraftBukkit start - send nearest chunks first
-                if (i1 > 1 || i1 < -1 || j1 > 1 || j1 < -1) {
+                Collections.sort(chunksToLoad, new ChunkCoordComparator(entityplayer));
-                    final int x = i;
+                for (ChunkCoordIntPair pair : chunksToLoad) {
-                    final int z = j;
+                    this.a(pair.x, pair.z, true).a(entityplayer);
-                    List<ChunkCoordIntPair> chunksToSend = entityplayer.chunkCoordIntPairQueue;
-
-                    java.util.Collections.sort(chunksToSend, new java.util.Comparator<ChunkCoordIntPair>() {
-                        public int compare(ChunkCoordIntPair a, ChunkCoordIntPair b) {
-                            return Math.max(Math.abs(a.x - x), Math.abs(a.z - z)) - Math.max(Math.abs(b.x - x), Math.abs(b.z - z));
                 }
-                    });
+
+                if (i1 > 1 || i1 < -1 || j1 > 1 || j1 < -1) {
+                    Collections.sort(entityplayer.chunkCoordIntPairQueue, new ChunkCoordComparator(entityplayer));
                 }
                 // CraftBukkit end
             }
@@ -249,4 +257,47 @@ public class PlayerManager {
     static Queue c(PlayerManager playermanager) { // CraftBukkit List -> Queue
         return playermanager.d;
     }
+
+    // CraftBukkit start - sorter to load nearby chunks first
+    private static class ChunkCoordComparator implements java.util.Comparator<ChunkCoordIntPair> {
+        private int x;
+        private int z;
+
+        public ChunkCoordComparator (EntityPlayer entityplayer) {
+            x = (int) entityplayer.locX >> 4;
+            z = (int) entityplayer.locZ >> 4;
+        }
+
+        public int compare(ChunkCoordIntPair a, ChunkCoordIntPair b) {
+            if (a.equals(b)) {
+                return 0;
+            }
+
+            // Subtract current position to set center point
+            int ax = a.x - this.x;
+            int az = a.z - this.z;
+            int bx = b.x - this.x;
+            int bz = b.z - this.z;
+
+            int result = ((ax - bx) * (ax + bx)) + ((az - bz) * (az + bz));
+            if (result != 0) {
+                return result;
+            }
+
+            if (ax < 0) {
+                if (bx < 0) {
+                    return bz - az;
+                } else {
+                    return -1;
+                }
+            } else {
+                if (bx < 0) {
+                    return 1;
+                } else {
+                    return az - bz;
+                }
+            }
+        }
+    }
+    // CraftBukkit end
 }

src/main/java/net/minecraft/server/RegionFile.java

@@ -87,6 +87,45 @@ public class RegionFile {
         }
     }
 
+    // CraftBukkit start - this is a copy (sort of) of the method below it, make sure they stay in sync
+    public synchronized boolean chunkExists(int i, int j) {
+        if (this.d(i, j)) {
+            return false;
+        } else {
+            try {
+                int k = this.e(i, j);
+
+                if (k == 0) {
+                    return false;
+                } else {
+                    int l = k >> 8;
+                    int i1 = k & 255;
+
+                    if (l + i1 > this.f.size()) {
+                        return false;
+                    }
+
+                    this.c.seek((long) (l * 4096));
+                    int j1 = this.c.readInt();
+
+                    if (j1 > 4096 * i1 || j1 <= 0) {
+                        return false;
+                    }
+
+                    byte b0 = this.c.readByte();
+                    if (b0 == 1 || b0 == 2) {
+                        return true;
+                    }
+                }
+            } catch (IOException ioexception) {
+                return false;
+            }
+        }
+
+        return false;
+    }
+    // CraftBukkit end
+
     public synchronized DataInputStream a(int i, int j) {
         if (this.d(i, j)) {
             return null;
@@ -211,7 +250,7 @@ public class RegionFile {
         }
     }
 
-    private void a(int i, byte[] abyte, int j) {
+    private void a(int i, byte[] abyte, int j) throws IOException { // CraftBukkit - added throws
         this.c.seek((long) (i * 4096));
         this.c.writeInt(j + 1);
         this.c.writeByte(2);
@@ -230,19 +269,19 @@ public class RegionFile {
         return this.e(i, j) != 0;
     }
 
-    private void a(int i, int j, int k) {
+    private void a(int i, int j, int k) throws IOException { // CraftBukkit - added throws
         this.d[i + j * 32] = k;
         this.c.seek((long) ((i + j * 32) * 4));
         this.c.writeInt(k);
     }
 
-    private void b(int i, int j, int k) {
+    private void b(int i, int j, int k) throws IOException { // CraftBukkit - added throws
         this.e[i + j * 32] = k;
         this.c.seek((long) (4096 + (i + j * 32) * 4));
         this.c.writeInt(k);
     }
 
-    public void c() {
+    public void c() throws IOException { // CraftBukkit - added throws
         if (this.c != null) {
             this.c.close();
         }

src/main/java/net/minecraft/server/ServerConfigurationManagerAbstract.java

@@ -15,6 +15,7 @@ import java.util.logging.Logger;
 import org.bukkit.Location;
 import org.bukkit.craftbukkit.CraftServer;
 import org.bukkit.craftbukkit.CraftWorld;
+import org.bukkit.craftbukkit.chunkio.ChunkIOExecutor;
 import org.bukkit.entity.Player;
 import org.bukkit.event.player.PlayerChangedWorldEvent;
 import org.bukkit.event.player.PlayerPortalEvent;
@@ -149,6 +150,7 @@ public abstract class ServerConfigurationManagerAbstract {
         this.players.add(entityplayer);
         WorldServer worldserver = this.server.getWorldServer(entityplayer.dimension);
 
+        // CraftBukkit start
         PlayerJoinEvent playerJoinEvent = new PlayerJoinEvent(this.cserver.getPlayer(entityplayer), "\u00A7e" + entityplayer.name + " joined the game.");
         this.cserver.getPluginManager().callEvent(playerJoinEvent);
 
@@ -158,6 +160,8 @@ public abstract class ServerConfigurationManagerAbstract {
             this.server.getServerConfigurationManager().sendAll(new Packet3Chat(joinMessage));
         }
         this.cserver.onPlayerJoin(playerJoinEvent.getPlayer());
+
+        ChunkIOExecutor.adjustPoolSize(this.getPlayerCount());
         // CraftBukkit end
 
         // CraftBukkit start - only add if the player wasn't moved in the event
@@ -207,6 +211,7 @@ public abstract class ServerConfigurationManagerAbstract {
         worldserver.kill(entityplayer);
         worldserver.getPlayerManager().removePlayer(entityplayer);
         this.players.remove(entityplayer);
+        ChunkIOExecutor.adjustPoolSize(this.getPlayerCount()); // CraftBukkit
 
         // CraftBukkit start - .name -> .listName, replace sendAll with loop
         Packet201PlayerInfo packet = new Packet201PlayerInfo(entityplayer.listName, false, 9999);

src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOExecutor.java

@@ -0,0 +1,32 @@
+package org.bukkit.craftbukkit.chunkio;
+
+import net.minecraft.server.Chunk;
+import net.minecraft.server.ChunkProviderServer;
+import net.minecraft.server.ChunkRegionLoader;
+import net.minecraft.server.World;
+import org.bukkit.craftbukkit.util.AsynchronousExecutor;
+import org.bukkit.craftbukkit.util.LongHash;
+
+public class ChunkIOExecutor {
+    static final int BASE_THREADS = 1;
+    static final int PLAYERS_PER_THREAD = 50;
+
+    private static final AsynchronousExecutor<QueuedChunk, Chunk, Runnable, RuntimeException> instance = new AsynchronousExecutor<QueuedChunk, Chunk, Runnable, RuntimeException>(new ChunkIOProvider(), BASE_THREADS);
+
+    public static void waitForChunkLoad(World world, int x, int z) {
+        instance.get(new QueuedChunk(LongHash.toLong(x, z), null, world, null));
+    }
+
+    public static void queueChunkLoad(World world, ChunkRegionLoader loader, ChunkProviderServer provider, int x, int z, Runnable runnable) {
+        instance.add(new QueuedChunk(LongHash.toLong(x, z), loader, world, provider), runnable);
+    }
+
+    public static void adjustPoolSize(int players) {
+        int size = Math.max(BASE_THREADS, (int) Math.ceil(players / PLAYERS_PER_THREAD));
+        instance.setActiveThreads(size);
+    }
+
+    public static void tick() {
+        instance.finishActive();
+    }
+}

src/main/java/org/bukkit/craftbukkit/chunkio/ChunkIOProvider.java

@@ -0,0 +1,73 @@
+package org.bukkit.craftbukkit.chunkio;
+
+import net.minecraft.server.Chunk;
+import net.minecraft.server.ChunkRegionLoader;
+import net.minecraft.server.NBTTagCompound;
+
+import org.bukkit.Server;
+import org.bukkit.craftbukkit.util.AsynchronousExecutor;
+import org.bukkit.craftbukkit.util.LongHash;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+class ChunkIOProvider implements AsynchronousExecutor.CallBackProvider<QueuedChunk, Chunk, Runnable, RuntimeException> {
+    private final AtomicInteger threadNumber = new AtomicInteger(1);
+
+    // async stuff
+    public Chunk callStage1(QueuedChunk queuedChunk) throws RuntimeException {
+        ChunkRegionLoader loader = queuedChunk.loader;
+        Object[] data = loader.loadChunk(queuedChunk.world, LongHash.msw(queuedChunk.coords), LongHash.lsw(queuedChunk.coords));
+
+        if (data != null) {
+            queuedChunk.compound = (NBTTagCompound) data[1];
+            return (Chunk) data[0];
+        }
+
+        return null;
+    }
+
+    // sync stuff
+    public void callStage2(QueuedChunk queuedChunk, Chunk chunk) throws RuntimeException {
+        if(chunk == null) {
+            // If the chunk loading failed just do it synchronously (may generate)
+            queuedChunk.provider.getChunkAt(LongHash.msw(queuedChunk.coords), LongHash.lsw(queuedChunk.coords));
+            return;
+        }
+
+        int x = LongHash.msw(queuedChunk.coords);
+        int z = LongHash.lsw(queuedChunk.coords);
+
+        // See if someone already loaded this chunk while we were working on it (API, etc)
+        if (queuedChunk.provider.chunks.containsKey(queuedChunk.coords)) {
+            // Make sure it isn't queued for unload, we need it
+            queuedChunk.provider.unloadQueue.remove(queuedChunk.coords);
+            return;
+        }
+
+        queuedChunk.loader.loadEntities(chunk, queuedChunk.compound.getCompound("Level"), queuedChunk.world);
+        chunk.n = queuedChunk.provider.world.getTime();
+        queuedChunk.provider.chunks.put(queuedChunk.coords, chunk);
+        chunk.addEntities();
+
+        if (queuedChunk.provider.chunkProvider != null) {
+            queuedChunk.provider.chunkProvider.recreateStructures(x, z);
+        }
+
+        Server server = queuedChunk.provider.world.getServer();
+        if (server != null) {
+            server.getPluginManager().callEvent(new org.bukkit.event.world.ChunkLoadEvent(chunk.bukkitChunk, false));
+        }
+        
+        chunk.a(queuedChunk.provider, queuedChunk.provider, x, z);
+    }
+
+    public void callStage3(QueuedChunk queuedChunk, Chunk chunk, Runnable runnable) throws RuntimeException {
+        runnable.run();
+    }
+
+    public Thread newThread(Runnable runnable) {
+        Thread thread = new Thread(runnable, "Chunk I/O Executor Thread-" + threadNumber.getAndIncrement());
+        thread.setDaemon(true);
+        return thread;
+    }
+}

src/main/java/org/bukkit/craftbukkit/chunkio/QueuedChunk.java

@@ -0,0 +1,36 @@
+package org.bukkit.craftbukkit.chunkio;
+
+import net.minecraft.server.ChunkProviderServer;
+import net.minecraft.server.ChunkRegionLoader;
+import net.minecraft.server.NBTTagCompound;
+import net.minecraft.server.World;
+
+class QueuedChunk {
+    long coords;
+    ChunkRegionLoader loader;
+    World world;
+    ChunkProviderServer provider;
+    NBTTagCompound compound;
+
+    public QueuedChunk(long coords, ChunkRegionLoader loader, World world, ChunkProviderServer provider) {
+        this.coords = coords;
+        this.loader = loader;
+        this.world = world;
+        this.provider = provider;
+    }
+
+    @Override
+    public int hashCode() {
+        return (int) coords ^ world.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object instanceof QueuedChunk) {
+            QueuedChunk other = (QueuedChunk) object;
+            return coords == other.coords && world == other.world;
+        }
+
+        return false;
+    }
+}