diff --git a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
index bec7dab41..5987311e1 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
@@ -32,7 +32,8 @@ import io.netty.channel.ChannelInitializer;
 import io.netty.channel.EventLoop;
 import io.netty.handler.timeout.ReadTimeoutHandler;
 import java.util.Collection;
-import java.util.Set;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -45,7 +46,7 @@ public class VelocityRegisteredServer implements RegisteredServer, ForwardingAud
 
   private final @Nullable VelocityServer server;
   private final ServerInfo serverInfo;
-  private final Set<ConnectedPlayer> players = ConcurrentHashMap.newKeySet();
+  private final Map<UUID, ConnectedPlayer> players = new ConcurrentHashMap<>();
 
   public VelocityRegisteredServer(@Nullable VelocityServer server, ServerInfo serverInfo) {
     this.server = server;
@@ -59,7 +60,7 @@ public class VelocityRegisteredServer implements RegisteredServer, ForwardingAud
 
   @Override
   public Collection<Player> getPlayersConnected() {
-    return ImmutableList.copyOf(players);
+    return ImmutableList.copyOf(players.values());
   }
 
   @Override
@@ -111,11 +112,11 @@ public class VelocityRegisteredServer implements RegisteredServer, ForwardingAud
   }
 
   public void addPlayer(ConnectedPlayer player) {
-    players.add(player);
+    players.put(player.getUniqueId(), player);
   }
 
   public void removePlayer(ConnectedPlayer player) {
-    players.remove(player);
+    players.remove(player.getUniqueId(), player);
   }
 
   @Override
@@ -130,7 +131,7 @@ public class VelocityRegisteredServer implements RegisteredServer, ForwardingAud
    * @return whether or not the message was sent
    */
   public boolean sendPluginMessage(ChannelIdentifier identifier, ByteBuf data) {
-    for (ConnectedPlayer player : players) {
+    for (ConnectedPlayer player : players.values()) {
       VelocityServerConnection connection = player.getConnectedServer();
       if (connection != null && connection.getServerInfo().equals(serverInfo)) {
         return connection.sendPluginMessage(identifier, data);