diff --git a/proxy/src/main/java/com/velocitypowered/proxy/server/ServerMap.java b/proxy/src/main/java/com/velocitypowered/proxy/server/ServerMap.java
index 30835df3e..357c97e95 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/server/ServerMap.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/server/ServerMap.java
@@ -7,13 +7,10 @@ import com.velocitypowered.api.proxy.server.ServerInfo;
 import com.velocitypowered.proxy.VelocityServer;
 
 import java.util.*;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 public class ServerMap {
     private final VelocityServer server;
     private final Map<String, RegisteredServer> servers = new HashMap<>();
-    private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
     public ServerMap(VelocityServer server) {
         this.server = server;
@@ -22,47 +19,27 @@ public class ServerMap {
     public Optional<RegisteredServer> getServer(String name) {
         Preconditions.checkNotNull(name, "server");
         String lowerName = name.toLowerCase(Locale.US);
-        lock.readLock().lock();
-        try {
-            return Optional.ofNullable(servers.get(lowerName));
-        } finally {
-            lock.readLock().unlock();
-        }
+        return Optional.ofNullable(servers.get(lowerName));
     }
 
     public Collection<RegisteredServer> getAllServers() {
-        lock.readLock().lock();
-        try {
-            return ImmutableList.copyOf(servers.values());
-        } finally {
-            lock.readLock().unlock();
-        }
+        return ImmutableList.copyOf(servers.values());
     }
 
     public RegisteredServer register(ServerInfo serverInfo) {
         Preconditions.checkNotNull(serverInfo, "serverInfo");
         String lowerName = serverInfo.getName().toLowerCase(Locale.US);
-        lock.writeLock().lock();
-        try {
-            VelocityRegisteredServer rs = new VelocityRegisteredServer(server, serverInfo);
-            Preconditions.checkArgument(servers.putIfAbsent(lowerName, rs) == null, "Server with name %s already registered", serverInfo.getName());
-            return rs;
-        } finally {
-            lock.writeLock().unlock();
-        }
+        VelocityRegisteredServer rs = new VelocityRegisteredServer(server, serverInfo);
+        Preconditions.checkArgument(servers.putIfAbsent(lowerName, rs) == null, "Server with name %s already registered", serverInfo.getName());
+        return rs;
     }
 
     public void unregister(ServerInfo serverInfo) {
         Preconditions.checkNotNull(serverInfo, "serverInfo");
         String lowerName = serverInfo.getName().toLowerCase(Locale.US);
-        lock.writeLock().lock();
-        try {
-            RegisteredServer rs = servers.get(lowerName);
-            Preconditions.checkArgument(rs != null, "Server with name %s is not registered!", serverInfo.getName());
-            Preconditions.checkArgument(rs.getServerInfo().equals(serverInfo), "Trying to remove server %s with differing information", serverInfo.getName());
-            servers.remove(lowerName);
-        } finally {
-            lock.writeLock().unlock();
-        }
+        RegisteredServer rs = servers.get(lowerName);
+        Preconditions.checkArgument(rs != null, "Server with name %s is not registered!", serverInfo.getName());
+        Preconditions.checkArgument(rs.getServerInfo().equals(serverInfo), "Trying to remove server %s with differing information", serverInfo.getName());
+        Preconditions.checkState(servers.remove(lowerName, rs), "Server with name %s replaced whilst unregistering", serverInfo.getName());
     }
 }
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
index 18dfef0d0..b937fc926 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
@@ -25,6 +25,7 @@ import io.netty.handler.timeout.ReadTimeoutHandler;
 
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -36,8 +37,7 @@ import static com.velocitypowered.proxy.network.Connections.MINECRAFT_ENCODER;
 public class VelocityRegisteredServer implements RegisteredServer {
     private final VelocityServer server;
     private final ServerInfo serverInfo;
-    private final Set<ConnectedPlayer> players = new HashSet<>();
-    private final ReadWriteLock playersLock = new ReentrantReadWriteLock();
+    private final Set<ConnectedPlayer> players = ConcurrentHashMap.newKeySet();
 
     public VelocityRegisteredServer(VelocityServer server, ServerInfo serverInfo) {
         this.server = server;
@@ -51,12 +51,7 @@ public class VelocityRegisteredServer implements RegisteredServer {
 
     @Override
     public Collection<Player> getPlayersConnected() {
-        playersLock.readLock().lock();
-        try {
-            return ImmutableList.copyOf(players);
-        } finally {
-            playersLock.readLock().unlock();
-        }
+        return ImmutableList.copyOf(players);
     }
 
     @Override
@@ -94,43 +89,23 @@ public class VelocityRegisteredServer implements RegisteredServer {
     }
 
     public void addPlayer(ConnectedPlayer player) {
-        playersLock.writeLock().lock();
-        try {
-            players.add(player);
-        } finally {
-            playersLock.writeLock().unlock();
-        }
+        players.add(player);
     }
 
     public void removePlayer(ConnectedPlayer player) {
-        playersLock.writeLock().lock();
-        try {
-            players.remove(player);
-        } finally {
-            playersLock.writeLock().unlock();
-        }
+        players.remove(player);
     }
 
     @Override
     public boolean sendPluginMessage(ChannelIdentifier identifier, byte[] data) {
-        ServerConnection backendConnection = null;
-        playersLock.readLock().lock();
-        try {
-            for (ConnectedPlayer player : players) {
-                if (player.getConnectedServer() != null && player.getConnectedServer().getServerInfo().equals(serverInfo)) {
-                    backendConnection = player.getConnectedServer();
-                    break;
-                }
+        for (ConnectedPlayer player : players) {
+            if (player.getConnectedServer() != null && player.getConnectedServer().getServerInfo().equals(serverInfo)) {
+                ServerConnection connection = player.getConnectedServer();
+                return connection.sendPluginMessage(identifier, data);
             }
-
-            if (backendConnection == null) {
-                return false;
-            }
-        } finally {
-            playersLock.readLock().unlock();
         }
 
-        return backendConnection.sendPluginMessage(identifier, data);
+        return false;
     }
 
     @Override