diff --git a/proxy/src/main/java/com/velocitypowered/proxy/VelocityServer.java b/proxy/src/main/java/com/velocitypowered/proxy/VelocityServer.java
index ba7a982e9..78b327cef 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/VelocityServer.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/VelocityServer.java
@@ -242,11 +242,6 @@ public class VelocityServer implements ProxyServer {
       return false;
     }
 
-    // If we have a new bind address, bind to it
-    if (!configuration.getBind().equals(newConfiguration.getBind())) {
-      this.cm.bind(newConfiguration.getBind());
-    }
-
     // Re-register servers
     for (Map.Entry<String, String> entry : newConfiguration.getServers().entrySet()) {
       ServerInfo newInfo =
@@ -259,6 +254,23 @@ public class VelocityServer implements ProxyServer {
       }
     }
 
+    // If we have a new bind address, bind to it
+    if (!configuration.getBind().equals(newConfiguration.getBind())) {
+      this.cm.bind(newConfiguration.getBind());
+      this.cm.shutdown(configuration.getBind());
+    }
+
+    if (configuration.isQueryEnabled() && (!newConfiguration.isQueryEnabled()
+        || newConfiguration.getQueryPort() != configuration.getQueryPort())) {
+      this.cm.shutdown(new InetSocketAddress(
+          configuration.getBind().getHostString(), configuration.getQueryPort()));
+    }
+
+    if (newConfiguration.isQueryEnabled()) {
+      this.cm.queryBind(newConfiguration.getBind().getHostString(),
+          newConfiguration.getQueryPort());
+    }
+
     ipAttemptLimiter = Ratelimiters.createWithMilliseconds(newConfiguration.getLoginRatelimit());
     this.configuration = newConfiguration;
     eventManager.fireAndForget(new ProxyReloadEvent());
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/network/ConnectionManager.java b/proxy/src/main/java/com/velocitypowered/proxy/network/ConnectionManager.java
index 92d5a657d..26296a2bc 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/network/ConnectionManager.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/network/ConnectionManager.java
@@ -1,5 +1,6 @@
 package com.velocitypowered.proxy.network;
 
+import com.google.common.base.Preconditions;
 import com.velocitypowered.natives.util.Natives;
 import com.velocitypowered.proxy.VelocityServer;
 import com.velocitypowered.proxy.protocol.netty.GS4QueryHandler;
@@ -11,7 +12,9 @@ import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.WriteBufferWaterMark;
 import java.net.InetSocketAddress;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -21,7 +24,7 @@ public final class ConnectionManager {
   private static final WriteBufferWaterMark SERVER_WRITE_MARK = new WriteBufferWaterMark(1 << 16,
       1 << 18);
   private static final Logger LOGGER = LogManager.getLogger(ConnectionManager.class);
-  private final Set<Channel> endpoints = new HashSet<>();
+  private final Map<InetSocketAddress, Channel> endpoints = new HashMap<>();
   private final TransportType transportType;
   private final EventLoopGroup bossGroup;
   private final EventLoopGroup workerGroup;
@@ -55,7 +58,7 @@ public final class ConnectionManager {
         .addListener((ChannelFutureListener) future -> {
           final Channel channel = future.channel();
           if (future.isSuccess()) {
-            this.endpoints.add(channel);
+            this.endpoints.put(address, channel);
             LOGGER.info("Listening on {}", channel.localAddress());
           } else {
             LOGGER.error("Can't bind to {}", address, future.cause());
@@ -64,16 +67,17 @@ public final class ConnectionManager {
   }
 
   public void queryBind(final String hostname, final int port) {
+    InetSocketAddress address = new InetSocketAddress(hostname, port);
     final Bootstrap bootstrap = new Bootstrap()
         .channel(this.transportType.datagramChannelClass)
         .group(this.workerGroup)
         .handler(new GS4QueryHandler(this.server))
-        .localAddress(hostname, port);
+        .localAddress(address);
     bootstrap.bind()
         .addListener((ChannelFutureListener) future -> {
           final Channel channel = future.channel();
           if (future.isSuccess()) {
-            this.endpoints.add(channel);
+            this.endpoints.put(address, channel);
             LOGGER.info("Listening for GS4 query on {}", channel.localAddress());
           } else {
             LOGGER.error("Can't bind to {}", bootstrap.config().localAddress(), future.cause());
@@ -90,8 +94,15 @@ public final class ConnectionManager {
             this.server.getConfiguration().getConnectTimeout());
   }
 
+  public void shutdown(InetSocketAddress oldBind) {
+    Channel serverChannel = endpoints.remove(oldBind);
+    Preconditions.checkState(serverChannel != null, "Endpoint %s not registered", oldBind);
+    LOGGER.info("Closing endpoint {}", serverChannel.localAddress());
+    serverChannel.close().syncUninterruptibly();
+  }
+
   public void shutdown() {
-    for (final Channel endpoint : this.endpoints) {
+    for (final Channel endpoint : this.endpoints.values()) {
       try {
         LOGGER.info("Closing endpoint {}", endpoint.localAddress());
         endpoint.close().sync();