If the channel becomes inactive, the connection's closed, too.

proxy/src/main/java/com/velocitypowered/proxy/connection/MinecraftConnection.java

@@ -70,6 +70,8 @@ public class MinecraftConnection extends ChannelInboundHandlerAdapter {
         if (association != null) {
             logger.info("{} has disconnected", association);
         }
+
+        teardown();
     }
 
     @Override