Only unregister event handlers/listeners if the correct matching plugin is given

Currently, unregister will succeed & unregister a handler/listener if any valid plugin is given, not necessarily the one the handler was registered with.

proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java

@@ -161,17 +161,19 @@ public class VelocityEventManager implements EventManager {
   public void unregisterListener(Object plugin, Object listener) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(listener, "listener");
-    registeredListenersByPlugin.remove(plugin, listener);
+    if (registeredListenersByPlugin.remove(plugin, listener)) {
       methodAdapter.unregister(listener);
     }
+  }
 
   @Override
   public <E> void unregister(Object plugin, EventHandler<E> handler) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(handler, "listener");
-    registeredHandlersByPlugin.remove(plugin, handler);
+    if (registeredHandlersByPlugin.remove(plugin, handler)) {
       unregisterHandler(handler);
     }
+  }
 
   public boolean shutdown() throws InterruptedException {
     service.shutdown();