Only unregister event handlers/listeners if the correct matching plugin is given

Currently, unregister will succeed & unregister a handler/listener if any valid plugin is given, not necessarily the one the handler was registered with.

proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java

@@ -161,16 +161,18 @@ public class VelocityEventManager implements EventManager {
   public void unregisterListener(Object plugin, Object listener) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(listener, "listener");
-    registeredListenersByPlugin.remove(plugin, listener);
-    methodAdapter.unregister(listener);
+    if (registeredListenersByPlugin.remove(plugin, listener)) {
+      methodAdapter.unregister(listener);
+    }
   }
 
   @Override
   public <E> void unregister(Object plugin, EventHandler<E> handler) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(handler, "listener");
-    registeredHandlersByPlugin.remove(plugin, handler);
-    unregisterHandler(handler);
+    if (registeredHandlersByPlugin.remove(plugin, handler)) {
+      unregisterHandler(handler);
+    }
   }
 
   public boolean shutdown() throws InterruptedException {