diff --git a/proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java b/proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java
index a89f589a9..2ef08dade 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/plugin/VelocityEventManager.java
@@ -161,16 +161,18 @@ public class VelocityEventManager implements EventManager {
   public void unregisterListener(Object plugin, Object listener) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(listener, "listener");
-    registeredListenersByPlugin.remove(plugin, listener);
-    methodAdapter.unregister(listener);
+    if (registeredListenersByPlugin.remove(plugin, listener)) {
+      methodAdapter.unregister(listener);
+    }
   }
 
   @Override
   public <E> void unregister(Object plugin, EventHandler<E> handler) {
     ensurePlugin(plugin);
     Preconditions.checkNotNull(handler, "listener");
-    registeredHandlersByPlugin.remove(plugin, handler);
-    unregisterHandler(handler);
+    if (registeredHandlersByPlugin.remove(plugin, handler)) {
+      unregisterHandler(handler);
+    }
   }
 
   public boolean shutdown() throws InterruptedException {