Use a HashSet here as it is more appropriate for the situation.

proxy/src/main/java/com/velocitypowered/proxy/scheduler/VelocityScheduler.java

@@ -12,7 +12,7 @@ import com.velocitypowered.api.scheduler.TaskStatus;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicReference;
@@ -21,8 +21,8 @@ public class VelocityScheduler implements Scheduler {
     private final PluginManager pluginManager;
     private final ExecutorService taskService;
     private final ScheduledExecutorService timerExecutionService;
-    private final Multimap<Object, ScheduledTask> tasksByPlugin = Multimaps.synchronizedListMultimap(
-            Multimaps.newListMultimap(new IdentityHashMap<>(), ArrayList::new));
+    private final Multimap<Object, ScheduledTask> tasksByPlugin = Multimaps.synchronizedMultimap(
+            Multimaps.newSetMultimap(new IdentityHashMap<>(), HashSet::new));
 
     public VelocityScheduler(PluginManager pluginManager) {
         this.pluginManager = pluginManager;