diff --git a/api/src/main/java/com/viaversion/viaversion/api/connection/StorableObject.java b/api/src/main/java/com/viaversion/viaversion/api/connection/StorableObject.java
index 13304a7d7..00a1ccb00 100644
--- a/api/src/main/java/com/viaversion/viaversion/api/connection/StorableObject.java
+++ b/api/src/main/java/com/viaversion/viaversion/api/connection/StorableObject.java
@@ -38,4 +38,7 @@ public interface StorableObject {
     default boolean clearOnServerSwitch() {
         return true;
     }
+
+    default void onRemove() {
+    }
 }
diff --git a/common/src/main/java/com/viaversion/viaversion/connection/ConnectionManagerImpl.java b/common/src/main/java/com/viaversion/viaversion/connection/ConnectionManagerImpl.java
index 4565da4cb..d06b5b6a1 100644
--- a/common/src/main/java/com/viaversion/viaversion/connection/ConnectionManagerImpl.java
+++ b/common/src/main/java/com/viaversion/viaversion/connection/ConnectionManagerImpl.java
@@ -72,6 +72,8 @@ public class ConnectionManagerImpl implements ConnectionManager {
             UUID id = connection.getProtocolInfo().getUuid();
             clients.remove(id);
         }
+
+        connection.clearStoredObjects();
     }
 
     @Override
diff --git a/common/src/main/java/com/viaversion/viaversion/connection/UserConnectionImpl.java b/common/src/main/java/com/viaversion/viaversion/connection/UserConnectionImpl.java
index 6485020fb..444d7feed 100644
--- a/common/src/main/java/com/viaversion/viaversion/connection/UserConnectionImpl.java
+++ b/common/src/main/java/com/viaversion/viaversion/connection/UserConnectionImpl.java
@@ -94,12 +94,19 @@ public class UserConnectionImpl implements UserConnection {
 
     @Override
     public <T extends StorableObject> @Nullable T remove(Class<T> objectClass) {
-        return (T) storedObjects.remove(objectClass);
+        final StorableObject object = storedObjects.remove(objectClass);
+        if (object != null) {
+            object.onRemove();
+        }
+        return (T) object;
     }
 
     @Override
     public void put(StorableObject object) {
-        storedObjects.put(object.getClass(), object);
+        final StorableObject previousObject = storedObjects.put(object.getClass(), object);
+        if (previousObject != null) {
+            previousObject.onRemove();
+        }
     }
 
     @Override
@@ -122,12 +129,21 @@ public class UserConnectionImpl implements UserConnection {
     @Override
     public void clearStoredObjects(boolean isServerSwitch) {
         if (isServerSwitch) {
-            storedObjects.values().removeIf(StorableObject::clearOnServerSwitch);
+            storedObjects.values().removeIf(storableObject -> {
+                if (storableObject.clearOnServerSwitch()) {
+                    storableObject.onRemove();
+                    return true;
+                }
+                return false;
+            });
             for (EntityTracker tracker : entityTrackers.values()) {
                 tracker.clearEntities();
                 tracker.trackClientEntity();
             }
         } else {
+            for (StorableObject object : storedObjects.values()) {
+                object.onRemove();
+            }
             storedObjects.clear();
             entityTrackers.clear();
         }
diff --git a/common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_2to1_20/storage/ConfigurationState.java b/common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_2to1_20/storage/ConfigurationState.java
index c17af1175..79e669499 100644
--- a/common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_2to1_20/storage/ConfigurationState.java
+++ b/common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_2to1_20/storage/ConfigurationState.java
@@ -47,7 +47,7 @@ public class ConfigurationState implements StorableObject {
         packetQueue.add(toQueuedPacket(wrapper, clientbound, false));
     }
 
-    private QueuedPacket toQueuedPacket(final PacketWrapper wrapper, final boolean clientbound, final boolean skip1_20_2Pipeline) throws Exception {
+    private QueuedPacket toQueuedPacket(final PacketWrapper wrapper, final boolean clientbound, final boolean skipCurrentPipeline) throws Exception {
         // Caching packet buffers is cursed, copy to heap buffers to make sure we don't start leaking in dumb cases
         final ByteBuf copy = Unpooled.buffer();
         final PacketType packetType = wrapper.getPacketType();
@@ -56,7 +56,7 @@ public class ConfigurationState implements StorableObject {
         //noinspection deprecation
         wrapper.setId(-1);
         wrapper.writeToBuffer(copy);
-        return new QueuedPacket(copy, clientbound, packetType, packetId, skip1_20_2Pipeline);
+        return new QueuedPacket(copy, clientbound, packetType, packetId, skipCurrentPipeline);
     }
 
     public void setJoinGamePacket(final PacketWrapper wrapper) throws Exception {
@@ -69,6 +69,16 @@ public class ConfigurationState implements StorableObject {
         return false; // This might be bad
     }
 
+    @Override
+    public void onRemove() {
+        for (final QueuedPacket packet : packetQueue) {
+            packet.buf().release();
+        }
+        if (joinGamePacket != null) {
+            joinGamePacket.buf().release();
+        }
+    }
+
     public void sendQueuedPackets(final UserConnection connection) throws Exception {
         if (joinGamePacket != null) {
             packetQueue.add(0, joinGamePacket);
@@ -89,9 +99,9 @@ public class ConfigurationState implements StorableObject {
                 }
 
                 if (packet.clientbound()) {
-                    queuedWrapper.send(Protocol1_20_2To1_20.class, packet.skip1_20_2Pipeline());
+                    queuedWrapper.send(Protocol1_20_2To1_20.class, packet.skipCurrentPipeline());
                 } else {
-                    queuedWrapper.sendToServer(Protocol1_20_2To1_20.class, packet.skip1_20_2Pipeline());
+                    queuedWrapper.sendToServer(Protocol1_20_2To1_20.class, packet.skipCurrentPipeline());
                 }
             } finally {
                 packet.buf().release();
@@ -118,15 +128,15 @@ public class ConfigurationState implements StorableObject {
         private final boolean clientbound;
         private final PacketType packetType;
         private final int packetId;
-        private final boolean skip1_20_2Pipeline;
+        private final boolean skipCurrentPipeline;
 
         private QueuedPacket(final ByteBuf buf, final boolean clientbound, final PacketType packetType,
-                             final int packetId, final boolean skip1_20_2Pipeline) {
+                             final int packetId, final boolean skipCurrentPipeline) {
             this.buf = buf;
             this.clientbound = clientbound;
             this.packetType = packetType;
             this.packetId = packetId;
-            this.skip1_20_2Pipeline = skip1_20_2Pipeline;
+            this.skipCurrentPipeline = skipCurrentPipeline;
         }
 
         public ByteBuf buf() {
@@ -145,8 +155,8 @@ public class ConfigurationState implements StorableObject {
             return packetType;
         }
 
-        public boolean skip1_20_2Pipeline() {
-            return skip1_20_2Pipeline;
+        public boolean skipCurrentPipeline() {
+            return skipCurrentPipeline;
         }
 
         @Override
@@ -156,7 +166,7 @@ public class ConfigurationState implements StorableObject {
                     ", clientbound=" + clientbound +
                     ", packetType=" + packetType +
                     ", packetId=" + packetId +
-                    ", skip1_20_2Pipeline=" + skip1_20_2Pipeline +
+                    ", skipCurrentPipeline=" + skipCurrentPipeline +
                     '}';
         }
     }