diff --git a/paper-server/src/main/java/org/bukkit/craftbukkit/inventory/CraftMetaItem.java b/paper-server/src/main/java/org/bukkit/craftbukkit/inventory/CraftMetaItem.java
index eec5c087f7..cd12782b0b 100644
--- a/paper-server/src/main/java/org/bukkit/craftbukkit/inventory/CraftMetaItem.java
+++ b/paper-server/src/main/java/org/bukkit/craftbukkit/inventory/CraftMetaItem.java
@@ -619,7 +619,18 @@ class CraftMetaItem implements ItemMeta, Damageable, Repairable, BlockDataMeta {
             ByteArrayInputStream buf = new ByteArrayInputStream(Base64.getDecoder().decode(unhandled));
             try {
                 NBTTagCompound unhandledTag = NBTCompressedStreamTools.readCompressed(buf, NBTReadLimiter.unlimitedHeap());
-                unhandledTags.copy(DataComponentPatch.CODEC.parse(MinecraftServer.getDefaultRegistryAccess().createSerializationContext(DynamicOpsNBT.INSTANCE), unhandledTag).result().get());
+                DataComponentPatch unhandledPatch = DataComponentPatch.CODEC.parse(MinecraftServer.getDefaultRegistryAccess().createSerializationContext(DynamicOpsNBT.INSTANCE), unhandledTag).result().get();
+                this.unhandledTags.copy(unhandledPatch);
+
+                for (Entry<DataComponentType<?>, Optional<?>> entry : unhandledPatch.entrySet()) {
+                    // Move removed unhandled tags to dedicated removedTags
+                    if (!entry.getValue().isPresent()) {
+                        DataComponentType<?> key = entry.getKey();
+
+                        this.unhandledTags.clear(key);
+                        this.removedTags.add(key);
+                    }
+                }
             } catch (IOException ex) {
                 Logger.getLogger(CraftMetaItem.class.getName()).log(Level.SEVERE, null, ex);
             }