diff --git a/common/src/main/java/com/viaversion/viabackwards/protocol/v1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java b/common/src/main/java/com/viaversion/viabackwards/protocol/v1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java
index c7bcb0e3..3728c693 100644
--- a/common/src/main/java/com/viaversion/viabackwards/protocol/v1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java
+++ b/common/src/main/java/com/viaversion/viabackwards/protocol/v1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java
@@ -30,6 +30,7 @@ import com.viaversion.viaversion.api.connection.UserConnection;
 import com.viaversion.viaversion.api.minecraft.Holder;
 import com.viaversion.viaversion.api.minecraft.Particle;
 import com.viaversion.viaversion.api.minecraft.SoundEvent;
+import com.viaversion.viaversion.api.minecraft.data.StructuredData;
 import com.viaversion.viaversion.api.minecraft.data.StructuredDataContainer;
 import com.viaversion.viaversion.api.minecraft.data.StructuredDataKey;
 import com.viaversion.viaversion.api.minecraft.item.Item;
@@ -363,7 +364,15 @@ public final class BlockItemPacketRewriter1_20_5 extends BackwardsStructuredItem
             data.set(StructuredDataKey.FIREWORKS, new Fireworks(1, new FireworkExplosion[0]));
         }
 
+        final StructuredData<CompoundTag> customData = data.getNonEmpty(StructuredDataKey.CUSTOM_DATA);
         final Item oldItem = vvProtocol.getItemRewriter().toOldItem(connection, item, DATA_CONVERTER);
+
+        if (customData != null) {
+            // We later don't know which tags are custom data and which are not because the VV conversion
+            // keeps converted data, so we backup the original custom data and restore it later
+            oldItem.tag().put(nbtTagName(), customData.value().copy());
+        }
+
         if (oldItem.tag() != null && oldItem.tag().isEmpty()) {
             // Improve item equality checks by removing empty tags
             oldItem.setTag(null);
@@ -386,9 +395,14 @@ public final class BlockItemPacketRewriter1_20_5 extends BackwardsStructuredItem
         // Convert to structured item first
         final Item structuredItem = vvProtocol.getItemRewriter().toStructuredItem(connection, item);
 
+        if (item.tag() != null && item.tag().remove(nbtTagName()) instanceof final CompoundTag tag) {
+            // Set original custom data from backup
+            structuredItem.dataContainer().set(StructuredDataKey.CUSTOM_DATA, tag);
+        }
+
         structuredItem.dataContainer().setIdLookup(protocol, false);
         enchantmentRewriter.handleToServer(structuredItem);
 
         return super.handleItemToServer(connection, structuredItem);
     }
-}
\ No newline at end of file
+}
diff --git a/gradle.properties b/gradle.properties
index 655b90b3..ed0c5956 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,4 +1,4 @@
-projectVersion=5.0.3
+projectVersion=5.0.4-SNAPSHOT
 
 # Smile emoji
 mcVersions=1.21,1.20.6,1.20.5,1.20.4, 1.20.3, 1.20.2, 1.20.1, 1.20, 1.19.4, 1.19.3, 1.19.2, 1.19.1, 1.19, 1.18.2, 1.18.1, 1.18, 1.17.1, 1.17, 1.16.5, 1.16.4, 1.16.3, 1.16.2, 1.16.1, 1.16, 1.15.2, 1.15.1, 1.15, 1.14.4, 1.14.3, 1.14.2, 1.14.1, 1.14, 1.13.2, 1.13.1, 1.13, 1.12.2, 1.12.1, 1.12, 1.11.2, 1.11.1, 1.11, 1.10.2, 1.10.1, 1.10