Store item identifiers, make identifier files more compact via global table

api/src/main/java/com/viaversion/viaversion/api/data/FullMappings.java

@@ -27,7 +27,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
 /**
  * Mappings containing the full string identifier mappings.
  */
-public interface FullMappings extends Mappings {
+public interface FullMappings extends BiMappings {
 
     /**
      * Returns the unmapped integer id for the given identifier, or -1 if not found.

api/src/main/java/com/viaversion/viaversion/api/data/FullMappingsBase.java

@@ -22,6 +22,7 @@
  */
 package com.viaversion.viaversion.api.data;
 
+import com.google.common.base.Preconditions;
 import com.viaversion.viaversion.util.Key;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
@@ -37,6 +38,7 @@ public class FullMappingsBase implements FullMappings {
     private final Mappings mappings;
 
     public FullMappingsBase(final List<String> unmappedIdentifiers, final List<String> mappedIdentifiers, final Mappings mappings) {
+        Preconditions.checkNotNull(mappings, "Mappings cannot be null");
         this.mappings = mappings;
         this.stringToId = toInverseMap(unmappedIdentifiers);
         this.mappedStringToId = toInverseMap(mappedIdentifiers);

api/src/main/java/com/viaversion/viaversion/api/data/MappingData.java

@@ -89,8 +89,21 @@ public interface MappingData {
      */
     @Nullable List<TagData> getTags(RegistryType type);
 
+    /**
+     * Returns item mappings.
+     *
+     * @return item mappings
+     */
     @Nullable BiMappings getItemMappings();
 
+    /**
+     * Returns item mappings if they also have identifier data present.
+     *
+     * @return item mappings if they also have identifier data present
+     * @see #getItemMappings()
+     */
+    @Nullable FullMappings getFullItemMappings();
+
     @Nullable ParticleMappings getParticleMappings();
 
     @Nullable Mappings getBlockMappings();

api/src/main/java/com/viaversion/viaversion/api/data/MappingDataBase.java

@@ -24,8 +24,6 @@ package com.viaversion.viaversion.api.data;
 
 import com.github.steveice10.opennbt.tag.builtin.CompoundTag;
 import com.github.steveice10.opennbt.tag.builtin.IntArrayTag;
-import com.github.steveice10.opennbt.tag.builtin.ListTag;
-import com.github.steveice10.opennbt.tag.builtin.StringTag;
 import com.github.steveice10.opennbt.tag.builtin.Tag;
 import com.viaversion.viaversion.api.Via;
 import com.viaversion.viaversion.api.minecraft.RegistryType;
@@ -35,7 +33,6 @@ import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class MappingDataBase implements MappingData {
@@ -80,28 +77,29 @@ public class MappingDataBase implements MappingData {
         enchantmentMappings = loadMappings(data, "enchantments");
         paintingMappings = loadMappings(data, "paintings");
         attributeMappings = loadMappings(data, "attributes");
-        itemMappings = loadBiMappings(data, "items");
+
 
         final CompoundTag unmappedIdentifierData = readUnmappedIdentifiersFile("identifiers-" + unmappedVersion + ".nbt");
         final CompoundTag mappedIdentifierData = readMappedIdentifiersFile("identifiers-" + mappedVersion + ".nbt");
         if (unmappedIdentifierData != null && mappedIdentifierData != null) {
+            itemMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "items");
             entityMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "entities");
             argumentTypeMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "argumenttypes");
             recipeSerializerMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "recipe_serializers");
             itemDataSerializerMappings = loadFullMappings(data, unmappedIdentifierData, mappedIdentifierData, "data_component_type");
 
-            final ListTag<StringTag> unmappedParticles = unmappedIdentifierData.getListTag("particles", StringTag.class);
-            final ListTag<StringTag> mappedParticles = mappedIdentifierData.getListTag("particles", StringTag.class);
+            final List<String> unmappedParticles = identifiersFromGlobalIds(unmappedIdentifierData, "particles");
+            final List<String> mappedParticles = identifiersFromGlobalIds(mappedIdentifierData, "particles");
             if (unmappedParticles != null && mappedParticles != null) {
                 Mappings particleMappings = loadMappings(data, "particles");
                 if (particleMappings == null) {
                     particleMappings = new IdentityMappings(unmappedParticles.size(), mappedParticles.size());
                 }
-
-                final List<String> identifiers = unmappedParticles.stream().map(StringTag::getValue).collect(Collectors.toList());
-                final List<String> mappedIdentifiers = mappedParticles.stream().map(StringTag::getValue).collect(Collectors.toList());
-                this.particleMappings = new ParticleMappings(identifiers, mappedIdentifiers, particleMappings);
+                this.particleMappings = new ParticleMappings(unmappedParticles, mappedParticles, particleMappings);
             }
+        } else {
+            // Might not have identifiers in older versions
+            itemMappings = loadBiMappings(data, "items");
         }
 
         final CompoundTag tagsTag = data.getCompoundTag("tags");
@@ -114,6 +112,10 @@ public class MappingDataBase implements MappingData {
         loadExtras(data);
     }
 
+    protected @Nullable List<String> identifiersFromGlobalIds(final CompoundTag mappingsTag, final String key) {
+        return MappingDataLoader.INSTANCE.identifiersFromGlobalIds(mappingsTag, key);
+    }
+
     protected @Nullable CompoundTag readMappingsFile(final String name) {
         return MappingDataLoader.INSTANCE.loadNBT(name);
     }
@@ -194,6 +196,14 @@ public class MappingDataBase implements MappingData {
         return itemMappings;
     }
 
+    @Override
+    public @Nullable FullMappings getFullItemMappings() {
+        if (itemMappings instanceof FullMappings) {
+            return (FullMappings) itemMappings;
+        }
+        return null;
+    }
+
     @Override
     public @Nullable ParticleMappings getParticleMappings() {
         return particleMappings;

api/src/main/java/com/viaversion/viaversion/api/data/MappingDataLoader.java

@@ -28,6 +28,7 @@ import com.github.steveice10.opennbt.tag.builtin.IntArrayTag;
 import com.github.steveice10.opennbt.tag.builtin.IntTag;
 import com.github.steveice10.opennbt.tag.builtin.ListTag;
 import com.github.steveice10.opennbt.tag.builtin.StringTag;
+import com.github.steveice10.opennbt.tag.builtin.Tag;
 import com.github.steveice10.opennbt.tag.io.NBTIO;
 import com.github.steveice10.opennbt.tag.io.TagReader;
 import com.google.common.annotations.Beta;
@@ -40,13 +41,16 @@ import com.viaversion.viaversion.api.Via;
 import com.viaversion.viaversion.util.GsonUtil;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -54,14 +58,14 @@ import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class MappingDataLoader {
 
+    public static final MappingDataLoader INSTANCE = new MappingDataLoader(MappingDataLoader.class, "assets/viaversion/data/");
     public static final TagReader<CompoundTag> MAPPINGS_READER = NBTIO.reader(CompoundTag.class).named();
+    private static final Map<String, String[]> GLOBAL_IDENTIFIER_INDEXES = new HashMap<>();
     private static final byte DIRECT_ID = 0;
     private static final byte SHIFTS_ID = 1;
     private static final byte CHANGES_ID = 2;
     private static final byte IDENTITY_ID = 3;
 
-    public static final MappingDataLoader INSTANCE = new MappingDataLoader(MappingDataLoader.class, "assets/viaversion/data/");
-
     private final Map<String, CompoundTag> mappingsCache = new HashMap<>();
     private final Class<?> dataLoaderClass;
     private final String dataPath;
@@ -72,6 +76,40 @@ public class MappingDataLoader {
         this.dataPath = dataPath;
     }
 
+    public static void loadGlobalIdentifiers() {
+        // Load in a file with all the identifiers we need, so that we don't need to duplicate them
+        // for every single new version with only a couple of changes in them.
+        final CompoundTag globalIdentifiers = INSTANCE.loadNBT("identifier-table.nbt");
+        for (final Map.Entry<String, Tag> entry : globalIdentifiers.entrySet()) {
+            //noinspection unchecked
+            final ListTag<StringTag> value = (ListTag<StringTag>) entry.getValue();
+            final String[] array = new String[value.size()];
+            for (int i = 0, size = value.size(); i < size; i++) {
+                array[i] = value.get(i).getValue();
+            }
+            GLOBAL_IDENTIFIER_INDEXES.put(entry.getKey(), array);
+        }
+    }
+
+    /**
+     * Returns the global id of the identifier in the registry.
+     *
+     * @param registry registry key
+     * @param globalId global id
+     * @return identifier
+     * @throws IllegalArgumentException if the registry key is invalid
+     */
+    public @Nullable String identifierFromGlobalId(final String registry, final int globalId) {
+        final String[] array = GLOBAL_IDENTIFIER_INDEXES.get(registry);
+        if (array == null) {
+            throw new IllegalArgumentException("Unknown global identifier key: " + registry);
+        }
+        if (globalId < 0 || globalId >= array.length) {
+            throw new IllegalArgumentException("Unknown global identifier index: " + globalId);
+        }
+        return array[globalId];
+    }
+
     public void clearCache() {
         mappingsCache.clear();
         cacheValid = false;
@@ -146,7 +184,7 @@ public class MappingDataLoader {
             return null;
         }
 
-        try (final InputStream stream = resource) {
+        try (final InputStream stream = new BufferedInputStream(resource)) {
             return MAPPINGS_READER.read(stream);
         } catch (final IOException e) {
             throw new RuntimeException(e);
@@ -238,23 +276,32 @@ public class MappingDataLoader {
         return mappingsSupplier.create(mappings, mappedSizeTag.asInt());
     }
 
-    public FullMappings loadFullMappings(final CompoundTag mappingsTag, final CompoundTag unmappedIdentifiers, final CompoundTag mappedIdentifiers, final String key) {
-        final ListTag<StringTag> unmappedElements = unmappedIdentifiers.getListTag(key, StringTag.class);
-        final ListTag<StringTag> mappedElements = mappedIdentifiers.getListTag(key, StringTag.class);
-        if (unmappedElements == null || mappedElements == null) {
+    public FullMappings loadFullMappings(final CompoundTag mappingsTag, final CompoundTag unmappedIdentifiersTag, final CompoundTag mappedIdentifiersTag, final String key) {
+        if (!unmappedIdentifiersTag.contains(key) || !mappedIdentifiersTag.contains(key)) {
             return null;
         }
 
+        final List<String> unmappedIdentifiers = identifiersFromGlobalIds(unmappedIdentifiersTag, key);
+        final List<String> mappedIdentifiers = identifiersFromGlobalIds(mappedIdentifiersTag, key);
         Mappings mappings = loadMappings(mappingsTag, key);
         if (mappings == null) {
-            mappings = new IdentityMappings(unmappedElements.size(), mappedElements.size());
+            mappings = new IdentityMappings(unmappedIdentifiers.size(), mappedIdentifiers.size());
         }
 
-        return new FullMappingsBase(
-            unmappedElements.stream().map(StringTag::getValue).collect(Collectors.toList()),
-            mappedElements.stream().map(StringTag::getValue).collect(Collectors.toList()),
-            mappings
-        );
+        return new FullMappingsBase(unmappedIdentifiers, mappedIdentifiers, mappings);
+    }
+
+    public @Nullable List<String> identifiersFromGlobalIds(final CompoundTag mappingsTag, final String key) {
+        final Mappings mappings = loadMappings(mappingsTag, key);
+        if (mappings == null) {
+            return null;
+        }
+
+        final List<String> identifiers = new ArrayList<>(mappings.size());
+        for (int i = 0; i < mappings.size(); i++) {
+            identifiers.add(identifierFromGlobalId(key, mappings.getNewId(i)));
+        }
+        return identifiers;
     }
 
     /**

api/src/main/java/com/viaversion/viaversion/api/data/Mappings.java

@@ -84,10 +84,4 @@ public interface Mappings {
      * @return mappings with keys and values swapped
      */
     Mappings inverse();
-
-    @FunctionalInterface
-    interface MappingsSupplier<T extends Mappings> {
-
-        T supply(int[] mappings, int mappedIds);
-    }
 }

common/src/main/java/com/viaversion/viaversion/ViaManagerImpl.java

@@ -21,6 +21,7 @@ import com.viaversion.viaversion.api.Via;
 import com.viaversion.viaversion.api.ViaManager;
 import com.viaversion.viaversion.api.configuration.ConfigurationProvider;
 import com.viaversion.viaversion.api.connection.ConnectionManager;
+import com.viaversion.viaversion.api.data.MappingDataLoader;
 import com.viaversion.viaversion.api.debug.DebugHandler;
 import com.viaversion.viaversion.api.platform.PlatformTask;
 import com.viaversion.viaversion.api.platform.UnsupportedSoftware;
@@ -93,6 +94,8 @@ public class ViaManagerImpl implements ViaManager {
             loadServerProtocol();
         }
 
+        MappingDataLoader.loadGlobalIdentifiers();
+
         // Register protocols
         protocolManager.registerProtocols();
 

common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/data/MappingData.java

@@ -26,7 +26,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class MappingData extends MappingDataBase {
 
-    private KeyMappings items;
     private KeyMappings blocks;
     private KeyMappings sounds;
 
@@ -39,19 +38,10 @@ public class MappingData extends MappingDataBase {
         super.loadExtras(data);
 
         final CompoundTag extraMappings = MappingDataLoader.INSTANCE.loadNBT("extra-identifiers-1.20.3.nbt");
-        items = new KeyMappings(extraMappings.getListTag("items", StringTag.class));
         blocks = new KeyMappings(extraMappings.getListTag("blocks", StringTag.class));
         sounds = new KeyMappings(extraMappings.getListTag("sounds", StringTag.class));
     }
 
-    public int itemId(final String name) {
-        return items.keyToId(name);
-    }
-
-    public @Nullable String itemName(final int id) {
-        return items.idToKey(id);
-    }
-
     public int blockId(final String name) {
         return blocks.keyToId(name);
     }

common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/BlockItemPacketRewriter1_20_5.java

@@ -544,7 +544,7 @@ public final class BlockItemPacketRewriter1_20_5 extends ItemRewriter<Clientboun
     }
 
     private int unmappedItemId(final String name) {
-        return protocol.getMappingData().itemId(name);
+        return protocol.getMappingData().getFullItemMappings().id(name);
     }
 
     private int toMappedItemId(final String name) {

common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/ComponentRewriter1_20_5.java

@@ -185,7 +185,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
                 return;
             }
 
-            final int itemId = Protocol1_20_5To1_20_3.MAPPINGS.itemId(idTag.getValue());
+            final int itemId = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().id(idTag.getValue());
             if (itemId == -1) {
                 return;
             }
@@ -216,7 +216,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
                 return;
             }
 
-            final String itemName = Protocol1_20_5To1_20_3.MAPPINGS.itemName(newItem.identifier());
+            final String itemName = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(newItem.identifier());
             if (itemName != null) {
                 contentsTag.putString("id", itemName);
             }
@@ -606,7 +606,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
             final ArmorTrimMaterial armorTrimMaterial = material.value();
             final CompoundTag materialTag = new CompoundTag();
             materialTag.putString("asset_name", armorTrimMaterial.assetName());
-            final String ingredient = Protocol1_20_5To1_20_3.MAPPINGS.itemName(armorTrimMaterial.itemId());
+            final String ingredient = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(armorTrimMaterial.itemId());
             if (ingredient == null) {
                 throw new IllegalArgumentException("Unknown item: " + armorTrimMaterial.itemId());
             }
@@ -628,7 +628,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
             final ArmorTrimPattern armorTrimPattern = pattern.value();
             final CompoundTag patternTag = new CompoundTag();
             patternTag.put("asset_id", convertIdentifier(armorTrimPattern.assetName()));
-            final String templateItem = Protocol1_20_5To1_20_3.MAPPINGS.itemName(armorTrimPattern.itemId());
+            final String templateItem = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(armorTrimPattern.itemId());
             if (templateItem == null) {
                 throw new IllegalArgumentException("Unknown item: " + armorTrimPattern.itemId());
             }
@@ -769,7 +769,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
     protected ListTag<StringTag> convertPotDecorations(final PotDecorations value) {
         final ListTag<StringTag> tag = new ListTag<>(StringTag.class);
         for (final int decoration : value.itemIds()) {
-            final String item = Protocol1_20_5To1_20_3.MAPPINGS.itemName(decoration);
+            final String item = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(decoration);
             if (item == null) {
                 throw new IllegalArgumentException("Unknown item: " + decoration);
             }
@@ -877,7 +877,7 @@ public class ComponentRewriter1_20_5 extends ComponentRewriter<ClientboundPacket
     }
 
     protected void convertItem(final CompoundTag tag, final Item item) {
-        final String name = Protocol1_20_5To1_20_3.MAPPINGS.itemName(item.identifier());
+        final String name = Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(item.identifier());
         if (name == null) {
             throw new IllegalArgumentException("Unknown item: " + item.identifier());
         }

common/src/main/java/com/viaversion/viaversion/protocols/protocol1_20_5to1_20_3/rewriter/StructuredDataConverter.java

@@ -627,7 +627,7 @@ public final class StructuredDataConverter {
 
     private String toMappedItemName(final int id) {
         final int mappedId = unmappedItemId(id);
-        return mappedId != -1 ? Protocol1_20_5To1_20_3.MAPPINGS.itemName(mappedId) : "";
+        return mappedId != -1 ? Protocol1_20_5To1_20_3.MAPPINGS.getFullItemMappings().identifier(mappedId) : "";
     }
 
     private static CompoundTag getBlockEntityTag(final CompoundTag tag) {