Merge pull request 'GDPR query processor' (#271) from dsgvo-query into master

Reviewed-on: SteamWar/BungeeCore#271
Reviewed-by: YoyoNow <jwsteam@nidido.de>

src/GDPRQueryREADME.md

@@ -0,0 +1,29 @@
+# SteamWar GDPR report
+
+## Copyright notice
+The provided build worlds contain the world design of SteamWar contributors and is subject to their copyright.
+The build worlds are therefore provided for personal use only.
+
+## Categories of personal data processed
+- IP address
+- Minecraft account
+- E-Mail address (if using a website account)
+
+## Processing purposes of personal data and person-related data
+- Provision of SteamWar user functionality (Minecraft account, IP address, BuildWorlds, BuildInventories, BuildMembers, Elo, IgnoredPlayers, SchematicMembers, SchematicChecksessions, Schematics, PersonalKits, UserData, UserConfigs)
+- Manual analysis and punishment of player misbehaviour (BannedIPs, log files, Punishments)
+- Statistical analysis (Sessions, Fights, SchematicChecksessions)
+- Historical data storage (Fights)
+- Technical error analysis (log files)
+- Provision of the SteamWar website functionality (IP address, E-Mail address)
+
+## Data Accessors
+- SteamWar software
+- SteamWar Administration
+- SteamWar Development
+- SteamWar Moderation (limited to Punishments and SchematicChecksessions)
+
+## Storage duration
+- Data for provision of functionality is stored until user triggered deletion
+- Log file storage duration is targeted at one month, but might differ due to technical reasons
+- Historical and statistical data deletion is not intended

src/de/steamwar/bungeecore/BungeeCore.java

@@ -128,6 +128,7 @@ public class BungeeCore extends Plugin {
         new ListCommand();
         new StatCommand();
         new VerifyCommand();
+        new GDPRQuery();
 
         // Punishment Commands:
         new PunishmentCommand("ban", Punishment.PunishmentType.Ban);

src/de/steamwar/bungeecore/commands/GDPRQuery.java

@@ -0,0 +1,275 @@
+package de.steamwar.bungeecore.commands;
+
+import de.steamwar.bungeecore.BungeeCore;
+import de.steamwar.bungeecore.Message;
+import de.steamwar.bungeecore.sql.Statement;
+import de.steamwar.bungeecore.sql.SteamwarUser;
+import net.md_5.bungee.BungeeCord;
+import net.md_5.bungee.api.CommandSender;
+import net.md_5.bungee.api.connection.ProxiedPlayer;
+
+import java.io.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+public class GDPRQuery extends BasicCommand {
+
+    public GDPRQuery() {
+        super("gdprquery", "bungeecore.softreload");
+    }
+
+    @Override
+    public void execute(CommandSender sender, String[] args) {
+        if(!(sender instanceof ProxiedPlayer))
+            return;
+
+        ProxiedPlayer player = (ProxiedPlayer) sender;
+        SteamwarUser user = args.length == 0 ? SteamwarUser.get(player.getUniqueId()) : SteamwarUser.get(args[0]);
+        if(user == null) {
+            Message.send("UNKNOWN_PLAYER", player);
+            return;
+        }
+
+        BungeeCord.getInstance().getScheduler().runAsync(BungeeCore.get(), () -> {
+            try {
+                createZip(player, user);
+            } catch (IOException e) {
+                throw new SecurityException("Could not create zip", e);
+            }
+        });
+    }
+
+    private void createZip(ProxiedPlayer player, SteamwarUser user) throws IOException {
+        printUpdate(player, "GDPR_STATUS_WEBSITE");
+
+        ZipOutputStream out = new ZipOutputStream(new FileOutputStream(user.getUserName() + ".zip"));
+
+        copy(getClass().getClassLoader().getResourceAsStream("GDPRQueryREADME.md"), out, "README.md");
+        copy(getClass().getClassLoader().getResourceAsStream("GDPRQueryREADME.md"), out, "README.txt");
+
+        printUpdate(player, "GDPR_STATUS_WORLD");
+        copyBauwelt(user, out, "/home/minecraft/userworlds/" + user.getUuid().toString(), "BuildWorld12");
+        copyBauwelt(user, out, "/home/minecraft/userworlds15/" + user.getId(), "BuildWorld15");
+
+        printUpdate(player, "GDPR_STATUS_INVENTORIES");
+        copyPlayerdata(user, out, "/home/minecraft/userworlds", "BuildInventories12");
+        copyPlayerdata(user, out, "/home/minecraft/userworlds15", "BuildInventories15");
+
+        printUpdate(player, "GDPR_STATUS_DATABASE");
+        sqlCSV(user, out, bannedIPs, "BannedIPs.csv");
+        sqlCSV(user, out, bauweltMember, "BuildMember.csv");
+        sqlCSV(user, out, bauweltMembers, "BuildMembers.csv");
+        sqlCSV(user, out, checkedSchems, "SchematicChecksessions.csv");
+        sqlCSV(user, out, elo, "Elo.csv");
+        sqlCSV(user, out, fights, "Fights.csv");
+        sqlCSV(user, out, ignoredPlayers, "IgnoredPlayers.csv");
+        sqlCSV(user, out, ignoringPlayers, "IgnoringPlayers.csv");
+        sqlCSV(user, out, schematicMember, "SchematicMember.csv");
+        sqlCSV(user, out, schematicMembers, "SchematicMembers.csv");
+        sqlCSV(user, out, pollAnswers, "PollAnswers.csv");
+        sqlCSV(user, out, punishments, "Punishments.csv");
+        sqlCSV(user, out, sessions, "Sessions.csv");
+        sqlCSV(user, out, userData, "UserData.csv");
+        sqlCSV(user, out, personalKits, "PersonalKits.csv");
+        sqlCSV(user, out, schematics, "Schematics.csv");
+
+        personalKits(user, out);
+        schematics(user, out);
+        userConfig(user, out);
+
+        printUpdate(player, "GDPR_STATUS_LOGS");
+        copyLogs(user, out, new File("/logs"), "logs");
+
+        out.close();
+        printUpdate(player, "GDPR_STATUS_FINISHED");
+    }
+
+    private static final Statement bannedIPs = new Statement("SELECT Timestamp, IP FROM BannedUserIPs WHERE UserID = ?");
+    private static final Statement bauweltMember = new Statement("SELECT BauweltID AS Bauwelt, WorldEdit, World FROM BauweltMember WHERE MemberID = ?");
+    private static final Statement bauweltMembers = new Statement("SELECT u.UserName AS 'User', m.WorldEdit AS WorldEdit, m.World AS World FROM BauweltMember m INNER JOIN UserData u ON m.MemberID = u.id WHERE m.BauweltID = ?");
+    private static final Statement checkedSchems = new Statement("SELECT NodeName AS Schematic, StartTime, EndTime, DeclineReason AS Result FROM CheckedSchematic WHERE NodeOwner = ? ORDER BY StartTime ASC");
+    private static final Statement elo = new Statement("SELECT GameMode, Elo FROM Elo WHERE UserID = ?");
+    private static final Statement fights = new Statement("SELECT p.Team AS Team, p.Kit AS Kit, p.Kills AS Kills, p.IsOut AS Died, f.GameMode AS GameMode, f.Server AS Server, f.Arena AS Arena, f.StartTime AS StartTime, f.Duration AS Duration, (f.BlueLeader = p.UserID) AS IsBlueLeader, (f.RedLeader = p.UserID) AS IsRedLeader, f.Win AS Winner, f.WinCondition AS WinCondition FROM Fight f INNER JOIN FightPlayer p ON f.FightID = p.FightID WHERE p.UserID = ? ORDER BY StartTime ASC");
+    private static final Statement ignoredPlayers = new Statement("SELECT u.UserName AS IgnoredPlayer FROM IgnoredPlayers i INNER JOIN UserData u ON i.Ignored = u.id WHERE Ignorer = ?");
+    private static final Statement ignoringPlayers = new Statement("SELECT Ignorer AS IgnoringPlayers FROM IgnoredPlayers WHERE Ignored = ?");
+    private static final Statement schematicMember = new Statement("SELECT s.NodeName AS SchematicName, u.UserName AS SchematicOwner FROM NodeMember m INNER JOIN SchematicNode s ON m.NodeId = s.NodeId INNER JOIN UserData u ON s.NodeOwner = u.id WHERE m.UserId = ?");
+    private static final Statement schematicMembers = new Statement("SELECT s.NodeName AS SchematicName, u.UserName AS Member FROM NodeMember m INNER JOIN SchematicNode s ON m.NodeId = s.NodeId INNER JOIN UserData u ON m.UserId = u.id WHERE s.NodeOwner = ?");
+    private static final Statement pollAnswers = new Statement("SELECT Question, Answer FROM PollAnswer WHERE UserID = ?");
+    private static final Statement punishments = new Statement("SELECT Type, StartTime, EndTime, Perma, Reason FROM Punishments WHERE UserId = ?");
+    private static final Statement sessions = new Statement("SELECT StartTime, EndTime FROM Session WHERE UserID = ?");
+    private static final Statement userData = new Statement("SELECT * FROM UserData WHERE id = ?");
+    private static final Statement personalKits = new Statement("SELECT GameMode, Name, InUse FROM PersonalKit WHERE UserID = ?");
+    private static final Statement personalKitData = new Statement("SELECT GameMode, Name, Inventory, Armor FROM PersonalKit WHERE UserID = ?");
+    private static final Statement schematics = new Statement("SELECT NodeName AS SchematicName, ParentNode, LastUpdate, NodeItem, NodeType, NodeRank FROM SchematicNode WHERE NodeOwner = ?");
+    private static final Statement schematicData = new Statement("SELECT NodeName, ParentNode, NodeFormat, NodeData FROM SchematicNode WHERE NodeOwner = ?");
+    private static final Statement userConfig = new Statement("SELECT * FROM UserConfig WHERE User = ?");
+
+    private void sqlCSV(SteamwarUser user, ZipOutputStream out, Statement statement, String path) throws IOException {
+        write(stream -> statement.select(rs -> {
+            try {
+                OutputStreamWriter writer = new OutputStreamWriter(stream);
+                int columns = rs.getMetaData().getColumnCount();
+
+                for(int i = 1; i <= columns; i++) {
+                    writer.write(rs.getMetaData().getColumnLabel(i));
+                    writer.write(";");
+                }
+                writer.write("\n");
+
+                while(rs.next()) {
+                    for(int i = 1; i <= columns; i++) {
+                        try {
+                            writer.write(rs.getString(i));
+                        } catch (NullPointerException e) {
+                            // ignored
+                        }
+                        writer.write(";");
+                    }
+                    writer.write("\n");
+                }
+                writer.flush();
+            } catch (IOException e) {
+                throw new SecurityException("Could not write file", e);
+            }
+            return null;
+        }, user.getId()), out, path);
+    }
+
+    private void personalKits(SteamwarUser user, ZipOutputStream out) {
+        personalKitData.select(rs -> {
+            while(rs.next()) {
+                try {
+                    String path = "PersonalKit/" + rs.getString("GameMode") + "/" + rs.getString("Name");
+                    try(InputStream data = rs.getBinaryStream("Inventory")) {
+                        copy(data, out, path + ".Inventory.yml");
+                    }
+                    try(InputStream data = rs.getBinaryStream("Armor")) {
+                        copy(data, out, path + ".Armor.yml");
+                    }
+                } catch (IOException e) {
+                    throw new SecurityException("Could not export PersonalKits", e);
+                }
+            }
+            return null;
+        }, user.getId());
+    }
+
+    private void schematics(SteamwarUser user, ZipOutputStream out) {
+        schematicData.select(rs -> {
+            while(rs.next()) {
+                String name = (rs.getString("ParentNode") != null ? rs.getString("ParentNode") : "") + ":" + rs.getString("NodeName");
+                boolean format = rs.getBoolean("NodeFormat");
+                try(InputStream data = rs.getBinaryStream("NodeData")) {
+                    copy(data, out, "Schematics/" + name + (format ? ".schem" : ".schematic"));
+                } catch (IOException e) {
+                    throw new SecurityException("Could not export Schematic", e);
+                }
+            }
+            return null;
+        }, user.getId());
+    }
+
+    private void userConfig(SteamwarUser user, ZipOutputStream out) {
+        userConfig.select(rs -> {
+            while(rs.next()) {
+                String name = rs.getString("Config");
+                try(InputStream data = rs.getBinaryStream("Value")) {
+                    copy(data, out, name + ".yapion");
+                } catch (IOException e) {
+                    throw new SecurityException("Could not export UserConfig", e);
+                }
+            }
+            return null;
+        }, user.getId());
+    }
+
+    private void copyLogs(SteamwarUser user, ZipOutputStream out, File log, String outFile) throws IOException {
+        if (log.isDirectory()) {
+            for(File logfile : log.listFiles()) {
+                copyLogs(user, out, logfile, outFile + "/" + logfile.getName().replace(".gz", ""));
+            }
+        } else {
+            Process reader = new ProcessBuilder("zgrep", "^.*" + user.getUserName() + "\\( issued server command:\\| moved too quickly!\\| executed command:\\| lost connection:\\||\\|»\\|\\[\\|\\]\\).*$", log.getPath()).start();
+            copy(reader.getInputStream(), out, outFile);
+        }
+    }
+
+    private void copyBauwelt(SteamwarUser user, ZipOutputStream out, String inDir, String outDir) throws IOException {
+        File world = new File(inDir);
+        if(!world.exists())
+            return;
+
+        copy(new File(world, "level.dat"), out, outDir + "/level.dat");
+
+        File region = new File(world, "region");
+        for(File regionfile : region.listFiles()) {
+            copy(regionfile, out, outDir + "/region/" + regionfile.getName());
+        }
+
+        File poi = new File(world, "poi");
+        if(poi.exists()) {
+            for(File regionfile : poi.listFiles()) {
+                copy(regionfile, out, outDir + "/poi/" + regionfile.getName());
+            }
+        }
+
+        File playerdata = new File(world, "playerdata/" + user.getUuid().toString() + ".dat");
+        if(playerdata.exists())
+            copy(playerdata, out, outDir + "/playerdata/" + user.getUuid().toString() + ".dat");
+    }
+
+    private void copyPlayerdata(SteamwarUser user, ZipOutputStream out, String inDir, String outDir) throws IOException {
+        File worlds = new File(inDir);
+        String path = "playerdata/" + user.getUuid().toString() + ".dat";
+
+        int i = 0;
+        for(File world : worlds.listFiles()) {
+            File playerdata = new File(world, path);
+            if(!playerdata.exists())
+                continue;
+
+            copy(playerdata, out, outDir + "/" + (i++) + "/" + user.getUuid().toString() + ".dat");
+        }
+    }
+
+    private void printUpdate(ProxiedPlayer player, String message) {
+        if (player.isConnected())
+            Message.send(message, player);
+    }
+
+    private void copy(File file, ZipOutputStream out, String path) throws IOException {
+        try(FileInputStream in = new FileInputStream(file)) {
+            copy(in, out, path);
+        }
+    }
+
+    private void copy(InputStream in, ZipOutputStream out, String path) throws IOException {
+        boolean initialized = false;
+
+        int bytes;
+        for(byte[] buf = new byte[8192]; (bytes = in.read(buf)) > 0; ) {
+            if(!initialized) {
+                ZipEntry entry = new ZipEntry(path);
+                out.putNextEntry(entry);
+                initialized = true;
+            }
+
+            out.write(buf, 0, bytes);
+        }
+
+        if(initialized) {
+            out.closeEntry();
+        }
+    }
+
+    private void write(Writer writer, ZipOutputStream out, String path) throws IOException {
+        ZipEntry entry = new ZipEntry(path);
+        out.putNextEntry(entry);
+        writer.accept(out);
+        out.closeEntry();
+    }
+
+    private interface Writer {
+        void accept(OutputStream stream) throws IOException;
+    }
+}

src/de/steamwar/bungeecore/sql/Statement.java

@@ -88,7 +88,7 @@ public class Statement {
     private final String sql;
     private PreparedStatement st;
 
-    Statement(String sql) {
+    public Statement(String sql) {
         this.sql = sql;
         statements.add(this);
         try {
@@ -102,7 +102,7 @@ public class Statement {
         st = con.prepareStatement(sql);
     }
 
-    <T> T select(ResultSetUser<T> user, Object... objects) {
+    public <T> T select(ResultSetUser<T> user, Object... objects) {
         synchronized (statements) {
             return prepare(() -> {
                 ResultSet rs = st.executeQuery();
@@ -113,7 +113,7 @@ public class Statement {
         }
     }
 
-    void update(Object... objects) {
+    public void update(Object... objects) {
         synchronized (statements) {
             prepare(st::executeUpdate, objects);
         }
@@ -135,7 +135,7 @@ public class Statement {
         }
     }
 
-    interface ResultSetUser<T> {
+    public interface ResultSetUser<T> {
         T use(ResultSet rs) throws SQLException;
     }
 

src/de/steamwar/messages/BungeeCore.properties

@@ -549,4 +549,12 @@ VERIFY_SUCCESS=§7Erfolgreich mit dem Discord Account §e{0} §7verknüpft
 #Discord
 DISCORD_TICKET_MESSAGE=§7Ticket §e{0}§7» §f§l{1}: §7{2}
 DISCORD_TICKET_NEW=§7Ticket §e{0}§7» §aTicket wurde geöffnet!
-DISCORD_TICKET_CLOSED=§7Ticket §e{0}§7» §cTicket wurde geschlossen!
+DISCORD_TICKET_CLOSED=§7Ticket §e{0}§7» §cTicket wurde geschlossen!
+
+#GDPR Query
+GDPR_STATUS_WEBSITE=§7Website kann nicht automatisiert gepackt werden und muss daher manuell hinzugefügt werden.
+GDPR_STATUS_WORLD=§7Packe Bauwelten...
+GDPR_STATUS_INVENTORIES=§7Suche und packe Inventare...
+GDPR_STATUS_DATABASE=§7Packe Datenbankinhalte...
+GDPR_STATUS_LOGS=§7Suche und packe logs...
+GDPR_STATUS_FINISHED=§7Packen abgeschlossen