diff --git a/proxy/src/main/java/com/velocitypowered/proxy/connection/backend/VelocityServerConnection.java b/proxy/src/main/java/com/velocitypowered/proxy/connection/backend/VelocityServerConnection.java
index 51f80d285..c7fc697d8 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/connection/backend/VelocityServerConnection.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/connection/backend/VelocityServerConnection.java
@@ -80,7 +80,7 @@ public class VelocityServerConnection implements MinecraftConnectionAssociation,
             ch.pipeline()
                 .addLast(READ_TIMEOUT,
                     new ReadTimeoutHandler(server.getConfiguration().getReadTimeout(),
-                        TimeUnit.SECONDS))
+                        TimeUnit.MILLISECONDS))
                 .addLast(FRAME_DECODER, new MinecraftVarintFrameDecoder())
                 .addLast(FRAME_ENCODER, MinecraftVarintLengthEncoder.INSTANCE)
                 .addLast(MINECRAFT_DECODER,
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/network/ServerChannelInitializer.java b/proxy/src/main/java/com/velocitypowered/proxy/network/ServerChannelInitializer.java
index 9e167906f..f4083e311 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/network/ServerChannelInitializer.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/network/ServerChannelInitializer.java
@@ -39,7 +39,7 @@ public class ServerChannelInitializer extends ChannelInitializer<Channel> {
     ch.pipeline()
         .addLast(READ_TIMEOUT,
             new ReadTimeoutHandler(this.server.getConfiguration().getReadTimeout(),
-                TimeUnit.SECONDS))
+                TimeUnit.MILLISECONDS))
         .addLast(LEGACY_PING_DECODER, new LegacyPingDecoder())
         .addLast(FRAME_DECODER, new MinecraftVarintFrameDecoder())
         .addLast(LEGACY_PING_ENCODER, LegacyPingEncoder.INSTANCE)
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
index 4ea18802e..431c6e742 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/server/VelocityRegisteredServer.java
@@ -70,7 +70,7 @@ public class VelocityRegisteredServer implements RegisteredServer {
             ch.pipeline()
                 .addLast(READ_TIMEOUT,
                     new ReadTimeoutHandler(server.getConfiguration().getReadTimeout(),
-                        TimeUnit.SECONDS))
+                        TimeUnit.MILLISECONDS))
                 .addLast(FRAME_DECODER, new MinecraftVarintFrameDecoder())
                 .addLast(FRAME_ENCODER, MinecraftVarintLengthEncoder.INSTANCE)
                 .addLast(MINECRAFT_DECODER,