Implemented scalable snapshot sizes

core/assets/bundles/bundle.properties

@@ -437,3 +437,6 @@ block.itemvoid.name=Item Void
 block.liquidsource.name=Liquid Source
 block.powervoid.name=Power Void
 block.powerinfinite.name=Power Infinite
+block.unloader.name=Unloader
+block.sortedunloader.name=Sorted Unloader
+block.vault.name=Vault

core/src/io/anuke/mindustry/Vars.java

@@ -12,9 +12,11 @@ import io.anuke.mindustry.entities.effect.Fire;
 import io.anuke.mindustry.entities.effect.ItemDrop;
 import io.anuke.mindustry.entities.effect.Puddle;
 import io.anuke.mindustry.entities.effect.Shield;
+import io.anuke.mindustry.entities.traits.SyncTrait;
 import io.anuke.mindustry.entities.units.BaseUnit;
 import io.anuke.mindustry.game.Team;
 import io.anuke.mindustry.io.Version;
+import io.anuke.mindustry.net.Net;
 import io.anuke.ucore.entities.Entities;
 import io.anuke.ucore.entities.EntityGroup;
 import io.anuke.ucore.entities.trait.DrawTrait;
@@ -158,12 +160,20 @@ public class Vars{
 		fireGroup = Entities.addGroup(Fire.class, false).enableMapping();
 		unitGroups = new EntityGroup[Team.all.length];
 
-		threads = new ThreadHandler(Platform.instance.getThreadProvider());
-
 		for(Team team : Team.all){
 			unitGroups[team.ordinal()] = Entities.addGroup(BaseUnit.class).enableMapping();
 		}
 
+		for(EntityGroup<?> group : Entities.getAllGroups()){
+			group.setRemoveListener(entity -> {
+				if(entity instanceof SyncTrait && Net.client()){
+					netClient.addRemovedEntity(((SyncTrait) entity).getID());
+				}
+			});
+		}
+
+		threads = new ThreadHandler(Platform.instance.getThreadProvider());
+
 		mobile = Gdx.app.getType() == ApplicationType.Android || Gdx.app.getType() == ApplicationType.iOS || testMobile;
 		ios = Gdx.app.getType() == ApplicationType.iOS;
 		android = Gdx.app.getType() == ApplicationType.Android;

core/src/io/anuke/mindustry/core/NetClient.java

@@ -2,6 +2,7 @@ package io.anuke.mindustry.core;
 
 import com.badlogic.gdx.graphics.Color;
 import com.badlogic.gdx.utils.Base64Coder;
+import com.badlogic.gdx.utils.IntSet;
 import com.badlogic.gdx.utils.Pools;
 import io.anuke.annotations.Annotations.Remote;
 import io.anuke.annotations.Annotations.Variant;
@@ -24,6 +25,7 @@ import io.anuke.ucore.io.ReusableByteArrayInputStream;
 import io.anuke.ucore.io.delta.DEZDecoder;
 import io.anuke.ucore.modules.Module;
 import io.anuke.ucore.util.Log;
+import io.anuke.ucore.util.Mathf;
 import io.anuke.ucore.util.Timer;
 
 import java.io.DataInputStream;
@@ -47,10 +49,20 @@ public class NetClient extends Module {
     private int lastSent;
     /**Last snapshot recieved.*/
     private byte[] lastSnapshot;
+    /**Current snapshot that is being built from chinks.*/
+    private byte[] currentSnapshot;
+    /**Array of recieved chunk statuses.*/
+    private boolean[] recievedChunks;
+    /**Counter of how many chunks have been recieved.*/
+    private int recievedChunkCounter;
+    /**ID of snapshot that is currently being constructed.*/
+    private int currentSnapshotID;
     /**Last snapshot ID recieved.*/
     private int lastSnapshotID = -1;
     /**Decoder for uncompressing snapshots.*/
     private DEZDecoder decoder = new DEZDecoder();
+    /**List of entities that were removed, and need not be added while syncing.*/
+    private IntSet removed = new IntSet();
     /**Byte stream for reading in snapshots.*/
     private ReusableByteArrayInputStream byteStream = new ReusableByteArrayInputStream();
     private DataInputStream dataStream = new DataInputStream(byteStream);
@@ -63,9 +75,15 @@ public class NetClient extends Module {
             player.isAdmin = false;
 
             Net.setClientLoaded(false);
+            removed.clear();
             timeoutTime = 0f;
             connecting = true;
             quiet = false;
+            lastSent = 0;
+            lastSnapshot = null;
+            currentSnapshot = null;
+            currentSnapshotID = 0;
+            lastSnapshotID = -1;
 
             ui.chatfrag.clearMessages();
             ui.loadfrag.hide();
@@ -160,6 +178,14 @@ public class NetClient extends Module {
         Net.disconnect();
     }
 
+    public synchronized void addRemovedEntity(int id){
+        removed.add(id);
+    }
+
+    public synchronized boolean isEntityUsed(int id){
+        return removed.contains(id);
+    }
+
     void sync(){
 
         if(timer.get(0, playerSyncTime)){
@@ -213,13 +239,47 @@ public class NetClient extends Module {
     }
 
     @Remote(variants = Variant.one, unreliable = true)
-    public static void onSnapshot(byte[] snapshot, int snapshotID){
+    public static void onSnapshot(byte[] chunk, int snapshotID, short chunkID, short totalLength){
         //skip snapshot IDs that have already been recieved
         if(snapshotID == netClient.lastSnapshotID){
             return;
         }
 
         try {
+            byte[] snapshot;
+
+            //total length exceeds that needed to hold one snapshot, therefore, it is split into chunks
+            if(totalLength > NetServer.maxSnapshotSize) {
+                //total amount of chunks to recieve
+                int totalChunks = Mathf.ceil((float) totalLength / NetServer.maxSnapshotSize);
+
+                //reset status when a new snapshot sending begins
+                if (netClient.currentSnapshotID != snapshotID) {
+                    netClient.currentSnapshotID = snapshotID;
+                    netClient.currentSnapshot = new byte[totalLength];
+                    netClient.recievedChunkCounter = 0;
+                    netClient.recievedChunks = new boolean[totalChunks];
+                }
+
+                //if this chunk hasn't been recieved yet...
+                if (!netClient.recievedChunks[chunkID]) {
+                    netClient.recievedChunks[chunkID] = true;
+                    netClient.recievedChunkCounter ++; //update recieved status
+                    //copy the recieved bytes into the holding array
+                    System.arraycopy(chunk, 0, netClient.currentSnapshot, chunkID * NetServer.maxSnapshotSize,
+                            Math.min(NetServer.maxSnapshotSize, totalLength - chunkID * NetServer.maxSnapshotSize));
+                }
+
+                //when all chunks have been recieved, begin
+                if(netClient.recievedChunkCounter >= totalChunks){
+                    snapshot = netClient.currentSnapshot;
+                }else{
+                    return;
+                }
+            }else{
+                snapshot = chunk;
+            }
+
             byte[] result;
             int length;
             if (snapshotID == 0) { //fresh snapshot
@@ -236,7 +296,7 @@ public class NetClient extends Module {
 
             netClient.lastSnapshotID = snapshotID;
 
-            //set stream bytes to begin write
+            //set stream bytes to begin snapshot reaeding
             netClient.byteStream.setBytes(result, 0, length);
 
             //get data input for reading from the stream
@@ -287,8 +347,9 @@ public class NetClient extends Module {
                         throw new RuntimeException("Error reading entity of type '"+ group.getType() + "': Read length mismatch [write=" + readLength + ", read=" + (netClient.byteStream.position() - position - 1)+ "]");
                     }
 
-                    if(add){
+                    if(add && !netClient.isEntityUsed(entity.getID())){
                         entity.add();
+                        netClient.addRemovedEntity(entity.getID());
                     }
                 }
             }

core/src/io/anuke/mindustry/core/NetServer.java

@@ -34,10 +34,14 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 
 import static io.anuke.mindustry.Vars.*;
 
 public class NetServer extends Module{
+    public final static int maxSnapshotSize = 2047;
+
+    private final static boolean showSnapshotSize = false;
     private final static float serverSyncTime = 4, kickDuration = 30 * 1000;
     private final static Vector2 vector = new Vector2();
     /**If a play goes away of their server-side coordinates by this distance, they get teleported back.*/
@@ -319,7 +323,7 @@ public class NetServer extends Module{
 
                 //if the player hasn't acknowledged that it has recieved the packet, send the same thing again
                 if(connection.lastSentSnapshotID > connection.lastSnapshotID){
-                    Call.onSnapshot(connection.id, connection.lastSentSnapshot, connection.lastSentSnapshotID);
+                    sendSplitSnapshot(connection.id, connection.lastSentSnapshot, connection.lastSentSnapshotID);
                     return;
                 }else{
                     //set up last confirmed snapshot to the last one that was sent, otherwise
@@ -394,13 +398,17 @@ public class NetServer extends Module{
                 byte[] bytes = syncStream.toByteArray();
                 connection.lastSentSnapshot = bytes;
                 if(connection.lastSnapshotID == -1){
+                    if(showSnapshotSize) Log.info("Sent raw snapshot: {0} bytes.", bytes.length);
                     //no snapshot to diff, send it all
-                    Call.onSnapshot(connection.id, bytes, 0);
+                    //Call.onSnapshot(connection.id, bytes, 0, 0);
+                    sendSplitSnapshot(connection.id, bytes, 0);
                     connection.lastSnapshotID = 0;
                 }else{
                     //send diff, otherwise
                     byte[] diff = ByteDeltaEncoder.toDiff(new ByteMatcherHash(connection.lastSnapshot, bytes), encoder);
-                    Call.onSnapshot(connection.id, diff, connection.lastSnapshotID + 1);
+                    if(showSnapshotSize) Log.info("Shrank snapshot: {0} -> {1}", bytes.length, diff.length);
+                    //Call.onSnapshot(connection.id, diff, connection.lastSnapshotID + 1, 0);
+                    sendSplitSnapshot(connection.id, diff, connection.lastSnapshotID + 1);
                     //increment snapshot ID
                     connection.lastSentSnapshotID ++;
                 }
@@ -411,6 +419,27 @@ public class NetServer extends Module{
         }
     }
 
+    /**Sends a raw byte[] snapshot to a client, splitting up into chunks when needed.*/
+    private static void sendSplitSnapshot(int userid, byte[] bytes, int snapshotID){
+        if(bytes.length < maxSnapshotSize){
+            Call.onSnapshot(userid, bytes, snapshotID, (short)0, (short)bytes.length);
+        }else{
+            int remaining = bytes.length;
+            int offset = 0;
+            int chunkid = 0;
+            while(remaining > 0){
+                int used = Math.min(remaining, maxSnapshotSize);
+                //TODO optimize to *not* copy the bytes directly, but instead re-use all arrays that are of length = maxSnapshotSize
+                byte[] toSend = Arrays.copyOfRange(bytes, offset, Math.min(offset + maxSnapshotSize, bytes.length));
+                Call.onSnapshot(userid, toSend, snapshotID, (short)chunkid, (short)bytes.length);
+
+                remaining -= used;
+                offset += used;
+                chunkid ++;
+            }
+        }
+    }
+
     private static void onDisconnect(Player player){
         Call.sendMessage("[accent]" + player.name + " has disconnected.");
         Call.onPlayerDisconnect(player.id);

core/src/io/anuke/mindustry/entities/units/UnitDrops.java

@@ -1,14 +1,20 @@
 package io.anuke.mindustry.entities.units;
 
+import io.anuke.mindustry.Vars;
 import io.anuke.mindustry.content.Items;
 import io.anuke.mindustry.gen.CallEntity;
 import io.anuke.mindustry.type.Item;
 import io.anuke.ucore.util.Mathf;
 
 public class UnitDrops {
+    private static final int maxItems = 200;
     private static Item[] dropTable;
 
     public static void dropItems(BaseUnit unit){
+        if(Vars.itemGroup.size() > maxItems){
+            return;
+        }
+
         if(dropTable == null){
             dropTable = new Item[]{Items.tungsten, Items.lead, Items.carbide};
         }

core/src/io/anuke/mindustry/type/Recipe.java

@@ -8,6 +8,7 @@ import io.anuke.mindustry.game.UnlockableContent;
 import io.anuke.mindustry.world.Block;
 import io.anuke.ucore.util.Bundles;
 import io.anuke.ucore.util.Log;
+import io.anuke.ucore.util.Strings;
 
 import java.util.Arrays;
 
@@ -59,7 +60,8 @@ public class Recipe implements UnlockableContent{
     @Override
     public void init() {
         if(!Bundles.has("block." + result.name + ".name")) {
-            Log.err("WARNING: Recipe block '{0}' does not have a formal name defined.", result.name);
+            Log.err("WARNING: Recipe block '{0}' does not have a formal name defined. Add the following to bundle.properties:", result.name);
+            Log.err("block.{0}.name={1}", result.name, Strings.capitalize(result.name.replace('-', '_')));
         }/*else if(result.fullDescription == null){
             Log.err("WARNING: Recipe block '{0}' does not have a description defined.", result.name);
         }*/

core/src/io/anuke/mindustry/type/Weapon.java

@@ -132,6 +132,7 @@ public class Weapon extends Upgrade {
 
 	@Remote(targets = Loc.server, called = Loc.both, in = In.entities, unreliable = true)
 	public static void onPlayerShootWeapon(Player player, float x, float y, float rotation, boolean left){
+		if(player == null) return;
 		//clients do not see their own shoot events: they are simulated completely clientside to prevent laggy visuals
 		//messing with the firerate or any other stats does not affect the server (take that, script kiddies!)
 		if(Net.client() && player == Vars.players[0]){
@@ -143,6 +144,7 @@ public class Weapon extends Upgrade {
 
     @Remote(targets = Loc.server, called = Loc.both, in = In.entities, unreliable = true)
     public static void onGenericShootWeapon(ShooterTrait shooter, float x, float y, float rotation, boolean left){
+		if(shooter == null) return;
         shootDirect(shooter, x, y, rotation, left);
     }
 

kryonet/src/io/anuke/kryonet/KryoClient.java

@@ -54,7 +54,7 @@ public class KryoClient implements ClientProvider{
             }
         };
 
-        client = new Client(8192, 2048, connection -> new ByteSerializer());
+        client = new Client(8192, 4096, connection -> new ByteSerializer());
         client.setDiscoveryHandler(handler);
 
         Listener listener = new Listener(){

kryonet/src/io/anuke/kryonet/KryoServer.java

@@ -52,7 +52,7 @@ public class KryoServer implements ServerProvider {
     int lastconnection = 0;
 
     public KryoServer(){
-        server = new Server(4096*2, 2048, connection -> new ByteSerializer());
+        server = new Server(4096*2, 4096, connection -> new ByteSerializer());
         server.setDiscoveryHandler((datagramChannel, fromAddress) -> {
             ByteBuffer buffer = NetworkIO.writeServerData();
             buffer.position(0);