Prepared sync pipeline for content-based config sync

.gitignore

@@ -27,5 +27,6 @@ bin/
 # fabric
 
 run/
+run-client/
 output/
 *.log

src/main/java/ru/bclib/api/dataexchange/DataExchangeAPI.java

@@ -4,6 +4,7 @@ import com.google.common.collect.Lists;
 import net.fabricmc.api.EnvType;
 import net.fabricmc.api.Environment;
 import net.minecraft.network.FriendlyByteBuf;
+import ru.bclib.api.dataexchange.handler.AutoSyncID;
 import ru.bclib.api.dataexchange.handler.DataExchange;
 import ru.bclib.config.Config;
 

src/main/java/ru/bclib/api/dataexchange/FileHash.java

@@ -3,6 +3,7 @@ package ru.bclib.api.dataexchange;
 import net.minecraft.network.FriendlyByteBuf;
 import org.jetbrains.annotations.NotNull;
 import ru.bclib.BCLib;
+import ru.bclib.api.dataexchange.handler.AutoSyncID;
 import ru.bclib.api.dataexchange.handler.DataExchange;
 
 import java.io.File;
@@ -22,7 +23,7 @@ import java.util.Objects;
  * You can compare instances using {@link #equals(Object)} to determine if two files are
  * identical.
  */
-public class FileHash extends DataExchange.AutoSyncID {
+public class FileHash extends AutoSyncID {
     /**
      * The md5-hash of the file
      */

src/main/java/ru/bclib/api/dataexchange/handler/AutoFileSyncEntry.java

@@ -11,7 +11,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-class AutoFileSyncEntry extends DataExchange.AutoSyncID {
+class AutoFileSyncEntry extends AutoSyncID {
     public final DataExchange.NeedTransferPredicate needTransfer;
     public final File fileName;
     public final boolean requestContent;
@@ -54,13 +54,13 @@ class AutoFileSyncEntry extends DataExchange.AutoSyncID {
         return serializeFileContent(buf);
     }
 
-    public static Triple<AutoFileSyncEntry, byte[], DataExchange.AutoSyncID> deserializeContent(FriendlyByteBuf buf) {
+    public static Triple<AutoFileSyncEntry, byte[], AutoSyncID> deserializeContent(FriendlyByteBuf buf) {
         final String modID = DataHandler.readString(buf);
         final String uniqueID = DataHandler.readString(buf);
         byte[] data = deserializeFileContent(buf);
 
         AutoFileSyncEntry entry = AutoFileSyncEntry.findMatching(modID, uniqueID);
-        return new Triple<>(entry, data, new DataExchange.AutoSyncID(modID, uniqueID));
+        return new Triple<>(entry, data, new AutoSyncID(modID, uniqueID));
     }
 
 
@@ -108,7 +108,7 @@ class AutoFileSyncEntry extends DataExchange.AutoSyncID {
         return findMatching(hash.modID, hash.uniqueID);
     }
 
-    public static AutoFileSyncEntry findMatching(DataExchange.AutoSyncID aid) {
+    public static AutoFileSyncEntry findMatching(AutoSyncID aid) {
         return findMatching(aid.modID, aid.uniqueID);
     }
 

src/main/java/ru/bclib/api/dataexchange/handler/AutoSyncID.java

@@ -0,0 +1,59 @@
+package ru.bclib.api.dataexchange.handler;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.Objects;
+
+public class AutoSyncID {
+	static class WithContentOverride extends AutoSyncID {
+		final FileContentWrapper contentWrapper;
+		final File localFile;
+		
+		WithContentOverride(String modID, String uniqueID, FileContentWrapper contentWrapper, File localFile) {
+			super(modID, uniqueID);
+			this.contentWrapper = contentWrapper;
+			this.localFile = localFile;
+		}
+	}
+	/**
+	 * A Unique ID for the referenced File.
+	 * <p>
+	 * Files with the same {@link #modID} need to have a unique IDs. Normally the filename from FileHash(String, File, byte[], int, int)
+	 * is used to generated that ID, but you can directly specify one using FileHash(String, String, byte[], int, int).
+	 */
+	@NotNull
+	public final String uniqueID;
+	
+	/**
+	 * The ID of the Mod that is registering the File
+	 */
+	@NotNull
+	public final String modID;
+	
+	public AutoSyncID(String modID, String uniqueID) {
+		Objects.nonNull(modID);
+		Objects.nonNull(uniqueID);
+		
+		this.modID = modID;
+		this.uniqueID = uniqueID;
+	}
+	
+	@Override
+	public String toString() {
+		return modID + "." + uniqueID;
+	}
+	
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) return true;
+		if (!(o instanceof AutoSyncID)) return false;
+		AutoSyncID that = (AutoSyncID) o;
+		return uniqueID.equals(that.uniqueID) && modID.equals(that.modID);
+	}
+	
+	@Override
+	public int hashCode() {
+		return Objects.hash(uniqueID, modID);
+	}
+}

src/main/java/ru/bclib/api/dataexchange/handler/DataExchange.java

@@ -4,82 +4,42 @@ import net.fabricmc.api.EnvType;
 import net.fabricmc.api.Environment;
 import net.fabricmc.fabric.api.client.networking.v1.ClientPlayConnectionEvents;
 import net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents;
-import org.jetbrains.annotations.NotNull;
 import ru.bclib.api.dataexchange.ConnectorClientside;
 import ru.bclib.api.dataexchange.ConnectorServerside;
 import ru.bclib.api.dataexchange.DataExchangeAPI;
 import ru.bclib.api.dataexchange.DataHandler;
 import ru.bclib.api.dataexchange.DataHandlerDescriptor;
 import ru.bclib.api.dataexchange.FileHash;
-import ru.bclib.util.Triple;
 
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiConsumer;
 
 abstract public class DataExchange {
     @FunctionalInterface
     public interface NeedTransferPredicate  {
-        public boolean test(FileHash clientHash, FileHash serverHash, byte[] content);
+        public boolean test(FileHash clientHash, FileHash serverHash, FileContentWrapper content);
-    }
-    
-    public static class AutoSyncID {
-        /**
-         * A Unique ID for the referenced File.
-         * <p>
-         * Files with the same {@link #modID} need to have a unique IDs. Normally the filename from FileHash(String, File, byte[], int, int)
-         * is used to generated that ID, but you can directly specify one using FileHash(String, String, byte[], int, int).
-         */
-        @NotNull
-        public final String uniqueID;
-
-        /**
-         * The ID of the Mod that is registering the File
-         */
-        @NotNull
-        public final String modID;
-
-        public AutoSyncID(String modID, String uniqueID) {
-            Objects.nonNull(modID);
-            Objects.nonNull(uniqueID);
-
-            this.modID = modID;
-            this.uniqueID = uniqueID;
-        }
-    
-        @Override
-        public String toString() {
-            return modID+"."+uniqueID;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (!(o instanceof AutoSyncID)) return false;
-            AutoSyncID that = (AutoSyncID) o;
-            return uniqueID.equals(that.uniqueID) && modID.equals(that.modID);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(uniqueID, modID);
-        }
     }
     
     protected final static List<BiConsumer<AutoSyncID, File>> onWriteCallbacks = new ArrayList<>(2);
 
-    final static class AutoSyncTriple extends Triple<FileHash, byte[], AutoFileSyncEntry>{
+    final static class AutoSyncTriple {
-        public AutoSyncTriple(FileHash first, byte[] second, AutoFileSyncEntry third) {
+        public final FileHash serverHash;
-            super(first, second, third);
+        public final byte[] serverContent;
+        public final AutoFileSyncEntry localMatch;
+        
+        public AutoSyncTriple(FileHash serverHash, byte[] serverContent, AutoFileSyncEntry localMatch) {
+            this.serverHash = serverHash;
+            this.serverContent = serverContent;
+            this.localMatch = localMatch;
         }
     
         @Override
         public String toString() {
-            return first.modID+"."+first.uniqueID;
+            return serverHash.modID+"."+serverHash.uniqueID;
         }
     }
 

src/main/java/ru/bclib/api/dataexchange/handler/FileContentWrapper.java

@@ -0,0 +1,69 @@
+package ru.bclib.api.dataexchange.handler;
+
+import ru.bclib.BCLib;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public class FileContentWrapper {
+	private byte[] rawContent;
+	private ByteArrayOutputStream outputStream;
+	
+	FileContentWrapper(byte[] content){
+		this.rawContent = content;
+		this.outputStream = null;
+	}
+	
+	public byte[] getOriginalContent() {
+		return rawContent;
+	}
+	
+	public byte[] getRawContent() {
+		if (outputStream!=null){
+			return outputStream.toByteArray();
+		}
+		return rawContent;
+	}
+	
+	private void invalidateOutputStream(){
+		if (this.outputStream!=null){
+			try {
+				this.outputStream.close();
+			}
+			catch (IOException e) {
+				BCLib.LOGGER.debug(e);
+			}
+		}
+		this.outputStream = null;
+	}
+	
+	public void setRawContent(byte[] rawContent) {
+		this.rawContent = rawContent;
+		invalidateOutputStream();
+	}
+	
+	public void syncWithOutputStream(){
+		if (outputStream!=null){
+			setRawContent(getRawContent());
+		}
+	}
+	
+	public ByteArrayInputStream getInputStream(){
+		if (rawContent==null) return new ByteArrayInputStream(new byte[0]);
+		return new ByteArrayInputStream(rawContent);
+	}
+	
+	public ByteArrayOutputStream getOrCreateOutputStream(){
+		if (this.outputStream == null){
+			return this.getEmptyOutputStream();
+		}
+		return this.outputStream;
+	}
+	
+	public ByteArrayOutputStream getEmptyOutputStream(){
+		invalidateOutputStream();
+		this.outputStream = new ByteArrayOutputStream(this.rawContent.length);
+		return this.outputStream;
+	}
+}

src/main/java/ru/bclib/api/dataexchange/handler/HelloClient.java

@@ -14,7 +14,7 @@ import ru.bclib.BCLib;
 import ru.bclib.api.dataexchange.DataExchangeAPI;
 import ru.bclib.api.dataexchange.DataHandler;
 import ru.bclib.api.dataexchange.DataHandlerDescriptor;
-import ru.bclib.api.dataexchange.handler.DataExchange.AutoSyncID;
+import ru.bclib.api.dataexchange.handler.AutoSyncID.WithContentOverride;
 import ru.bclib.api.datafixer.DataFixerAPI;
 import ru.bclib.config.Configs;
 import ru.bclib.gui.screens.SyncFilesScreen;
@@ -126,6 +126,7 @@ public class HelloClient extends DataHandler {
 		}
 	}
 	
+	
 	@Override
 	protected void runOnGameThread(Minecraft client, MinecraftServer server, boolean isClient) {
 		final boolean debugHashes = Configs.CLIENT_CONFIG.getBoolean(Configs.MAIN_SYNC_CATEGORY, "debugHashes", false);
@@ -137,7 +138,9 @@ public class HelloClient extends DataHandler {
 		// 	return;
 		// }
 		
-		List<AutoSyncID> filesToRequest = new ArrayList<>(4);
+		final List<AutoSyncID> filesToRequest = new ArrayList<>(2);
+		
+		
 		
 		for (Entry<String, String> e : modVersion.entrySet()){
 			String ver = getModVersion(e.getKey());
@@ -147,26 +150,32 @@ public class HelloClient extends DataHandler {
 		if (autoSyncedFiles.size()>0) {
 			BCLib.LOGGER.info("Files offered by Server:");
 		}
-		final String requestText = SendFiles.acceptFiles()?"requesting":"differs";
 		for (DataExchange.AutoSyncTriple e : autoSyncedFiles) {
-			boolean willRequest = false;
+			String actionString = "";
-			if (e.third == null) {
+			FileContentWrapper contentWrapper = new FileContentWrapper(e.serverContent);
-				willRequest = true;
+			if (e.localMatch == null) {
-				filesToRequest.add(new AutoSyncID(e.first.modID, e.first.uniqueID));
+				actionString = "(new, prepare update)";
-			} else if (e.third.needTransfer.test(e.third.getFileHash(), e.first, e.second)) {
+				filesToRequest.add(new AutoSyncID(e.serverHash.modID, e.serverHash.uniqueID));
-				willRequest = true;
+			} else if (e.localMatch.needTransfer.test(e.localMatch.getFileHash(), e.serverHash, contentWrapper)) {
-				filesToRequest.add(new AutoSyncID(e.first.modID, e.first.uniqueID));
+				actionString = "(prepare update)";
+				//we did not yet receive the new content
+				if (contentWrapper.getRawContent() == null) {
+					filesToRequest.add(new AutoSyncID(e.serverHash.modID, e.serverHash.uniqueID));
+				} else {
+					filesToRequest.add(new AutoSyncID.WithContentOverride(e.serverHash.modID, e.serverHash.uniqueID, contentWrapper, e.localMatch.fileName));
+				}
 			}
 			
-			BCLib.LOGGER.info("    - " + e + ": " + (willRequest ? (" ("+requestText+")" ):""));
+			BCLib.LOGGER.info("    - " + e + ": " + actionString);
 			if (debugHashes) {
-				BCLib.LOGGER.info("      * " + e.first + " (Server)");
+				BCLib.LOGGER.info("      * " + e.serverHash + " (Server)");
-				BCLib.LOGGER.info("      * " + e.third.getFileHash() + " (Client)");
+				BCLib.LOGGER.info("      * " + e.localMatch.getFileHash() + " (Client)");
+				BCLib.LOGGER.info("      * local Content " + (contentWrapper.getRawContent() == null));
 			}
 		}
 
 		if (filesToRequest.size()>0 && SendFiles.acceptFiles()) {
-			showDonwloadConfigs(client, filesToRequest);
+			showDownloadConfigs(client, filesToRequest);
 			return;
 		}
 	}
@@ -185,11 +194,27 @@ public class HelloClient extends DataHandler {
 	}
 	
 	@Environment(EnvType.CLIENT)
-	protected void showDonwloadConfigs(Minecraft client, List<AutoSyncID> files){
+	protected void showDownloadConfigs(Minecraft client, List<AutoSyncID> files){
 		client.setScreen(new SyncFilesScreen((download) -> {
 			Minecraft.getInstance().setScreen((Screen)null);
 			if (download){
-				requestFileDownloads(files);
+				BCLib.LOGGER.info("Updating local Files:");
+				List<AutoSyncID.WithContentOverride> localChanges = new ArrayList<>(files.toArray().length);
+				List<AutoSyncID> requestFiles = new ArrayList<>(files.toArray().length);
+				
+				files.forEach(aid -> {
+					if (aid instanceof WithContentOverride) {
+						final WithContentOverride aidc = (WithContentOverride)aid;
+						BCLib.LOGGER.info("    - " + aid + " (updating Content)");
+						
+						SendFiles.writeSyncedFile(aid, aidc.contentWrapper.getRawContent(), aidc.localFile);
+					} else {
+						requestFiles.add(aid);
+						BCLib.LOGGER.info("    - " + aid + " (requesting)");
+					}
+				});
+				
+				requestFileDownloads(requestFiles);
 			}
 		}));
 

src/main/java/ru/bclib/api/dataexchange/handler/RequestFiles.java

@@ -8,7 +8,6 @@ import net.minecraft.server.MinecraftServer;
 import ru.bclib.BCLib;
 import ru.bclib.api.dataexchange.DataHandler;
 import ru.bclib.api.dataexchange.DataHandlerDescriptor;
-import ru.bclib.api.dataexchange.handler.DataExchange.AutoSyncID;
 
 import java.util.ArrayList;
 import java.util.List;

src/main/java/ru/bclib/api/dataexchange/handler/SendFiles.java

@@ -16,6 +16,7 @@ import ru.bclib.gui.screens.ConfirmRestartScreen;
 import ru.bclib.util.Pair;
 import ru.bclib.util.Triple;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -77,7 +78,7 @@ public class SendFiles extends DataHandler {
 			receivedFiles = new ArrayList<>(size);
 			BCLib.LOGGER.info("Server sent " + size + " Files:");
 			for (int i = 0; i < size; i++) {
-				Triple<AutoFileSyncEntry, byte[], DataExchange.AutoSyncID> p = AutoFileSyncEntry.deserializeContent(buf);
+				Triple<AutoFileSyncEntry, byte[], AutoSyncID> p = AutoFileSyncEntry.deserializeContent(buf);
 				if (p.first != null) {
 					receivedFiles.add(p);
 					BCLib.LOGGER.info("    - " + p.first + " (" + p.second.length + " Bytes)");
@@ -95,20 +96,25 @@ public class SendFiles extends DataHandler {
 			for (Pair<AutoFileSyncEntry, byte[]> entry : receivedFiles) {
 				final AutoFileSyncEntry e = entry.first;
 				final byte[] data = entry.second;
-				Path path = e.fileName.toPath();
+				
-				BCLib.LOGGER.info("    - Writing " + path + " (" + data.length + " Bytes)");
+				writeSyncedFile(e, data, e.fileName);
-				try {
-					Files.write(path, data);
-					DataExchange.didReceiveFile(e, e.fileName);
-				} catch (IOException ioException) {
-					BCLib.LOGGER.error("    --> Writing " + e.fileName + " failed: " + ioException);
-				}
 			}
 
 			showConfirmRestart(client);
 		}
 	}
 	
+	public static void writeSyncedFile(AutoSyncID e, byte[] data, File fileName) {
+		Path path = fileName.toPath();
+		BCLib.LOGGER.info("    - Writing " + path + " (" + data.length + " Bytes)");
+		try {
+			Files.write(path, data);
+			DataExchange.didReceiveFile(e, fileName);
+		} catch (IOException ioException) {
+			BCLib.LOGGER.error("    --> Writing " + fileName + " failed: " + ioException);
+		}
+	}
+	
 	@Environment(EnvType.CLIENT)
 	protected void showConfirmRestart(Minecraft client){
 		client.setScreen(new ConfirmRestartScreen(() -> {

src/main/java/ru/bclib/config/Config.java

@@ -1,31 +1,40 @@
 package ru.bclib.config;
 
+import com.google.gson.JsonObject;
 import org.jetbrains.annotations.Nullable;
 import ru.bclib.BCLib;
 import ru.bclib.api.dataexchange.DataExchangeAPI;
-import ru.bclib.api.dataexchange.handler.DataExchange;
+import ru.bclib.api.dataexchange.FileHash;
+import ru.bclib.api.dataexchange.handler.AutoSyncID;
+import ru.bclib.api.dataexchange.handler.FileContentWrapper;
 import ru.bclib.config.ConfigKeeper.BooleanEntry;
 import ru.bclib.config.ConfigKeeper.Entry;
 import ru.bclib.config.ConfigKeeper.FloatEntry;
 import ru.bclib.config.ConfigKeeper.IntegerEntry;
 import ru.bclib.config.ConfigKeeper.RangeEntry;
 import ru.bclib.config.ConfigKeeper.StringEntry;
+import ru.bclib.util.JsonFactory;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.util.HashMap;
 import java.util.Map;
 
 public abstract class Config {
-	protected final static Map<DataExchange.AutoSyncID, Config> autoSyncConfigs = new HashMap<>();
+	protected final static Map<AutoSyncID, Config> autoSyncConfigs = new HashMap<>();
 	protected final ConfigKeeper keeper;
 	protected final boolean autoSync;
 	protected abstract void registerEntries();
 
 	protected Config(String modID, String group) {
-		this(modID, group, true);
+		this(modID, group, true, false);
 	}
 	
-	protected Config(String modID, String group, boolean autoSync) {
+	protected Config(String modID, String group, boolean autoSync){
+		this(modID, group, autoSync, false);
+	}
+	
+	protected Config(String modID, String group, boolean autoSync, boolean diffContent) {
 		BCLib.LOGGER.info("Registered Config " + modID+"."+group+" ("+autoSync+")");
 		this.keeper = new ConfigKeeper(modID, group);
 		this.registerEntries();
@@ -33,16 +42,22 @@ public abstract class Config {
 
 		if (autoSync) {
 			final String uid = "CONFIG_" + modID + "_" + group;
-			final DataExchange.AutoSyncID aid = new DataExchange.AutoSyncID(BCLib.MOD_ID, uid);
+			final AutoSyncID aid = new AutoSyncID(BCLib.MOD_ID, uid);
-			DataExchangeAPI.addAutoSyncFile(aid.modID, aid.uniqueID, keeper.getConfigFile());
+			DataExchangeAPI.addAutoSyncFile(aid.modID, aid.uniqueID, keeper.getConfigFile(), this::compareForSync );
 			autoSyncConfigs.put(aid, this);
 		}
 	}
 	
+	private boolean compareForSync(FileHash fileHash, FileHash fileHash1, FileContentWrapper content) {
+		ByteArrayInputStream inputStream = content.getInputStream();
+		final JsonObject other = JsonFactory.getJsonObject(inputStream);
+		return this.keeper.compareAndUpdateForSync(other);
+	}
+	
 	public void saveChanges() {
 		this.keeper.save();
 	}
-	public static void reloadSyncedConfig(DataExchange.AutoSyncID aid, File file){
+	public static void reloadSyncedConfig(AutoSyncID aid, File file){
 		Config cfg =  autoSyncConfigs.get(aid);
 		if (cfg!=null) {
 			cfg.reload();

src/main/java/ru/bclib/config/ConfigKeeper.java

@@ -30,6 +30,11 @@ public final class ConfigKeeper {
 		return this.writer.getConfigFile();
 	}
 	
+	boolean compareAndUpdateForSync(JsonObject other) {
+		final JsonObject me = this.configObject;
+		return true;
+	}
+	
 	public void save() {
 		if (!changed) return;
 		this.writer.save();

src/main/java/ru/bclib/config/PathConfig.java

@@ -7,6 +7,10 @@ import ru.bclib.config.ConfigKeeper.IntegerRange;
 
 public class PathConfig extends Config {
 
+	public PathConfig(String modID, String group, boolean autoSync, boolean diffContent) {
+		super(modID, group, autoSync, diffContent);
+	}
+	
 	public PathConfig(String modID, String group, boolean autoSync) {
 		super(modID, group, autoSync);
 	}

src/main/java/ru/bclib/util/JsonFactory.java

@@ -19,6 +19,8 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.Reader;
 
 public class JsonFactory {
@@ -103,6 +105,11 @@ public class JsonFactory {
 		}
 	}
 	
+	public static void storeJson(OutputStream outStream, JsonElement jsonObject) {
+		OutputStreamWriter writer = new OutputStreamWriter(outStream);
+		GSON.toJson(jsonObject, writer);
+	}
+	
 	public static int getInt(JsonObject object, String member, int def) {
 		JsonElement elem = object.get(member);
 		return elem == null ? def : elem.getAsInt();