MinecraftMods/BCLib
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Prepared sync pipeline for content-based config sync

Browse source
This commit is contained in:
Frank Bauer 2021-08-15 11:30:39 +02:00
parent 895b605523
commit f80b55aa50
14 changed files with 242 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -27,5 +27,6 @@ bin/
# fabric
run/
run-client/
output/
*.log

1
src/main/java/ru/bclib/api/dataexchange/DataExchangeAPI.java
View file

@ -4,6 +4,7 @@ import com.google.common.collect.Lists;
import net.fabricmc.api.EnvType;
import net.fabricmc.api.Environment;
import net.minecraft.network.FriendlyByteBuf;
import ru.bclib.api.dataexchange.handler.AutoSyncID;
import ru.bclib.api.dataexchange.handler.DataExchange;
import ru.bclib.config.Config;

3
src/main/java/ru/bclib/api/dataexchange/FileHash.java
View file

@ -3,6 +3,7 @@ package ru.bclib.api.dataexchange;
import net.minecraft.network.FriendlyByteBuf;
import org.jetbrains.annotations.NotNull;
import ru.bclib.BCLib;
import ru.bclib.api.dataexchange.handler.AutoSyncID;
import ru.bclib.api.dataexchange.handler.DataExchange;
import java.io.File;
@ -22,7 +23,7 @@ import java.util.Objects;
* You can compare instances using {@link #equals(Object)} to determine if two files are
* identical.
*/
public class FileHash extends DataExchange.AutoSyncID {
public class FileHash extends AutoSyncID {
/**
* The md5-hash of the file
*/

8
src/main/java/ru/bclib/api/dataexchange/handler/AutoFileSyncEntry.java
View file

@ -11,7 +11,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
class AutoFileSyncEntry extends DataExchange.AutoSyncID {
class AutoFileSyncEntry extends AutoSyncID {
public final DataExchange.NeedTransferPredicate needTransfer;
public final File fileName;
public final boolean requestContent;
@ -54,13 +54,13 @@ class AutoFileSyncEntry extends DataExchange.AutoSyncID {
return serializeFileContent(buf);
}
public static Triple<AutoFileSyncEntry, byte[], DataExchange.AutoSyncID> deserializeContent(FriendlyByteBuf buf) {
public static Triple<AutoFileSyncEntry, byte[], AutoSyncID> deserializeContent(FriendlyByteBuf buf) {
final String modID = DataHandler.readString(buf);
final String uniqueID = DataHandler.readString(buf);
byte[] data = deserializeFileContent(buf);
AutoFileSyncEntry entry = AutoFileSyncEntry.findMatching(modID, uniqueID);
return new Triple<>(entry, data, new DataExchange.AutoSyncID(modID, uniqueID));
return new Triple<>(entry, data, new AutoSyncID(modID, uniqueID));
}
@ -108,7 +108,7 @@ class AutoFileSyncEntry extends DataExchange.AutoSyncID {
return findMatching(hash.modID, hash.uniqueID);
}
public static AutoFileSyncEntry findMatching(DataExchange.AutoSyncID aid) {
public static AutoFileSyncEntry findMatching(AutoSyncID aid) {
return findMatching(aid.modID, aid.uniqueID);
}

59
src/main/java/ru/bclib/api/dataexchange/handler/AutoSyncID.java Normal file
View file

@ -0,0 +1,59 @@
package ru.bclib.api.dataexchange.handler;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.util.Objects;
public class AutoSyncID {
static class WithContentOverride extends AutoSyncID {
final FileContentWrapper contentWrapper;
final File localFile;
WithContentOverride(String modID, String uniqueID, FileContentWrapper contentWrapper, File localFile) {
super(modID, uniqueID);
this.contentWrapper = contentWrapper;
this.localFile = localFile;
}
}
/**
* A Unique ID for the referenced File.
* <p>
* Files with the same {@link #modID} need to have a unique IDs. Normally the filename from FileHash(String, File, byte[], int, int)
* is used to generated that ID, but you can directly specify one using FileHash(String, String, byte[], int, int).
*/
@NotNull
public final String uniqueID;
/**
* The ID of the Mod that is registering the File
*/
@NotNull
public final String modID;
public AutoSyncID(String modID, String uniqueID) {
Objects.nonNull(modID);
Objects.nonNull(uniqueID);
this.modID = modID;
this.uniqueID = uniqueID;
}
@Override
public String toString() {
return modID + "." + uniqueID;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof AutoSyncID)) return false;
AutoSyncID that = (AutoSyncID) o;
return uniqueID.equals(that.uniqueID) && modID.equals(that.modID);
}
@Override
public int hashCode() {
return Objects.hash(uniqueID, modID);
}
}

62
src/main/java/ru/bclib/api/dataexchange/handler/DataExchange.java
View file

@ -4,82 +4,42 @@ import net.fabricmc.api.EnvType;
import net.fabricmc.api.Environment;
import net.fabricmc.fabric.api.client.networking.v1.ClientPlayConnectionEvents;
import net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents;
import org.jetbrains.annotations.NotNull;
import ru.bclib.api.dataexchange.ConnectorClientside;
import ru.bclib.api.dataexchange.ConnectorServerside;
import ru.bclib.api.dataexchange.DataExchangeAPI;
import ru.bclib.api.dataexchange.DataHandler;
import ru.bclib.api.dataexchange.DataHandlerDescriptor;
import ru.bclib.api.dataexchange.FileHash;
import ru.bclib.util.Triple;
import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiConsumer;
abstract public class DataExchange {
@FunctionalInterface
public interface NeedTransferPredicate {
public boolean test(FileHash clientHash, FileHash serverHash, byte[] content);
public boolean test(FileHash clientHash, FileHash serverHash, FileContentWrapper content);
}
public static class AutoSyncID {
/**
* A Unique ID for the referenced File.
* <p>
* Files with the same {@link #modID} need to have a unique IDs. Normally the filename from FileHash(String, File, byte[], int, int)
* is used to generated that ID, but you can directly specify one using FileHash(String, String, byte[], int, int).
*/
@NotNull
public final String uniqueID;
/**
* The ID of the Mod that is registering the File
*/
@NotNull
public final String modID;
public AutoSyncID(String modID, String uniqueID) {
Objects.nonNull(modID);
Objects.nonNull(uniqueID);
this.modID = modID;
this.uniqueID = uniqueID;
}
@Override
public String toString() {
return modID+"."+uniqueID;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof AutoSyncID)) return false;
AutoSyncID that = (AutoSyncID) o;
return uniqueID.equals(that.uniqueID) && modID.equals(that.modID);
}
@Override
public int hashCode() {
return Objects.hash(uniqueID, modID);
}
}
protected final static List<BiConsumer<AutoSyncID, File>> onWriteCallbacks = new ArrayList<>(2);
final static class AutoSyncTriple extends Triple<FileHash, byte[], AutoFileSyncEntry>{
public AutoSyncTriple(FileHash first, byte[] second, AutoFileSyncEntry third) {
super(first, second, third);
final static class AutoSyncTriple {
public final FileHash serverHash;
public final byte[] serverContent;
public final AutoFileSyncEntry localMatch;
public AutoSyncTriple(FileHash serverHash, byte[] serverContent, AutoFileSyncEntry localMatch) {
this.serverHash = serverHash;
this.serverContent = serverContent;
this.localMatch = localMatch;
}
@Override
public String toString() {
return first.modID+"."+first.uniqueID;
return serverHash.modID+"."+serverHash.uniqueID;
}
}

69
src/main/java/ru/bclib/api/dataexchange/handler/FileContentWrapper.java Normal file
View file

@ -0,0 +1,69 @@
package ru.bclib.api.dataexchange.handler;
import ru.bclib.BCLib;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
public class FileContentWrapper {
private byte[] rawContent;
private ByteArrayOutputStream outputStream;
FileContentWrapper(byte[] content){
this.rawContent = content;
this.outputStream = null;
}
public byte[] getOriginalContent() {
return rawContent;
}
public byte[] getRawContent() {
if (outputStream!=null){
return outputStream.toByteArray();
}
return rawContent;
}
private void invalidateOutputStream(){
if (this.outputStream!=null){
try {
this.outputStream.close();
}
catch (IOException e) {
BCLib.LOGGER.debug(e);
}
}
this.outputStream = null;
}
public void setRawContent(byte[] rawContent) {
this.rawContent = rawContent;
invalidateOutputStream();
}
public void syncWithOutputStream(){
if (outputStream!=null){
setRawContent(getRawContent());
}
}
public ByteArrayInputStream getInputStream(){
if (rawContent==null) return new ByteArrayInputStream(new byte[0]);
return new ByteArrayInputStream(rawContent);
}
public ByteArrayOutputStream getOrCreateOutputStream(){
if (this.outputStream == null){
return this.getEmptyOutputStream();
}
return this.outputStream;
}
public ByteArrayOutputStream getEmptyOutputStream(){
invalidateOutputStream();
this.outputStream = new ByteArrayOutputStream(this.rawContent.length);
return this.outputStream;
}
}

57
src/main/java/ru/bclib/api/dataexchange/handler/HelloClient.java
View file

@ -14,7 +14,7 @@ import ru.bclib.BCLib;
import ru.bclib.api.dataexchange.DataExchangeAPI;
import ru.bclib.api.dataexchange.DataHandler;
import ru.bclib.api.dataexchange.DataHandlerDescriptor;
import ru.bclib.api.dataexchange.handler.DataExchange.AutoSyncID;
import ru.bclib.api.dataexchange.handler.AutoSyncID.WithContentOverride;
import ru.bclib.api.datafixer.DataFixerAPI;
import ru.bclib.config.Configs;
import ru.bclib.gui.screens.SyncFilesScreen;
@ -126,6 +126,7 @@ public class HelloClient extends DataHandler {
}
}
@Override
protected void runOnGameThread(Minecraft client, MinecraftServer server, boolean isClient) {
final boolean debugHashes = Configs.CLIENT_CONFIG.getBoolean(Configs.MAIN_SYNC_CATEGORY, "debugHashes", false);
@ -137,7 +138,9 @@ public class HelloClient extends DataHandler {
// return;
// }
List<AutoSyncID> filesToRequest = new ArrayList<>(4);
final List<AutoSyncID> filesToRequest = new ArrayList<>(2);
for (Entry<String, String> e : modVersion.entrySet()){
String ver = getModVersion(e.getKey());
@ -147,26 +150,32 @@ public class HelloClient extends DataHandler {
if (autoSyncedFiles.size()>0) {
BCLib.LOGGER.info("Files offered by Server:");
}
final String requestText = SendFiles.acceptFiles()?"requesting":"differs";
for (DataExchange.AutoSyncTriple e : autoSyncedFiles) {
boolean willRequest = false;
if (e.third == null) {
willRequest = true;
filesToRequest.add(new AutoSyncID(e.first.modID, e.first.uniqueID));
} else if (e.third.needTransfer.test(e.third.getFileHash(), e.first, e.second)) {
willRequest = true;
filesToRequest.add(new AutoSyncID(e.first.modID, e.first.uniqueID));
String actionString = "";
FileContentWrapper contentWrapper = new FileContentWrapper(e.serverContent);
if (e.localMatch == null) {
actionString = "(new, prepare update)";
filesToRequest.add(new AutoSyncID(e.serverHash.modID, e.serverHash.uniqueID));
} else if (e.localMatch.needTransfer.test(e.localMatch.getFileHash(), e.serverHash, contentWrapper)) {
actionString = "(prepare update)";
//we did not yet receive the new content
if (contentWrapper.getRawContent() == null) {
filesToRequest.add(new AutoSyncID(e.serverHash.modID, e.serverHash.uniqueID));
} else {
filesToRequest.add(new AutoSyncID.WithContentOverride(e.serverHash.modID, e.serverHash.uniqueID, contentWrapper, e.localMatch.fileName));
}
}
BCLib.LOGGER.info(" - " + e + ": " + (willRequest ? (" ("+requestText+")" ):""));
BCLib.LOGGER.info(" - " + e + ": " + actionString);
if (debugHashes) {
BCLib.LOGGER.info(" * " + e.first + " (Server)");
BCLib.LOGGER.info(" * " + e.third.getFileHash() + " (Client)");
BCLib.LOGGER.info(" * " + e.serverHash + " (Server)");
BCLib.LOGGER.info(" * " + e.localMatch.getFileHash() + " (Client)");
BCLib.LOGGER.info(" * local Content " + (contentWrapper.getRawContent() == null));
}
}
if (filesToRequest.size()>0 && SendFiles.acceptFiles()) {
showDonwloadConfigs(client, filesToRequest);
showDownloadConfigs(client, filesToRequest);
return;
}
}
@ -185,11 +194,27 @@ public class HelloClient extends DataHandler {
}
@Environment(EnvType.CLIENT)
protected void showDonwloadConfigs(Minecraft client, List<AutoSyncID> files){
protected void showDownloadConfigs(Minecraft client, List<AutoSyncID> files){
client.setScreen(new SyncFilesScreen((download) -> {
Minecraft.getInstance().setScreen((Screen)null);
if (download){
requestFileDownloads(files);
BCLib.LOGGER.info("Updating local Files:");
List<AutoSyncID.WithContentOverride> localChanges = new ArrayList<>(files.toArray().length);
List<AutoSyncID> requestFiles = new ArrayList<>(files.toArray().length);
files.forEach(aid -> {
if (aid instanceof WithContentOverride) {
final WithContentOverride aidc = (WithContentOverride)aid;
BCLib.LOGGER.info(" - " + aid + " (updating Content)");
SendFiles.writeSyncedFile(aid, aidc.contentWrapper.getRawContent(), aidc.localFile);
} else {
requestFiles.add(aid);
BCLib.LOGGER.info(" - " + aid + " (requesting)");
}
});
requestFileDownloads(requestFiles);
}
}));

1
src/main/java/ru/bclib/api/dataexchange/handler/RequestFiles.java
View file

@ -8,7 +8,6 @@ import net.minecraft.server.MinecraftServer;
import ru.bclib.BCLib;
import ru.bclib.api.dataexchange.DataHandler;
import ru.bclib.api.dataexchange.DataHandlerDescriptor;
import ru.bclib.api.dataexchange.handler.DataExchange.AutoSyncID;
import java.util.ArrayList;
import java.util.List;

26
src/main/java/ru/bclib/api/dataexchange/handler/SendFiles.java
View file

@ -16,6 +16,7 @@ import ru.bclib.gui.screens.ConfirmRestartScreen;
import ru.bclib.util.Pair;
import ru.bclib.util.Triple;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -77,7 +78,7 @@ public class SendFiles extends DataHandler {
receivedFiles = new ArrayList<>(size);
BCLib.LOGGER.info("Server sent " + size + " Files:");
for (int i = 0; i < size; i++) {
Triple<AutoFileSyncEntry, byte[], DataExchange.AutoSyncID> p = AutoFileSyncEntry.deserializeContent(buf);
Triple<AutoFileSyncEntry, byte[], AutoSyncID> p = AutoFileSyncEntry.deserializeContent(buf);
if (p.first != null) {
receivedFiles.add(p);
BCLib.LOGGER.info(" - " + p.first + " (" + p.second.length + " Bytes)");
@ -95,20 +96,25 @@ public class SendFiles extends DataHandler {
for (Pair<AutoFileSyncEntry, byte[]> entry : receivedFiles) {
final AutoFileSyncEntry e = entry.first;
final byte[] data = entry.second;
Path path = e.fileName.toPath();
BCLib.LOGGER.info(" - Writing " + path + " (" + data.length + " Bytes)");
try {
Files.write(path, data);
DataExchange.didReceiveFile(e, e.fileName);
} catch (IOException ioException) {
BCLib.LOGGER.error(" --> Writing " + e.fileName + " failed: " + ioException);
}
writeSyncedFile(e, data, e.fileName);
}
showConfirmRestart(client);
}
}
public static void writeSyncedFile(AutoSyncID e, byte[] data, File fileName) {
Path path = fileName.toPath();
BCLib.LOGGER.info(" - Writing " + path + " (" + data.length + " Bytes)");
try {
Files.write(path, data);
DataExchange.didReceiveFile(e, fileName);
} catch (IOException ioException) {
BCLib.LOGGER.error(" --> Writing " + fileName + " failed: " + ioException);
}
}
@Environment(EnvType.CLIENT)
protected void showConfirmRestart(Minecraft client){
client.setScreen(new ConfirmRestartScreen(() -> {

29
src/main/java/ru/bclib/config/Config.java
View file

@ -1,31 +1,40 @@
package ru.bclib.config;
import com.google.gson.JsonObject;
import org.jetbrains.annotations.Nullable;
import ru.bclib.BCLib;
import ru.bclib.api.dataexchange.DataExchangeAPI;
import ru.bclib.api.dataexchange.handler.DataExchange;
import ru.bclib.api.dataexchange.FileHash;
import ru.bclib.api.dataexchange.handler.AutoSyncID;
import ru.bclib.api.dataexchange.handler.FileContentWrapper;
import ru.bclib.config.ConfigKeeper.BooleanEntry;
import ru.bclib.config.ConfigKeeper.Entry;
import ru.bclib.config.ConfigKeeper.FloatEntry;
import ru.bclib.config.ConfigKeeper.IntegerEntry;
import ru.bclib.config.ConfigKeeper.RangeEntry;
import ru.bclib.config.ConfigKeeper.StringEntry;
import ru.bclib.util.JsonFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
public abstract class Config {
protected final static Map<DataExchange.AutoSyncID, Config> autoSyncConfigs = new HashMap<>();
protected final static Map<AutoSyncID, Config> autoSyncConfigs = new HashMap<>();
protected final ConfigKeeper keeper;
protected final boolean autoSync;
protected abstract void registerEntries();
protected Config(String modID, String group) {
this(modID, group, true);
this(modID, group, true, false);
}
protected Config(String modID, String group, boolean autoSync) {
protected Config(String modID, String group, boolean autoSync){
this(modID, group, autoSync, false);
}
protected Config(String modID, String group, boolean autoSync, boolean diffContent) {
BCLib.LOGGER.info("Registered Config " + modID+"."+group+" ("+autoSync+")");
this.keeper = new ConfigKeeper(modID, group);
this.registerEntries();
@ -33,16 +42,22 @@ public abstract class Config {
if (autoSync) {
final String uid = "CONFIG_" + modID + "_" + group;
final DataExchange.AutoSyncID aid = new DataExchange.AutoSyncID(BCLib.MOD_ID, uid);
DataExchangeAPI.addAutoSyncFile(aid.modID, aid.uniqueID, keeper.getConfigFile());
final AutoSyncID aid = new AutoSyncID(BCLib.MOD_ID, uid);
DataExchangeAPI.addAutoSyncFile(aid.modID, aid.uniqueID, keeper.getConfigFile(), this::compareForSync );
autoSyncConfigs.put(aid, this);
}
}
private boolean compareForSync(FileHash fileHash, FileHash fileHash1, FileContentWrapper content) {
ByteArrayInputStream inputStream = content.getInputStream();
final JsonObject other = JsonFactory.getJsonObject(inputStream);
return this.keeper.compareAndUpdateForSync(other);
}
public void saveChanges() {
this.keeper.save();
}
public static void reloadSyncedConfig(DataExchange.AutoSyncID aid, File file){
public static void reloadSyncedConfig(AutoSyncID aid, File file){
Config cfg = autoSyncConfigs.get(aid);
if (cfg!=null) {
cfg.reload();

5
src/main/java/ru/bclib/config/ConfigKeeper.java
View file

@ -30,6 +30,11 @@ public final class ConfigKeeper {
return this.writer.getConfigFile();
}
boolean compareAndUpdateForSync(JsonObject other) {
final JsonObject me = this.configObject;
return true;
}
public void save() {
if (!changed) return;
this.writer.save();

4
src/main/java/ru/bclib/config/PathConfig.java
View file

@ -7,6 +7,10 @@ import ru.bclib.config.ConfigKeeper.IntegerRange;
public class PathConfig extends Config {
public PathConfig(String modID, String group, boolean autoSync, boolean diffContent) {
super(modID, group, autoSync, diffContent);
}
public PathConfig(String modID, String group, boolean autoSync) {
super(modID, group, autoSync);
}

7
src/main/java/ru/bclib/util/JsonFactory.java
View file

@ -19,6 +19,8 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
public class JsonFactory {
@ -103,6 +105,11 @@ public class JsonFactory {
}
}
public static void storeJson(OutputStream outStream, JsonElement jsonObject) {
OutputStreamWriter writer = new OutputStreamWriter(outStream);
GSON.toJson(jsonObject, writer);
}
public static int getInt(JsonObject object, String member, int def) {
JsonElement elem = object.get(member);
return elem == null ? def : elem.getAsInt();