Import some useful code from MultiMC, add future TeamCity build index

CMakeLists.txt

@@ -71,6 +71,35 @@ target_include_directories(QtWebApp PUBLIC
 target_link_libraries(QtWebApp Qt5::Core Qt5::Network)
 
 add_executable(squeezer
+    src/tasks/SequentialTask.cpp
+    src/tasks/SequentialTask.h
+    src/tasks/Task.cpp
+    src/tasks/Task.h
+
+    src/net/Validator.h
+    src/net/Sink.h
+    src/net/NetJob.h
+    src/net/NetJob.cpp
+    src/net/NetAction.h
+    src/net/Mode.h
+    src/net/FileSink.h
+    src/net/FileSink.cpp
+    src/net/Download.h
+    src/net/Download.cpp
+    src/net/ChecksumValidator.h
+    src/net/ByteArraySink.h
+
+    src/mojang/PackageManifest.h
+    src/mojang/PackageManifest.cpp
+
+    src/pathmatcher/FSTreeMatcher.h
+    src/pathmatcher/IPathMatcher.h
+    src/pathmatcher/MultiMatcher.h
+    src/pathmatcher/RegexpMatcher.h
+
+    src/teamcity/BuildIndex.h
+    src/teamcity/BuildIndex.cpp
+
     src/controller/dumpcontroller.cpp
     src/controller/dumpcontroller.h
     src/controller/fileuploadcontroller.cpp
@@ -89,8 +118,15 @@ add_executable(squeezer
     src/main.cpp
     src/requestmapper.cpp
     src/requestmapper.h
+
+    src/FileSystem.cpp
+    src/FileSystem.h
+    src/Json.cpp
+    src/Json.h
+    src/QObjectPtr.h
 )
 target_link_libraries(squeezer QtWebApp)
+target_include_directories(squeezer PRIVATE src)
 
 install(TARGETS squeezer DESTINATION bin)
 install(DIRECTORY data/ DESTINATION .)

src/Exception.h

@@ -0,0 +1,32 @@
+// Licensed under the Apache-2.0 license. See README.md for details.
+
+#pragma once
+
+#include <QString>
+#include <QDebug>
+#include <exception>
+
+class Exception : public std::exception
+{
+public:
+    Exception(const QString &message) : std::exception(), m_message(message)
+    {
+        qCritical() << "Exception:" << message;
+    }
+    Exception(const Exception &other)
+        : std::exception(), m_message(other.cause())
+    {
+    }
+    virtual ~Exception() noexcept {}
+    const char *what() const noexcept
+    {
+        return m_message.toLatin1().constData();
+    }
+    QString cause() const
+    {
+        return m_message;
+    }
+
+private:
+    QString m_message;
+};

src/FileSystem.cpp

@@ -0,0 +1,457 @@
+// Licensed under the Apache-2.0 license. See README.md for details.
+
+#include "FileSystem.h"
+
+#include <QDir>
+#include <QFile>
+#include <QSaveFile>
+#include <QFileInfo>
+#include <QDebug>
+#include <QUrl>
+#include <QStandardPaths>
+#include <QTextStream>
+
+#if defined Q_OS_WIN32
+    #include <windows.h>
+    #include <string>
+    #include <sys/utime.h>
+    #include <winnls.h>
+    #include <shobjidl.h>
+    #include <objbase.h>
+    #include <objidl.h>
+    #include <shlguid.h>
+    #include <shlobj.h>
+#else
+    #include <utime.h>
+#endif
+
+namespace FS {
+
+void ensureExists(const QDir &dir)
+{
+    if (!QDir().mkpath(dir.absolutePath()))
+    {
+        throw FileSystemException("Unable to create folder " + dir.dirName() + " (" +
+                                      dir.absolutePath() + ")");
+    }
+}
+
+void write(const QString &filename, const QByteArray &data)
+{
+    ensureExists(QFileInfo(filename).dir());
+    QSaveFile file(filename);
+    if (!file.open(QSaveFile::WriteOnly))
+    {
+        throw FileSystemException("Couldn't open " + filename + " for writing: " +
+                                  file.errorString());
+    }
+    if (data.size() != file.write(data))
+    {
+        throw FileSystemException("Error writing data to " + filename + ": " +
+                                  file.errorString());
+    }
+    if (!file.commit())
+    {
+        throw FileSystemException("Error while committing data to " + filename + ": " +
+                                  file.errorString());
+    }
+}
+
+QByteArray read(const QString &filename)
+{
+    QFile file(filename);
+    if (!file.open(QFile::ReadOnly))
+    {
+        throw FileSystemException("Unable to open " + filename + " for reading: " +
+                                  file.errorString());
+    }
+    const qint64 size = file.size();
+    QByteArray data(int(size), 0);
+    const qint64 ret = file.read(data.data(), size);
+    if (ret == -1 || ret != size)
+    {
+        throw FileSystemException("Error reading data from " + filename + ": " +
+                                  file.errorString());
+    }
+    return data;
+}
+
+bool updateTimestamp(const QString& filename)
+{
+#ifdef Q_OS_WIN32
+    std::wstring filename_utf_16 = filename.toStdWString();
+    return (_wutime64(filename_utf_16.c_str(), nullptr) == 0);
+#else
+    QByteArray filenameBA = QFile::encodeName(filename);
+    return (utime(filenameBA.data(), nullptr) == 0);
+#endif
+}
+
+bool ensureFilePathExists(QString filenamepath)
+{
+    QFileInfo a(filenamepath);
+    QDir dir;
+    QString ensuredPath = a.path();
+    bool success = dir.mkpath(ensuredPath);
+    return success;
+}
+
+bool ensureFolderPathExists(QString foldernamepath)
+{
+    QFileInfo a(foldernamepath);
+    QDir dir;
+    QString ensuredPath = a.filePath();
+    bool success = dir.mkpath(ensuredPath);
+    return success;
+}
+
+bool copy::operator()(const QString &offset)
+{
+    //NOTE always deep copy on windows. the alternatives are too messy.
+    #if defined Q_OS_WIN32
+    m_followSymlinks = true;
+    #endif
+
+    auto src = PathCombine(m_src.absolutePath(), offset);
+    auto dst = PathCombine(m_dst.absolutePath(), offset);
+
+    QFileInfo currentSrc(src);
+    if (!currentSrc.exists())
+        return false;
+
+    if(!m_followSymlinks && currentSrc.isSymLink())
+    {
+        qDebug() << "creating symlink" << src << " - " << dst;
+        if (!ensureFilePathExists(dst))
+        {
+            qWarning() << "Cannot create path!";
+            return false;
+        }
+        return QFile::link(currentSrc.symLinkTarget(), dst);
+    }
+    else if(currentSrc.isFile())
+    {
+        qDebug() << "copying file" << src << " - " << dst;
+        if (!ensureFilePathExists(dst))
+        {
+            qWarning() << "Cannot create path!";
+            return false;
+        }
+        return QFile::copy(src, dst);
+    }
+    else if(currentSrc.isDir())
+    {
+        qDebug() << "recursing" << offset;
+        if (!ensureFolderPathExists(dst))
+        {
+            qWarning() << "Cannot create path!";
+            return false;
+        }
+        QDir currentDir(src);
+        for(auto & f : currentDir.entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System))
+        {
+            auto inner_offset = PathCombine(offset, f);
+            // ignore and skip stuff that matches the blacklist.
+            if(m_blacklist && m_blacklist->matches(inner_offset))
+            {
+                continue;
+            }
+            if(!operator()(inner_offset))
+            {
+                qWarning() << "Failed to copy" << inner_offset;
+                return false;
+            }
+        }
+    }
+    else
+    {
+        qCritical() << "Copy ERROR: Unknown filesystem object:" << src;
+        return false;
+    }
+    return true;
+}
+
+bool deletePath(QString path)
+{
+    bool OK = true;
+    QFileInfo finfo(path);
+    if(finfo.isFile()) {
+        return QFile::remove(path);
+    }
+
+    QDir dir(path);
+
+    if (!dir.exists())
+    {
+        return OK;
+    }
+    auto allEntries = dir.entryInfoList(QDir::NoDotAndDotDot | QDir::System | QDir::Hidden |
+                                        QDir::AllDirs | QDir::Files,
+                                        QDir::DirsFirst);
+
+    for(auto & info: allEntries)
+    {
+#if defined Q_OS_WIN32
+        QString nativePath = QDir::toNativeSeparators(info.absoluteFilePath());
+        auto wString = nativePath.toStdWString();
+        DWORD dwAttrs = GetFileAttributesW(wString.c_str());
+        // Windows: check for junctions, reparse points and other nasty things of that sort
+        if(dwAttrs & FILE_ATTRIBUTE_REPARSE_POINT)
+        {
+            if (info.isFile())
+            {
+                OK &= QFile::remove(info.absoluteFilePath());
+            }
+            else if (info.isDir())
+            {
+                OK &= dir.rmdir(info.absoluteFilePath());
+            }
+        }
+#else
+        // We do not trust Qt with reparse points, but do trust it with unix symlinks.
+        if(info.isSymLink())
+        {
+            OK &= QFile::remove(info.absoluteFilePath());
+        }
+#endif
+        else if (info.isDir())
+        {
+            OK &= deletePath(info.absoluteFilePath());
+        }
+        else if (info.isFile())
+        {
+            OK &= QFile::remove(info.absoluteFilePath());
+        }
+        else
+        {
+            OK = false;
+            qCritical() << "Delete ERROR: Unknown filesystem object:" << info.absoluteFilePath();
+        }
+    }
+    OK &= dir.rmdir(dir.absolutePath());
+    return OK;
+}
+
+
+QString PathCombine(const QString & path1, const QString & path2)
+{
+    if(!path1.size())
+        return path2;
+    if(!path2.size())
+        return path1;
+    return QDir::cleanPath(path1 + QDir::separator() + path2);
+}
+
+QString PathCombine(const QString & path1, const QString & path2, const QString & path3)
+{
+    return PathCombine(PathCombine(path1, path2), path3);
+}
+
+QString PathCombine(const QString & path1, const QString & path2, const QString & path3, const QString & path4)
+{
+    return PathCombine(PathCombine(path1, path2, path3), path4);
+}
+
+QString AbsolutePath(QString path)
+{
+    return QFileInfo(path).absolutePath();
+}
+
+QString ResolveExecutable(QString path)
+{
+    if (path.isEmpty())
+    {
+        return QString();
+    }
+    if(!path.contains('/'))
+    {
+        path = QStandardPaths::findExecutable(path);
+    }
+    QFileInfo pathInfo(path);
+    if(!pathInfo.exists() || !pathInfo.isExecutable())
+    {
+        return QString();
+    }
+    return pathInfo.absoluteFilePath();
+}
+
+/**
+ * Normalize path
+ *
+ * Any paths inside the current folder will be normalized to relative paths (to current)
+ * Other paths will be made absolute
+ */
+QString NormalizePath(QString path)
+{
+    QDir a = QDir::currentPath();
+    QString currentAbsolute = a.absolutePath();
+
+    QDir b(path);
+    QString newAbsolute = b.absolutePath();
+
+    if (newAbsolute.startsWith(currentAbsolute))
+    {
+        return a.relativeFilePath(newAbsolute);
+    }
+    else
+    {
+        return newAbsolute;
+    }
+}
+
+QString badFilenameChars = "\"\\/?<>:;*|!+\r\n";
+
+QString RemoveInvalidFilenameChars(QString string, QChar replaceWith)
+{
+    for (int i = 0; i < string.length(); i++)
+    {
+        if (badFilenameChars.contains(string[i]))
+        {
+            string[i] = replaceWith;
+        }
+    }
+    return string;
+}
+
+QString DirNameFromString(QString string, QString inDir)
+{
+    int num = 0;
+    QString baseName = RemoveInvalidFilenameChars(string, '-');
+    QString dirName;
+    do
+    {
+        if(num == 0)
+        {
+            dirName = baseName;
+        }
+        else
+        {
+            dirName = baseName + QString::number(num);;
+        }
+
+        // If it's over 9000
+        if (num > 9000)
+            return "";
+        num++;
+    } while (QFileInfo(PathCombine(inDir, dirName)).exists());
+    return dirName;
+}
+
+// Does the folder path contain any '!'? If yes, return true, otherwise false.
+// (This is a problem for Java)
+bool checkProblemticPathJava(QDir folder)
+{
+    QString pathfoldername = folder.absolutePath();
+    return pathfoldername.contains("!", Qt::CaseInsensitive);
+}
+
+// Win32 crap
+#if defined Q_OS_WIN
+
+bool called_coinit = false;
+
+HRESULT CreateLink(LPCSTR linkPath, LPCSTR targetPath, LPCSTR args)
+{
+    HRESULT hres;
+
+    if (!called_coinit)
+    {
+        hres = CoInitialize(NULL);
+        called_coinit = true;
+
+        if (!SUCCEEDED(hres))
+        {
+            qWarning("Failed to initialize COM. Error 0x%08lX", hres);
+            return hres;
+        }
+    }
+
+    IShellLink *link;
+    hres = CoCreateInstance(CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER, IID_IShellLink,
+                            (LPVOID *)&link);
+
+    if (SUCCEEDED(hres))
+    {
+        IPersistFile *persistFile;
+
+        link->SetPath(targetPath);
+        link->SetArguments(args);
+
+        hres = link->QueryInterface(IID_IPersistFile, (LPVOID *)&persistFile);
+        if (SUCCEEDED(hres))
+        {
+            WCHAR wstr[MAX_PATH];
+
+            MultiByteToWideChar(CP_ACP, 0, linkPath, -1, wstr, MAX_PATH);
+
+            hres = persistFile->Save(wstr, TRUE);
+            persistFile->Release();
+        }
+        link->Release();
+    }
+    return hres;
+}
+
+#endif
+
+QString getDesktopDir()
+{
+    return QStandardPaths::writableLocation(QStandardPaths::DesktopLocation);
+}
+
+// Cross-platform Shortcut creation
+bool createShortCut(QString location, QString dest, QStringList args, QString name,
+                          QString icon)
+{
+#if defined(Q_OS_LINUX) || defined(Q_OS_FREEBSD)
+    location = PathCombine(location, name + ".desktop");
+
+    QFile f(location);
+    f.open(QIODevice::WriteOnly | QIODevice::Text);
+    QTextStream stream(&f);
+
+    QString argstring;
+    if (!args.empty())
+        argstring = " '" + args.join("' '") + "'";
+
+    stream << "[Desktop Entry]"
+           << "\n";
+    stream << "Type=Application"
+           << "\n";
+    stream << "TryExec=" << dest.toLocal8Bit() << "\n";
+    stream << "Exec=" << dest.toLocal8Bit() << argstring.toLocal8Bit() << "\n";
+    stream << "Name=" << name.toLocal8Bit() << "\n";
+    stream << "Icon=" << icon.toLocal8Bit() << "\n";
+
+    stream.flush();
+    f.close();
+
+    f.setPermissions(f.permissions() | QFileDevice::ExeOwner | QFileDevice::ExeGroup |
+                     QFileDevice::ExeOther);
+
+    return true;
+#elif defined Q_OS_WIN
+    // TODO: Fix
+    //    QFile file(PathCombine(location, name + ".lnk"));
+    //    WCHAR *file_w;
+    //    WCHAR *dest_w;
+    //    WCHAR *args_w;
+    //    file.fileName().toWCharArray(file_w);
+    //    dest.toWCharArray(dest_w);
+
+    //    QString argStr;
+    //    for (int i = 0; i < args.count(); i++)
+    //    {
+    //        argStr.append(args[i]);
+    //        argStr.append(" ");
+    //    }
+    //    argStr.toWCharArray(args_w);
+
+    //    return SUCCEEDED(CreateLink(file_w, dest_w, args_w));
+    return false;
+#else
+    qWarning("Desktop Shortcuts not supported on your platform!");
+    return false;
+#endif
+}
+}

src/FileSystem.h

@@ -0,0 +1,127 @@
+// Licensed under the Apache-2.0 license. See README.md for details.
+
+#pragma once
+
+#include "Exception.h"
+#include "pathmatcher/IPathMatcher.h"
+
+#include <QDir>
+#include <QFlags>
+
+namespace FS
+{
+
+class FileSystemException : public ::Exception
+{
+public:
+    FileSystemException(const QString &message) : Exception(message) {}
+};
+
+/**
+ * write data to a file safely
+ */
+void write(const QString &filename, const QByteArray &data);
+
+/**
+ * read data from a file safely\
+ */
+QByteArray read(const QString &filename);
+
+/**
+ * Update the last changed timestamp of an existing file
+ */
+bool updateTimestamp(const QString & filename);
+
+/**
+ * Creates all the folders in a path for the specified path
+ * last segment of the path is treated as a file name and is ignored!
+ */
+bool ensureFilePathExists(QString filenamepath);
+
+/**
+ * Creates all the folders in a path for the specified path
+ * last segment of the path is treated as a folder name and is created!
+ */
+bool ensureFolderPathExists(QString filenamepath);
+
+class copy
+{
+public:
+    copy(const QString & src, const QString & dst)
+    {
+        m_src = src;
+        m_dst = dst;
+    }
+    copy & followSymlinks(const bool follow)
+    {
+        m_followSymlinks = follow;
+        return *this;
+    }
+    copy & blacklist(const IPathMatcher * filter)
+    {
+        m_blacklist = filter;
+        return *this;
+    }
+    bool operator()()
+    {
+        return operator()(QString());
+    }
+
+private:
+    bool operator()(const QString &offset);
+
+private:
+    bool m_followSymlinks = true;
+    const IPathMatcher * m_blacklist = nullptr;
+    QDir m_src;
+    QDir m_dst;
+};
+
+/**
+ * Delete a folder recursively
+ */
+bool deletePath(QString path);
+
+QString PathCombine(const QString &path1, const QString &path2);
+QString PathCombine(const QString &path1, const QString &path2, const QString &path3);
+QString PathCombine(const QString &path1, const QString &path2, const QString &path3, const QString &path4);
+
+QString AbsolutePath(QString path);
+
+/**
+ * Resolve an executable
+ *
+ * Will resolve:
+ *   single executable (by name)
+ *   relative path
+ *   absolute path
+ *
+ * @return absolute path to executable or null string
+ */
+QString ResolveExecutable(QString path);
+
+/**
+ * Normalize path
+ *
+ * Any paths inside the current directory will be normalized to relative paths (to current)
+ * Other paths will be made absolute
+ *
+ * Returns false if the path logic somehow filed (and normalizedPath in invalid)
+ */
+QString NormalizePath(QString path);
+
+QString RemoveInvalidFilenameChars(QString string, QChar replaceWith = '-');
+
+QString DirNameFromString(QString string, QString inDir = ".");
+
+/// Checks if the a given Path contains "!"
+bool checkProblemticPathJava(QDir folder);
+
+// Get the Directory representing the User's Desktop
+QString getDesktopDir();
+
+// Create a shortcut at *location*, pointing to *dest* called with the arguments *args*
+// call it *name* and assign it the icon *icon*
+// return true if operation succeeded
+bool createShortCut(QString location, QString dest, QStringList args, QString name, QString iconLocation);
+}

src/Json.cpp

@@ -0,0 +1,272 @@
+// Licensed under the Apache-2.0 license. See README.md for details.
+
+#include "Json.h"
+
+#include <QFile>
+
+#include "FileSystem.h"
+#include <math.h>
+
+namespace Json
+{
+void write(const QJsonDocument &doc, const QString &filename)
+{
+    FS::write(filename, doc.toJson());
+}
+void write(const QJsonObject &object, const QString &filename)
+{
+    write(QJsonDocument(object), filename);
+}
+void write(const QJsonArray &array, const QString &filename)
+{
+    write(QJsonDocument(array), filename);
+}
+
+QByteArray toBinary(const QJsonObject &obj)
+{
+    return QJsonDocument(obj).toBinaryData();
+}
+QByteArray toBinary(const QJsonArray &array)
+{
+    return QJsonDocument(array).toBinaryData();
+}
+QByteArray toText(const QJsonObject &obj)
+{
+    return QJsonDocument(obj).toJson(QJsonDocument::Compact);
+}
+QByteArray toText(const QJsonArray &array)
+{
+    return QJsonDocument(array).toJson(QJsonDocument::Compact);
+}
+
+static bool isBinaryJson(const QByteArray &data)
+{
+    decltype(QJsonDocument::BinaryFormatTag) tag = QJsonDocument::BinaryFormatTag;
+    return memcmp(data.constData(), &tag, sizeof(QJsonDocument::BinaryFormatTag)) == 0;
+}
+QJsonDocument requireDocument(const QByteArray &data, const QString &what)
+{
+    if (isBinaryJson(data))
+    {
+        QJsonDocument doc = QJsonDocument::fromBinaryData(data);
+        if (doc.isNull())
+        {
+            throw JsonException(what + ": Invalid JSON (binary JSON detected)");
+        }
+        return doc;
+    }
+    else
+    {
+        QJsonParseError error;
+        QJsonDocument doc = QJsonDocument::fromJson(data, &error);
+        if (error.error != QJsonParseError::NoError)
+        {
+            throw JsonException(what + ": Error parsing JSON: " + error.errorString());
+        }
+        return doc;
+    }
+}
+QJsonDocument requireDocument(const QString &filename, const QString &what)
+{
+    return requireDocument(FS::read(filename), what);
+}
+QJsonObject requireObject(const QJsonDocument &doc, const QString &what)
+{
+    if (!doc.isObject())
+    {
+        throw JsonException(what + " is not an object");
+    }
+    return doc.object();
+}
+QJsonArray requireArray(const QJsonDocument &doc, const QString &what)
+{
+    if (!doc.isArray())
+    {
+        throw JsonException(what + " is not an array");
+    }
+    return doc.array();
+}
+
+void writeString(QJsonObject &to, const QString &key, const QString &value)
+{
+    if (!value.isEmpty())
+    {
+        to.insert(key, value);
+    }
+}
+
+void writeStringList(QJsonObject &to, const QString &key, const QStringList &values)
+{
+    if (!values.isEmpty())
+    {
+        QJsonArray array;
+        for(auto value: values)
+        {
+            array.append(value);
+        }
+        to.insert(key, array);
+    }
+}
+
+template<>
+QJsonValue toJson<QUrl>(const QUrl &url)
+{
+    return QJsonValue(url.toString(QUrl::FullyEncoded));
+}
+template<>
+QJsonValue toJson<QByteArray>(const QByteArray &data)
+{
+    return QJsonValue(QString::fromLatin1(data.toHex()));
+}
+template<>
+QJsonValue toJson<QDateTime>(const QDateTime &datetime)
+{
+    return QJsonValue(datetime.toString(Qt::ISODate));
+}
+template<>
+QJsonValue toJson<QDir>(const QDir &dir)
+{
+    return QDir::current().relativeFilePath(dir.absolutePath());
+}
+template<>
+QJsonValue toJson<QUuid>(const QUuid &uuid)
+{
+    return uuid.toString();
+}
+template<>
+QJsonValue toJson<QVariant>(const QVariant &variant)
+{
+    return QJsonValue::fromVariant(variant);
+}
+
+
+template<> QByteArray requireIsType<QByteArray>(const QJsonValue &value, const QString &what)
+{
+    const QString string = ensureIsType<QString>(value, what);
+    // ensure that the string can be safely cast to Latin1
+    if (string != QString::fromLatin1(string.toLatin1()))
+    {
+        throw JsonException(what + " is not encodable as Latin1");
+    }
+    return QByteArray::fromHex(string.toLatin1());
+}
+
+template<> QJsonArray requireIsType<QJsonArray>(const QJsonValue &value, const QString &what)
+{
+    if (!value.isArray())
+    {
+        throw JsonException(what + " is not an array");
+    }
+    return value.toArray();
+}
+
+
+template<> QString requireIsType<QString>(const QJsonValue &value, const QString &what)
+{
+    if (!value.isString())
+    {
+        throw JsonException(what + " is not a string");
+    }
+    return value.toString();
+}
+
+template<> bool requireIsType<bool>(const QJsonValue &value, const QString &what)
+{
+    if (!value.isBool())
+    {
+        throw JsonException(what + " is not a bool");
+    }
+    return value.toBool();
+}
+
+template<> double requireIsType<double>(const QJsonValue &value, const QString &what)
+{
+    if (!value.isDouble())
+    {
+        throw JsonException(what + " is not a double");
+    }
+    return value.toDouble();
+}
+
+template<> int requireIsType<int>(const QJsonValue &value, const QString &what)
+{
+    const double doubl = requireIsType<double>(value, what);
+    if (fmod(doubl, 1) != 0)
+    {
+        throw JsonException(what + " is not an integer");
+    }
+    return int(doubl);
+}
+
+template<> QDateTime requireIsType<QDateTime>(const QJsonValue &value, const QString &what)
+{
+    const QString string = requireIsType<QString>(value, what);
+    const QDateTime datetime = QDateTime::fromString(string, Qt::ISODate);
+    if (!datetime.isValid())
+    {
+        throw JsonException(what + " is not a ISO formatted date/time value");
+    }
+    return datetime;
+}
+
+template<> QUrl requireIsType<QUrl>(const QJsonValue &value, const QString &what)
+{
+    const QString string = ensureIsType<QString>(value, what);
+    if (string.isEmpty())
+    {
+        return QUrl();
+    }
+    const QUrl url = QUrl(string, QUrl::StrictMode);
+    if (!url.isValid())
+    {
+        throw JsonException(what + " is not a correctly formatted URL");
+    }
+    return url;
+}
+
+template<> QDir requireIsType<QDir>(const QJsonValue &value, const QString &what)
+{
+    const QString string = requireIsType<QString>(value, what);
+    // FIXME: does not handle invalid characters!
+    return QDir::current().absoluteFilePath(string);
+}
+
+template<> QUuid requireIsType<QUuid>(const QJsonValue &value, const QString &what)
+{
+    const QString string = requireIsType<QString>(value, what);
+    const QUuid uuid = QUuid(string);
+    if (uuid.toString() != string) // converts back => valid
+    {
+        throw JsonException(what + " is not a valid UUID");
+    }
+    return uuid;
+}
+
+template<> QJsonObject requireIsType<QJsonObject>(const QJsonValue &value, const QString &what)
+{
+    if (!value.isObject())
+    {
+        throw JsonException(what + " is not an object");
+    }
+    return value.toObject();
+}
+
+template<> QVariant requireIsType<QVariant>(const QJsonValue &value, const QString &what)
+{
+    if (value.isNull() || value.isUndefined())
+    {
+        throw JsonException(what + " is null or undefined");
+    }
+    return value.toVariant();
+}
+
+template<> QJsonValue requireIsType<QJsonValue>(const QJsonValue &value, const QString &what)
+{
+    if (value.isNull() || value.isUndefined())
+    {
+        throw JsonException(what + " is null or undefined");
+    }
+    return value;
+}
+
+}

src/Json.h

@@ -0,0 +1,249 @@
+// Licensed under the Apache-2.0 license. See README.md for details.
+
+#pragma once
+
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QDateTime>
+#include <QUrl>
+#include <QDir>
+#include <QUuid>
+#include <QVariant>
+#include <memory>
+
+#include "Exception.h"
+
+namespace Json
+{
+class JsonException : public ::Exception
+{
+public:
+    JsonException(const QString &message) : Exception(message) {}
+};
+
+/// @throw FileSystemException
+void write(const QJsonDocument &doc, const QString &filename);
+/// @throw FileSystemException
+void write(const QJsonObject &object, const QString &filename);
+/// @throw FileSystemException
+void write(const QJsonArray &array, const QString &filename);
+
+QByteArray toBinary(const QJsonObject &obj);
+QByteArray toBinary(const QJsonArray &array);
+QByteArray toText(const QJsonObject &obj);
+QByteArray toText(const QJsonArray &array);
+
+/// @throw JsonException
+QJsonDocument requireDocument(const QByteArray &data, const QString &what = "Document");
+/// @throw JsonException
+QJsonDocument requireDocument(const QString &filename, const QString &what = "Document");
+/// @throw JsonException
+QJsonObject requireObject(const QJsonDocument &doc, const QString &what = "Document");
+/// @throw JsonException
+QJsonArray requireArray(const QJsonDocument &doc, const QString &what = "Document");
+
+/////////////////// WRITING ////////////////////
+
+void writeString(QJsonObject & to, const QString &key, const QString &value);
+void writeStringList(QJsonObject & to, const QString &key, const QStringList &values);
+
+template<typename T>
+QJsonValue toJson(const T &t)
+{
+    return QJsonValue(t);
+}
+template<>
+QJsonValue toJson<QUrl>(const QUrl &url);
+template<>
+QJsonValue toJson<QByteArray>(const QByteArray &data);
+template<>
+QJsonValue toJson<QDateTime>(const QDateTime &datetime);
+template<>
+QJsonValue toJson<QDir>(const QDir &dir);
+template<>
+QJsonValue toJson<QUuid>(const QUuid &uuid);
+template<>
+QJsonValue toJson<QVariant>(const QVariant &variant);
+
+template<typename T>
+QJsonArray toJsonArray(const QList<T> &container)
+{
+    QJsonArray array;
+    for (const T item : container)
+    {
+        array.append(toJson<T>(item));
+    }
+    return array;
+}
+
+////////////////// READING ////////////////////
+
+/// @throw JsonException
+template <typename T>
+T requireIsType(const QJsonValue &value, const QString &what = "Value");
+
+/// @throw JsonException
+template<> double requireIsType<double>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> bool requireIsType<bool>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> int requireIsType<int>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QJsonObject requireIsType<QJsonObject>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QJsonArray requireIsType<QJsonArray>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QJsonValue requireIsType<QJsonValue>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QByteArray requireIsType<QByteArray>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QDateTime requireIsType<QDateTime>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QVariant requireIsType<QVariant>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QString requireIsType<QString>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QUuid requireIsType<QUuid>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QDir requireIsType<QDir>(const QJsonValue &value, const QString &what);
+/// @throw JsonException
+template<> QUrl requireIsType<QUrl>(const QJsonValue &value, const QString &what);
+
+// the following functions are higher level functions, that make use of the above functions for
+// type conversion
+template <typename T>
+T ensureIsType(const QJsonValue &value, const T default_ = T(), const QString &what = "Value")
+{
+    if (value.isUndefined() || value.isNull())
+    {
+        return default_;
+    }
+    try
+    {
+        return requireIsType<T>(value, what);
+    }
+    catch (const JsonException &)
+    {
+        return default_;
+    }
+}
+
+/// @throw JsonException
+template <typename T>
+T requireIsType(const QJsonObject &parent, const QString &key, const QString &what = "__placeholder__")
+{
+    const QString localWhat = QString(what).replace("__placeholder__", '\'' + key + '\'');
+    if (!parent.contains(key))
+    {
+        throw JsonException(localWhat + "s parent does not contain " + localWhat);
+    }
+    return requireIsType<T>(parent.value(key), localWhat);
+}
+
+template <typename T>
+T ensureIsType(const QJsonObject &parent, const QString &key, const T default_ = T(), const QString &what = "__placeholder__")
+{
+    const QString localWhat = QString(what).replace("__placeholder__", '\'' + key + '\'');
+    if (!parent.contains(key))
+    {
+        return default_;
+    }
+    return ensureIsType<T>(parent.value(key), default_, localWhat);
+}
+
+template <typename T>
+QVector<T> requireIsArrayOf(const QJsonDocument &doc)
+{
+    const QJsonArray array = requireArray(doc);
+    QVector<T> out;
+    for (const QJsonValue val : array)
+    {
+        out.append(requireIsType<T>(val, "Document"));
+    }
+    return out;
+}
+
+template <typename T>
+QVector<T> ensureIsArrayOf(const QJsonValue &value, const QString &what = "Value")
+{
+    const QJsonArray array = ensureIsType<QJsonArray>(value, QJsonArray(), what);
+    QVector<T> out;
+    for (const QJsonValue val : array)
+    {
+        out.append(requireIsType<T>(val, what));
+    }
+    return out;
+}
+
+template <typename T>
+QVector<T> ensureIsArrayOf(const QJsonValue &value, const QVector<T> default_, const QString &what = "Value")
+{
+    if (value.isUndefined())
+    {
+        return default_;
+    }
+    return ensureIsArrayOf<T>(value, what);
+}
+
+/// @throw JsonException
+template <typename T>
+QVector<T> requireIsArrayOf(const QJsonObject &parent, const QString &key, const QString &what = "__placeholder__")
+{
+    const QString localWhat = QString(what).replace("__placeholder__", '\'' + key + '\'');
+    if (!parent.contains(key))
+    {
+        throw JsonException(localWhat + "s parent does not contain " + localWhat);
+    }
+    return ensureIsArrayOf<T>(parent.value(key), localWhat);
+}
+
+template <typename T>
+QVector<T> ensureIsArrayOf(const QJsonObject &parent, const QString &key,
+                         const QVector<T> &default_ = QVector<T>(), const QString &what = "__placeholder__")
+{
+    const QString localWhat = QString(what).replace("__placeholder__", '\'' + key + '\'');
+    if (!parent.contains(key))
+    {
+        return default_;
+    }
+    return ensureIsArrayOf<T>(parent.value(key), default_, localWhat);
+}
+
+// this macro part could be replaced by variadic functions that just pass on their arguments, but that wouldn't work well with IDE helpers
+#define JSON_HELPERFUNCTIONS(NAME, TYPE) \
+    inline TYPE requireValue##NAME(const QJsonValue &value, const QString &what = "Value") \
+    { \
+        return requireIsType<TYPE>(value, what); \
+    } \
+    inline TYPE ensureValue##NAME(const QJsonValue &value, const TYPE default_ = TYPE(), const QString &what = "Value") \
+    { \
+        return ensureIsType<TYPE>(value, default_, what); \
+    } \
+    inline TYPE require##NAME(const QJsonObject &parent, const QString &key, const QString &what = "__placeholder__") \
+    { \
+        return requireIsType<TYPE>(parent, key, what); \
+    } \
+    inline TYPE ensure##NAME(const QJsonObject &parent, const QString &key, const TYPE default_ = TYPE(), const QString &what = "__placeholder") \
+    { \
+        return ensureIsType<TYPE>(parent, key, default_, what); \
+    }
+
+JSON_HELPERFUNCTIONS(Array, QJsonArray)
+JSON_HELPERFUNCTIONS(Object, QJsonObject)
+JSON_HELPERFUNCTIONS(JsonValue, QJsonValue)
+JSON_HELPERFUNCTIONS(String, QString)
+JSON_HELPERFUNCTIONS(Boolean, bool)
+JSON_HELPERFUNCTIONS(Double, double)
+JSON_HELPERFUNCTIONS(Integer, int)
+JSON_HELPERFUNCTIONS(DateTime, QDateTime)
+JSON_HELPERFUNCTIONS(Url, QUrl)
+JSON_HELPERFUNCTIONS(ByteArray, QByteArray)
+JSON_HELPERFUNCTIONS(Dir, QDir)
+JSON_HELPERFUNCTIONS(Uuid, QUuid)
+JSON_HELPERFUNCTIONS(Variant, QVariant)
+
+#undef JSON_HELPERFUNCTIONS
+
+}
+using JSONValidationError = Json::JsonException;

src/QObjectPtr.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <QObject>
+
+namespace details
+{
+struct DeleteQObjectLater
+{
+    void operator()(QObject *obj) const
+    {
+        obj->deleteLater();
+    }
+};
+}
+/**
+ * A unique pointer class with unique pointer semantics intended for derivates of QObject
+ * Calls deleteLater() instead of destroying the contained object immediately
+ */
+template<typename T> using unique_qobject_ptr = std::unique_ptr<T, details::DeleteQObjectLater>;
+
+/**
+ * A shared pointer class with shared pointer semantics intended for derivates of QObject
+ * Calls deleteLater() instead of destroying the contained object immediately
+ */
+template <typename T>
+class shared_qobject_ptr
+{
+public:
+    shared_qobject_ptr(){}
+    shared_qobject_ptr(T * wrap)
+    {
+        reset(wrap);
+    }
+    shared_qobject_ptr(const shared_qobject_ptr<T>& other)
+    {
+        m_ptr = other.m_ptr;
+    }
+    template<typename Derived>
+    shared_qobject_ptr(const shared_qobject_ptr<Derived> &other)
+    {
+        m_ptr = other.unwrap();
+    }
+
+public:
+    void reset(T * wrap)
+    {
+        using namespace std::placeholders;
+        m_ptr.reset(wrap, std::bind(&QObject::deleteLater, _1));
+    }
+    void reset(const shared_qobject_ptr<T> &other)
+    {
+        m_ptr = other.m_ptr;
+    }
+    void reset()
+    {
+        m_ptr.reset();
+    }
+    T * get() const
+    {
+        return m_ptr.get();
+    }
+    T * operator->() const
+    {
+        return m_ptr.get();
+    }
+    T & operator*() const
+    {
+        return *m_ptr.get();
+    }
+    operator bool() const
+    {
+        return m_ptr.get() != nullptr;
+    }
+    const std::shared_ptr <T> unwrap() const
+    {
+        return m_ptr;
+    }
+    bool operator==(const shared_qobject_ptr<T>& other) {
+        return m_ptr == other.m_ptr;
+    }
+    bool operator!=(const shared_qobject_ptr<T>& other) {
+        return m_ptr != other.m_ptr;
+    }
+
+private:
+    std::shared_ptr <T> m_ptr;
+};

src/mojang/PackageManifest.cpp

@@ -0,0 +1,434 @@
+/*
+ * Copyright 2020 Petr Mrázek
+ *
+ * This source is subject to the Microsoft Permissive License (MS-PL).
+ * Please see the COPYING.md file for more information.
+ */
+
+#include "PackageManifest.h"
+#include <Json.h>
+#include <QDir>
+#include <QDirIterator>
+#include <QCryptographicHash>
+#include <QDebug>
+
+#ifndef Q_OS_WIN32
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#endif
+
+namespace mojang_files {
+
+const Hash hash_of_empty_string = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
+
+int Path::compare(const Path& rhs) const
+{
+    auto left_cursor = begin();
+    auto left_end = end();
+    auto right_cursor = rhs.begin();
+    auto right_end = rhs.end();
+
+    while (left_cursor != left_end && right_cursor != right_end)
+    {
+        if(*left_cursor < *right_cursor)
+        {
+            return -1;
+        }
+        else if(*left_cursor > *right_cursor)
+        {
+            return 1;
+        }
+        left_cursor++;
+        right_cursor++;
+    }
+
+    if(left_cursor == left_end)
+    {
+        if(right_cursor == right_end)
+        {
+            return 0;
+        }
+        return -1;
+    }
+    return 1;
+}
+
+void Package::addFile(const Path& path, const File& file) {
+    addFolder(path.parent_path());
+    files[path] = file;
+}
+
+void Package::addFolder(Path folder) {
+    if(!folder.has_parent_path()) {
+        return;
+    }
+    do {
+        folders.insert(folder);
+        folder = folder.parent_path();
+    } while(folder.has_parent_path());
+}
+
+void Package::addLink(const Path& path, const Path& target) {
+    addFolder(path.parent_path());
+    symlinks[path] = target;
+}
+
+void Package::addSource(const FileSource& source) {
+    sources[source.hash] = source;
+}
+
+
+namespace {
+void fromJson(QJsonDocument & doc, Package & out) {
+    std::set<Path> seen_paths;
+    if (!doc.isObject())
+    {
+        throw JSONValidationError("file manifest is not an object");
+    }
+    QJsonObject root = doc.object();
+
+    auto filesObj = Json::ensureObject(root, "files");
+    auto iter = filesObj.begin();
+    while (iter != filesObj.end())
+    {
+        Path objectPath = Path(iter.key());
+        auto value = iter.value();
+        iter++;
+        if(seen_paths.count(objectPath)) {
+            throw JSONValidationError("duplicate path inside manifest, the manifest is invalid");
+        }
+        if (!value.isObject())
+        {
+            throw JSONValidationError("file entry inside manifest is not an an object");
+        }
+        seen_paths.insert(objectPath);
+
+        auto fileObject = value.toObject();
+        auto type = Json::requireString(fileObject, "type");
+        if(type == "directory") {
+            out.addFolder(objectPath);
+            continue;
+        }
+        else if(type == "file") {
+            FileSource bestSource;
+            File file;
+            file.executable = Json::ensureBoolean(fileObject, QString("executable"), false);
+            auto downloads = Json::requireObject(fileObject, "downloads");
+            for(auto iter2 = downloads.begin(); iter2 != downloads.end(); iter2++) {
+                FileSource source;
+
+                auto downloadObject = Json::requireValueObject(iter2.value());
+                source.hash = Json::requireString(downloadObject, "sha1");
+                source.size = Json::requireInteger(downloadObject, "size");
+                source.url = Json::requireString(downloadObject, "url");
+
+                auto compression = iter2.key();
+                if(compression == "raw") {
+                    file.hash = source.hash;
+                    file.size = source.size;
+                    source.compression = Compression::Raw;
+                }
+                else if (compression == "lzma") {
+                    source.compression = Compression::Lzma;
+                }
+                else {
+                    continue;
+                }
+                bestSource.upgrade(source);
+            }
+            if(bestSource.isBad()) {
+                throw JSONValidationError("No valid compression method for file " + iter.key());
+            }
+            out.addFile(objectPath, file);
+            out.addSource(bestSource);
+        }
+        else if(type == "link") {
+            auto target = Json::requireString(fileObject, "target");
+            out.symlinks[objectPath] = target;
+            out.addLink(objectPath, target);
+        }
+        else {
+            throw JSONValidationError("Invalid item type in manifest: " + type);
+        }
+    }
+    // make sure the containing folder exists
+    out.folders.insert(Path());
+}
+}
+
+Package Package::fromManifestContents(const QByteArray& contents)
+{
+    Package out;
+    try
+    {
+        auto doc = Json::requireDocument(contents, "Manifest");
+        fromJson(doc, out);
+        return out;
+    }
+    catch (const Exception &e)
+    {
+        qDebug() << QString("Unable to parse manifest: %1").arg(e.cause());
+        out.valid = false;
+        return out;
+    }
+}
+
+Package Package::fromManifestFile(const QString & filename) {
+    Package out;
+    try
+    {
+        auto doc = Json::requireDocument(filename, filename);
+        fromJson(doc, out);
+        return out;
+    }
+    catch (const Exception &e)
+    {
+        qDebug() << QString("Unable to parse manifest file %1: %2").arg(filename, e.cause());
+        out.valid = false;
+        return out;
+    }
+}
+
+#ifndef Q_OS_WIN32
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+namespace {
+// FIXME: Qt obscures symlink targets by making them absolute. that is useless. this is the workaround - we do it ourselves
+bool actually_read_symlink_target(const QString & filepath, Path & out)
+{
+    struct ::stat st;
+    // FIXME: here, we assume the native filesystem encoding. May the Gods have mercy upon our Souls.
+    QByteArray nativePath = filepath.toUtf8();
+    const char * filepath_cstr = nativePath.data();
+
+    if (lstat(filepath_cstr, &st) != 0)
+    {
+        return false;
+    }
+
+    auto size = st.st_size ? st.st_size + 1 : PATH_MAX;
+    std::string temp(size, '\0');
+    // because we don't realiably know how long the damn thing actually is, we loop and expand. POSIX is naff
+    do
+    {
+        auto link_length = ::readlink(filepath_cstr, &temp[0], temp.size());
+        if(link_length == -1)
+        {
+            return false;
+        }
+        if(std::string::size_type(link_length) < temp.size())
+        {
+            // buffer was long enough and we managed to read the link target. RETURN here.
+            temp.resize(link_length);
+            out = Path(QString::fromUtf8(temp.c_str()));
+            return true;
+        }
+        temp.resize(temp.size() * 2);
+    } while (true);
+}
+}
+#endif
+
+// FIXME: Qt filesystem abstraction is bad, but ... let's hope it doesn't break too much?
+// FIXME: The error handling is just DEFICIENT
+Package Package::fromInspectedFolder(const QString& folderPath)
+{
+    QDir root(folderPath);
+
+    Package out;
+    QDirIterator iterator(folderPath, QDir::NoDotAndDotDot | QDir::AllEntries | QDir::System | QDir::Hidden, QDirIterator::Subdirectories);
+    while(iterator.hasNext()) {
+        iterator.next();
+
+        auto fileInfo = iterator.fileInfo();
+        auto relPath = root.relativeFilePath(fileInfo.filePath());
+        // FIXME: this is probably completely busted on Windows anyway, so just disable it.
+        // Qt makes shit up and doesn't understand the platform details
+        // TODO: Actually use a filesystem library that isn't terrible and has decen license.
+        //       I only know one, and I wrote it. Sadly, currently proprietary. PAIN.
+#ifndef Q_OS_WIN32
+        if(fileInfo.isSymLink()) {
+            Path targetPath;
+            if(!actually_read_symlink_target(fileInfo.filePath(), targetPath)) {
+                qCritical() << "Folder inspection: Unknown filesystem object:" << fileInfo.absoluteFilePath();
+                out.valid = false;
+            }
+            out.addLink(relPath, targetPath);
+        }
+        else
+#endif
+        if(fileInfo.isDir()) {
+            out.addFolder(relPath);
+        }
+        else if(fileInfo.isFile()) {
+            File f;
+            f.executable = fileInfo.isExecutable();
+            f.size = fileInfo.size();
+            // FIXME: async / optimize the hashing
+            QFile input(fileInfo.absoluteFilePath());
+            if(!input.open(QIODevice::ReadOnly)) {
+                qCritical() << "Folder inspection: Failed to open file:" << fileInfo.absoluteFilePath();
+                out.valid = false;
+                break;
+            }
+            f.hash = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Sha1).toHex().constData();
+            out.addFile(relPath, f);
+        }
+        else {
+            // Something else... oh my
+            qCritical() << "Folder inspection: Unknown filesystem object:" << fileInfo.absoluteFilePath();
+            out.valid = false;
+            break;
+        }
+    }
+    out.folders.insert(Path("."));
+    out.valid = true;
+    return out;
+}
+
+namespace {
+struct shallow_first_sort
+{
+    bool operator()(const Path &lhs, const Path &rhs) const
+    {
+        auto lhs_depth = lhs.length();
+        auto rhs_depth = rhs.length();
+        if(lhs_depth < rhs_depth)
+        {
+            return true;
+        }
+        else if(lhs_depth == rhs_depth)
+        {
+            if(lhs < rhs)
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+};
+
+struct deep_first_sort
+{
+    bool operator()(const Path &lhs, const Path &rhs) const
+    {
+        auto lhs_depth = lhs.length();
+        auto rhs_depth = rhs.length();
+        if(lhs_depth > rhs_depth)
+        {
+            return true;
+        }
+        else if(lhs_depth == rhs_depth)
+        {
+            if(lhs < rhs)
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+};
+}
+
+UpdateOperations UpdateOperations::resolve(const Package& from, const Package& to)
+{
+    UpdateOperations out;
+
+    if(!from.valid || !to.valid) {
+        out.valid = false;
+        return out;
+    }
+
+    // Files
+    for(auto iter = from.files.begin(); iter != from.files.end(); iter++) {
+        const auto &current_hash = iter->second.hash;
+        const auto &current_executable = iter->second.executable;
+        const auto &path = iter->first;
+
+        auto iter2 = to.files.find(path);
+        if(iter2 == to.files.end()) {
+            // removed
+            out.deletes.push_back(path);
+            continue;
+        }
+        auto new_hash = iter2->second.hash;
+        auto new_executable = iter2->second.executable;
+        if (current_hash != new_hash) {
+            out.deletes.push_back(path);
+            out.downloads.emplace(
+                std::pair<Path, FileDownload>{
+                    path,
+                    FileDownload(to.sources.at(iter2->second.hash), iter2->second.executable)
+                }
+            );
+        }
+        else if (current_executable != new_executable) {
+            out.executable_fixes[path] = new_executable;
+        }
+    }
+    for(auto iter = to.files.begin(); iter != to.files.end(); iter++) {
+        auto path = iter->first;
+        if(!from.files.count(path)) {
+            out.downloads.emplace(
+                std::pair<Path, FileDownload>{
+                    path,
+                    FileDownload(to.sources.at(iter->second.hash), iter->second.executable)
+                }
+            );
+        }
+    }
+
+    // Folders
+    std::set<Path, deep_first_sort> remove_folders;
+    std::set<Path, shallow_first_sort> make_folders;
+    for(auto from_path: from.folders) {
+        auto iter = to.folders.find(from_path);
+        if(iter == to.folders.end()) {
+            remove_folders.insert(from_path);
+        }
+    }
+    for(auto & rmdir: remove_folders) {
+        out.rmdirs.push_back(rmdir);
+    }
+    for(auto to_path: to.folders) {
+        auto iter = from.folders.find(to_path);
+        if(iter == from.folders.end()) {
+            make_folders.insert(to_path);
+        }
+    }
+    for(auto & mkdir: make_folders) {
+        out.mkdirs.push_back(mkdir);
+    }
+
+    // Symlinks
+    for(auto iter = from.symlinks.begin(); iter != from.symlinks.end(); iter++) {
+        const auto &current_target = iter->second;
+        const auto &path = iter->first;
+
+        auto iter2 = to.symlinks.find(path);
+        if(iter2 == to.symlinks.end()) {
+            // removed
+            out.deletes.push_back(path);
+            continue;
+        }
+        const auto &new_target = iter2->second;
+        if (current_target != new_target) {
+            out.deletes.push_back(path);
+            out.mklinks[path] = iter2->second;
+        }
+    }
+    for(auto iter = to.symlinks.begin(); iter != to.symlinks.end(); iter++) {
+        auto path = iter->first;
+        if(!from.symlinks.count(path)) {
+            out.mklinks[path] = iter->second;
+        }
+    }
+    out.valid = true;
+    return out;
+}
+
+}

src/mojang/PackageManifest.h

@@ -0,0 +1,178 @@
+/*
+ * Copyright 2020 Petr Mrázek
+ *
+ * This source is subject to the Microsoft Permissive License (MS-PL).
+ * Please see the COPYING.md file for more information.
+ */
+
+#pragma once
+
+#include <QString>
+#include <map>
+#include <set>
+#include <QStringList>
+#include "tasks/Task.h"
+
+namespace mojang_files {
+
+using Hash = QString;
+extern const Hash empty_hash;
+
+// simple-ish path implementation. assumes always relative and does not allow '..' entries
+class Path
+{
+public:
+    using parts_type = QStringList;
+
+    Path() = default;
+    Path(QString string) {
+        auto parts_in = string.split('/');
+        for(auto & part: parts_in) {
+            if(part.isEmpty() || part == ".") {
+                continue;
+            }
+            if(part == "..") {
+                if(parts.size()) {
+                    parts.pop_back();
+                }
+                continue;
+            }
+            parts.push_back(part);
+        }
+    }
+
+    bool has_parent_path() const
+    {
+        return parts.size() > 0;
+    }
+
+    Path parent_path() const
+    {
+        if (parts.empty())
+            return Path();
+        return Path(parts.begin(), std::prev(parts.end()));
+    }
+
+    bool empty() const
+    {
+        return parts.empty();
+    }
+
+    int length() const
+    {
+        return parts.length();
+    }
+
+    bool operator==(const Path & rhs) const {
+        return parts == rhs.parts;
+    }
+
+    bool operator!=(const Path & rhs) const {
+        return parts != rhs.parts;
+    }
+
+    inline bool operator<(const Path& rhs) const
+    {
+        return compare(rhs) < 0;
+    }
+
+    parts_type::const_iterator begin() const
+    {
+        return parts.begin();
+    }
+
+    parts_type::const_iterator end() const
+    {
+        return parts.end();
+    }
+
+    QString toString() const {
+        return parts.join("/");
+    }
+
+private:
+    Path(const parts_type::const_iterator & start, const parts_type::const_iterator & end) {
+        auto cursor = start;
+        while(cursor != end) {
+            parts.push_back(*cursor);
+            cursor++;
+        }
+    }
+    int compare(const Path& p) const;
+
+    parts_type parts;
+};
+
+
+enum class Compression {
+    Raw,
+    Lzma,
+    Unknown
+};
+
+
+struct FileSource
+{
+    Compression compression = Compression::Unknown;
+    Hash hash;
+    QString url;
+    std::size_t size = 0;
+    void upgrade(const FileSource & other) {
+        if(compression == Compression::Unknown || other.size < size) {
+            *this = other;
+        }
+    }
+    bool isBad() const {
+        return compression == Compression::Unknown;
+    }
+};
+
+struct File
+{
+    Hash hash;
+    bool executable;
+    std::uint64_t size = 0;
+};
+
+struct Package {
+    static Package fromInspectedFolder(const QString &folderPath);
+    static Package fromManifestFile(const QString &path);
+    static Package fromManifestContents(const QByteArray& contents);
+
+    explicit operator bool() const
+    {
+        return valid;
+    }
+    void addFolder(Path folder);
+    void addFile(const Path & path, const File & file);
+    void addLink(const Path & path, const Path & target);
+    void addSource(const FileSource & source);
+
+    std::map<Hash, FileSource> sources;
+    bool valid = true;
+    std::set<Path> folders;
+    std::map<Path, File> files;
+    std::map<Path, Path> symlinks;
+};
+
+struct FileDownload : FileSource
+{
+    FileDownload(const FileSource& source, bool executable) {
+        static_cast<FileSource &> (*this) = source;
+        this->executable = executable;
+    }
+    bool executable = false;
+};
+
+struct UpdateOperations {
+    static UpdateOperations resolve(const Package & from, const Package & to);
+    bool valid = false;
+    std::vector<Path> deletes;
+    std::vector<Path> rmdirs;
+    std::vector<Path> mkdirs;
+    std::map<Path, FileDownload> downloads;
+    std::map<Path, Path> mklinks;
+    std::map<Path, bool> executable_fixes;
+};
+
+}

src/mojang/PackageManifest_test.cpp

@@ -0,0 +1,351 @@
+/*
+ * Copyright 2020 Petr Mrázek
+ *
+ * This source is subject to the Microsoft Permissive License (MS-PL).
+ * Please see the COPYING.md file for more information.
+ */
+
+#include <QTest>
+#include <QDebug>
+#include "TestUtil.h"
+
+#include "mojang/PackageManifest.h"
+
+using namespace mojang_files;
+
+QDebug operator<<(QDebug debug, const Path &path)
+{
+    debug << path.toString();
+    return debug;
+}
+
+class PackageManifestTest : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void test_parse();
+    void test_parse_file();
+    void test_inspect();
+#ifndef Q_OS_WIN32
+    void test_inspect_symlinks();
+#endif
+    void mkdir_deep();
+    void rmdir_deep();
+
+    void identical_file();
+    void changed_file();
+    void added_file();
+    void removed_file();
+};
+
+namespace {
+QByteArray basic_manifest = R"END(
+{
+    "files": {
+        "a/b.txt": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/b.txt",
+                    "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                    "size": 0
+                }
+            },
+            "executable": true
+        },
+        "a/b/c": {
+            "type": "directory"
+        },
+        "a/b/c.txt": {
+            "type": "link",
+            "target": "../b.txt"
+        }
+    }
+}
+)END";
+}
+
+void PackageManifestTest::test_parse()
+{
+    auto manifest = Package::fromManifestContents(basic_manifest);
+    QVERIFY(manifest.valid == true);
+    QVERIFY(manifest.files.size() == 1);
+    QVERIFY(manifest.files.count(Path("a/b.txt")));
+    auto &file = manifest.files[Path("a/b.txt")];
+    QVERIFY(file.executable == true);
+    QVERIFY(file.hash == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
+    QVERIFY(file.size == 0);
+    QVERIFY(manifest.folders.size() == 4);
+    QVERIFY(manifest.folders.count(Path(".")));
+    QVERIFY(manifest.folders.count(Path("a")));
+    QVERIFY(manifest.folders.count(Path("a/b")));
+    QVERIFY(manifest.folders.count(Path("a/b/c")));
+    QVERIFY(manifest.symlinks.size() == 1);
+    auto symlinkPath = Path("a/b/c.txt");
+    QVERIFY(manifest.symlinks.count(symlinkPath));
+    auto &symlink = manifest.symlinks[symlinkPath];
+    QVERIFY(symlink == Path("../b.txt"));
+    QVERIFY(manifest.sources.size() == 1);
+}
+
+void PackageManifestTest::test_parse_file() {
+    auto path = QFINDTESTDATA("testdata/1.8.0_202-x64.json");
+    auto manifest = Package::fromManifestFile(path);
+    QVERIFY(manifest.valid == true);
+}
+
+
+void PackageManifestTest::test_inspect() {
+    auto path = QFINDTESTDATA("testdata/inspect_win/");
+    auto manifest = Package::fromInspectedFolder(path);
+    QVERIFY(manifest.valid == true);
+    QVERIFY(manifest.files.size() == 2);
+    QVERIFY(manifest.files.count(Path("a/b.txt")));
+    auto &file1 = manifest.files[Path("a/b.txt")];
+    QVERIFY(file1.executable == false);
+    QVERIFY(file1.hash == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
+    QVERIFY(file1.size == 0);
+    QVERIFY(manifest.files.count(Path("a/b/b.txt")));
+    auto &file2 = manifest.files[Path("a/b/b.txt")];
+    QVERIFY(file2.executable == false);
+    QVERIFY(file2.hash == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
+    QVERIFY(file2.size == 0);
+    QVERIFY(manifest.folders.size() == 3);
+    QVERIFY(manifest.folders.count(Path(".")));
+    QVERIFY(manifest.folders.count(Path("a")));
+    QVERIFY(manifest.folders.count(Path("a/b")));
+    QVERIFY(manifest.symlinks.size() == 0);
+}
+
+#ifndef Q_OS_WIN32
+void PackageManifestTest::test_inspect_symlinks() {
+    auto path = QFINDTESTDATA("testdata/inspect/");
+    auto manifest = Package::fromInspectedFolder(path);
+    QVERIFY(manifest.valid == true);
+    QVERIFY(manifest.files.size() == 1);
+    QVERIFY(manifest.files.count(Path("a/b.txt")));
+    auto &file = manifest.files[Path("a/b.txt")];
+    QVERIFY(file.executable == true);
+    QVERIFY(file.hash == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
+    QVERIFY(file.size == 0);
+    QVERIFY(manifest.folders.size() == 3);
+    QVERIFY(manifest.folders.count(Path(".")));
+    QVERIFY(manifest.folders.count(Path("a")));
+    QVERIFY(manifest.folders.count(Path("a/b")));
+    QVERIFY(manifest.symlinks.size() == 1);
+    QVERIFY(manifest.symlinks.count(Path("a/b/b.txt")));
+    qDebug() << manifest.symlinks[Path("a/b/b.txt")];
+    QVERIFY(manifest.symlinks[Path("a/b/b.txt")] == Path("../b.txt"));
+}
+#endif
+
+void PackageManifestTest::mkdir_deep() {
+
+    Package from;
+    auto to = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/e": {
+            "type": "directory"
+        }
+    }
+}
+)END");
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 0);
+    QVERIFY(operations.rmdirs.size() == 0);
+
+    QVERIFY(operations.mkdirs.size() == 6);
+    QVERIFY(operations.mkdirs[0] == Path("."));
+    QVERIFY(operations.mkdirs[1] == Path("a"));
+    QVERIFY(operations.mkdirs[2] == Path("a/b"));
+    QVERIFY(operations.mkdirs[3] == Path("a/b/c"));
+    QVERIFY(operations.mkdirs[4] == Path("a/b/c/d"));
+    QVERIFY(operations.mkdirs[5] == Path("a/b/c/d/e"));
+
+    QVERIFY(operations.downloads.size() == 0);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+void PackageManifestTest::rmdir_deep() {
+
+    Package to;
+    auto from = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/e": {
+            "type": "directory"
+        }
+    }
+}
+)END");
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 0);
+
+    QVERIFY(operations.rmdirs.size() == 6);
+    QVERIFY(operations.rmdirs[0] == Path("a/b/c/d/e"));
+    QVERIFY(operations.rmdirs[1] == Path("a/b/c/d"));
+    QVERIFY(operations.rmdirs[2] == Path("a/b/c"));
+    QVERIFY(operations.rmdirs[3] == Path("a/b"));
+    QVERIFY(operations.rmdirs[4] == Path("a"));
+    QVERIFY(operations.rmdirs[5] == Path("."));
+
+    QVERIFY(operations.mkdirs.size() == 0);
+    QVERIFY(operations.downloads.size() == 0);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+void PackageManifestTest::identical_file() {
+    QByteArray manifest = R"END(
+{
+    "files": {
+        "a/b/c/d/empty.txt": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/empty.txt",
+                    "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                    "size": 0
+                }
+            },
+            "executable": false
+        }
+    }
+}
+)END";
+    auto from = Package::fromManifestContents(manifest);
+    auto to = Package::fromManifestContents(manifest);
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 0);
+    QVERIFY(operations.rmdirs.size() == 0);
+    QVERIFY(operations.mkdirs.size() == 0);
+    QVERIFY(operations.downloads.size() == 0);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+void PackageManifestTest::changed_file() {
+    auto from = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/file": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/empty.txt",
+                    "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                    "size": 0
+                }
+            },
+            "executable": false
+        }
+    }
+}
+)END");
+    auto to = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/file": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/space.txt",
+                    "sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
+                    "size": 1
+                }
+            },
+            "executable": false
+        }
+    }
+}
+)END");
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 1);
+    QCOMPARE(operations.deletes[0], Path("a/b/c/d/file"));
+    QVERIFY(operations.rmdirs.size() == 0);
+    QVERIFY(operations.mkdirs.size() == 0);
+    QVERIFY(operations.downloads.size() == 1);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+void PackageManifestTest::added_file() {
+    auto from = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d": {
+            "type": "directory"
+        }
+    }
+}
+)END");
+    auto to = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/file": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/space.txt",
+                    "sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
+                    "size": 1
+                }
+            },
+            "executable": false
+        }
+    }
+}
+)END");
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 0);
+    QVERIFY(operations.rmdirs.size() == 0);
+    QVERIFY(operations.mkdirs.size() == 0);
+    QVERIFY(operations.downloads.size() == 1);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+void PackageManifestTest::removed_file() {
+    auto from = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d/file": {
+            "type": "file",
+            "downloads": {
+                "raw": {
+                    "url": "http://dethware.org/space.txt",
+                    "sha1": "dd122581c8cd44d0227f9c305581ffcb4b6f1b46",
+                    "size": 1
+                }
+            },
+            "executable": false
+        }
+    }
+}
+)END");
+    auto to = Package::fromManifestContents(R"END(
+{
+    "files": {
+        "a/b/c/d": {
+            "type": "directory"
+        }
+    }
+}
+)END");
+    auto operations = UpdateOperations::resolve(from, to);
+    QVERIFY(operations.deletes.size() == 1);
+    QCOMPARE(operations.deletes[0], Path("a/b/c/d/file"));
+    QVERIFY(operations.rmdirs.size() == 0);
+    QVERIFY(operations.mkdirs.size() == 0);
+    QVERIFY(operations.downloads.size() == 0);
+    QVERIFY(operations.mklinks.size() == 0);
+    QVERIFY(operations.executable_fixes.size() == 0);
+}
+
+QTEST_GUILESS_MAIN(PackageManifestTest)
+
+#include "PackageManifest_test.moc"
+

src/mojang/testdata/inspect/a/b/b.txt

@@ -0,0 +1 @@
+../b.txt

src/net/ByteArraySink.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "Sink.h"
+
+namespace Net {
+/*
+ * Sink object for downloads that uses an external QByteArray it doesn't own as a target.
+ */
+class ByteArraySink : public Sink
+{
+public:
+    ByteArraySink(QByteArray *output)
+        :m_output(output)
+    {
+        // nil
+    };
+
+    virtual ~ByteArraySink()
+    {
+        // nil
+    }
+
+public:
+    JobStatus init(QNetworkRequest & request) override
+    {
+        m_output->clear();
+        if(initAllValidators(request))
+            return Job_InProgress;
+        return Job_Failed;
+    };
+
+    JobStatus write(QByteArray & data) override
+    {
+        m_output->append(data);
+        if(writeAllValidators(data))
+            return Job_InProgress;
+        return Job_Failed;
+    }
+
+    JobStatus abort() override
+    {
+        m_output->clear();
+        failAllValidators();
+        return Job_Failed;
+    }
+
+    JobStatus finalize(QNetworkReply &reply) override
+    {
+        if(finalizeAllValidators(reply))
+            return Job_Finished;
+        return Job_Failed;
+    }
+
+    bool hasLocalData() override
+    {
+        return false;
+    }
+
+private:
+    QByteArray * m_output;
+};
+}

src/net/ChecksumValidator.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "Validator.h"
+#include <QCryptographicHash>
+#include <memory>
+#include <QFile>
+
+namespace Net {
+class ChecksumValidator: public Validator
+{
+public: /* con/des */
+    ChecksumValidator(QCryptographicHash::Algorithm algorithm, QByteArray expected = QByteArray())
+        :m_checksum(algorithm), m_expected(expected)
+    {
+    };
+    virtual ~ChecksumValidator() {};
+
+public: /* methods */
+    bool init(QNetworkRequest &) override
+    {
+        m_checksum.reset();
+        return true;
+    }
+    bool write(QByteArray & data) override
+    {
+        m_checksum.addData(data);
+        return true;
+    }
+    bool abort() override
+    {
+        return true;
+    }
+    bool validate(QNetworkReply &) override
+    {
+        if(m_expected.size() && m_expected != hash())
+        {
+            qWarning() << "Checksum mismatch, download is bad.";
+            return false;
+        }
+        return true;
+    }
+    QByteArray hash()
+    {
+        return m_checksum.result();
+    }
+    void setExpected(QByteArray expected)
+    {
+        m_expected = expected;
+    }
+
+private: /* data */
+    QCryptographicHash m_checksum;
+    QByteArray m_expected;
+};
+}

src/net/Download.cpp

@@ -0,0 +1,296 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Download.h"
+
+#include <QFileInfo>
+#include <QDateTime>
+#include <QDebug>
+
+#include "FileSystem.h"
+#include "ChecksumValidator.h"
+#include "ByteArraySink.h"
+#include "FileSink.h"
+
+namespace Net {
+
+Download::Download():NetAction()
+{
+    m_status = Job_NotStarted;
+}
+
+Download::Ptr Download::makeByteArray(QUrl url, QByteArray *output, Options options)
+{
+    Download * dl = new Download();
+    dl->m_url = url;
+    dl->m_options = options;
+    dl->m_sink.reset(new ByteArraySink(output));
+    return dl;
+}
+
+Download::Ptr Download::makeFile(QUrl url, QString path, Options options)
+{
+    Download * dl = new Download();
+    dl->m_url = url;
+    dl->m_options = options;
+    dl->m_sink.reset(new FileSink(path));
+    return dl;
+}
+
+void Download::addValidator(Validator * v)
+{
+    m_sink->addValidator(v);
+}
+
+void Download::startImpl()
+{
+    if(m_status == Job_Aborted)
+    {
+        qWarning() << "Attempt to start an aborted Download:" << m_url.toString();
+        emit aborted(m_index_within_job);
+        return;
+    }
+    QNetworkRequest request(m_url);
+    m_status = m_sink->init(request);
+    switch(m_status)
+    {
+        case Job_Finished:
+            emit succeeded(m_index_within_job);
+            qDebug() << "Download cache hit " << m_url.toString();
+            return;
+        case Job_InProgress:
+            qDebug() << "Downloading " << m_url.toString();
+            break;
+        case Job_Failed_Proceed: // this is meaningless in this context. We do need a sink.
+        case Job_NotStarted:
+        case Job_Failed:
+            emit failed(m_index_within_job);
+            return;
+        case Job_Aborted:
+            return;
+    }
+
+    request.setHeader(QNetworkRequest::UserAgentHeader, "Squeezer");
+
+    QNetworkReply *rep = m_network->get(request);
+
+    m_reply.reset(rep);
+    connect(rep, SIGNAL(downloadProgress(qint64, qint64)), SLOT(downloadProgress(qint64, qint64)));
+    connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
+    connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError)));
+    connect(rep, &QNetworkReply::sslErrors, this, &Download::sslErrors);
+    connect(rep, &QNetworkReply::readyRead, this, &Download::downloadReadyRead);
+}
+
+void Download::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+    m_total_progress = bytesTotal;
+    m_progress = bytesReceived;
+    emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
+}
+
+void Download::downloadError(QNetworkReply::NetworkError error)
+{
+    if(error == QNetworkReply::OperationCanceledError)
+    {
+        qCritical() << "Aborted " << m_url.toString();
+        m_status = Job_Aborted;
+    }
+    else
+    {
+        if(m_options & Option::AcceptLocalFiles)
+        {
+            if(m_sink->hasLocalData())
+            {
+                m_status = Job_Failed_Proceed;
+                return;
+            }
+        }
+        // error happened during download.
+        qCritical() << "Failed " << m_url.toString() << " with reason " << error;
+        m_status = Job_Failed;
+    }
+}
+
+void Download::sslErrors(const QList<QSslError> & errors)
+{
+    int i = 1;
+    for (auto error : errors)
+    {
+        qCritical() << "Download" << m_url.toString() << "SSL Error #" << i << " : " << error.errorString();
+        auto cert = error.certificate();
+        qCritical() << "Certificate in question:\n" << cert.toText();
+        i++;
+    }
+}
+
+bool Download::handleRedirect()
+{
+    QUrl redirect = m_reply->header(QNetworkRequest::LocationHeader).toUrl();
+    if(!redirect.isValid())
+    {
+        if(!m_reply->hasRawHeader("Location"))
+        {
+            // no redirect -> it's fine to continue
+            return false;
+        }
+        // there is a Location header, but it's not correct. we need to apply some workarounds...
+        QByteArray redirectBA = m_reply->rawHeader("Location");
+        if(redirectBA.size() == 0)
+        {
+            // empty, yet present redirect header? WTF?
+            return false;
+        }
+        QString redirectStr = QString::fromUtf8(redirectBA);
+
+        if(redirectStr.startsWith("//"))
+        {
+            /*
+             * IF the URL begins with //, we need to insert the URL scheme.
+             * See: https://bugreports.qt.io/browse/QTBUG-41061
+             * See: http://tools.ietf.org/html/rfc3986#section-4.2
+             */
+            redirectStr = m_reply->url().scheme() + ":" + redirectStr;
+        }
+        else if(redirectStr.startsWith("/"))
+        {
+            /*
+             * IF the URL begins with /, we need to process it as a relative URL
+             */
+            auto url = m_reply->url();
+            url.setPath(redirectStr, QUrl::TolerantMode);
+            redirectStr = url.toString();
+        }
+
+        /*
+         * Next, make sure the URL is parsed in tolerant mode. Qt doesn't parse the location header in tolerant mode, which causes issues.
+         * FIXME: report Qt bug for this
+         */
+        redirect = QUrl(redirectStr, QUrl::TolerantMode);
+        if(!redirect.isValid())
+        {
+            qWarning() << "Failed to parse redirect URL:" << redirectStr;
+            downloadError(QNetworkReply::ProtocolFailure);
+            return false;
+        }
+        qDebug() << "Fixed location header:" << redirect;
+    }
+    else
+    {
+        qDebug() << "Location header:" << redirect;
+    }
+
+    m_url = QUrl(redirect.toString());
+    qDebug() << "Following redirect to " << m_url.toString();
+    start(m_network);
+    return true;
+}
+
+
+void Download::downloadFinished()
+{
+    // handle HTTP redirection first
+    if(handleRedirect())
+    {
+        qDebug() << "Download redirected:" << m_url.toString();
+        return;
+    }
+
+    // if the download failed before this point ...
+    if (m_status == Job_Failed_Proceed)
+    {
+        qDebug() << "Download failed but we are allowed to proceed:" << m_url.toString();
+        m_sink->abort();
+        m_reply.reset();
+        emit succeeded(m_index_within_job);
+        return;
+    }
+    else if (m_status == Job_Failed)
+    {
+        qDebug() << "Download failed in previous step:" << m_url.toString();
+        m_sink->abort();
+        m_reply.reset();
+        emit failed(m_index_within_job);
+        return;
+    }
+    else if(m_status == Job_Aborted)
+    {
+        qDebug() << "Download aborted in previous step:" << m_url.toString();
+        m_sink->abort();
+        m_reply.reset();
+        emit aborted(m_index_within_job);
+        return;
+    }
+
+    // make sure we got all the remaining data, if any
+    auto data = m_reply->readAll();
+    if(data.size())
+    {
+        qDebug() << "Writing extra" << data.size() << "bytes to" << m_target_path;
+        m_status = m_sink->write(data);
+    }
+
+    // otherwise, finalize the whole graph
+    m_status = m_sink->finalize(*m_reply.get());
+    if (m_status != Job_Finished)
+    {
+        qDebug() << "Download failed to finalize:" << m_url.toString();
+        m_sink->abort();
+        m_reply.reset();
+        emit failed(m_index_within_job);
+        return;
+    }
+    m_reply.reset();
+    qDebug() << "Download succeeded:" << m_url.toString();
+    emit succeeded(m_index_within_job);
+}
+
+void Download::downloadReadyRead()
+{
+    if(m_status == Job_InProgress)
+    {
+        auto data = m_reply->readAll();
+        m_status = m_sink->write(data);
+        if(m_status == Job_Failed)
+        {
+            qCritical() << "Failed to process response chunk for " << m_target_path;
+        }
+        // qDebug() << "Download" << m_url.toString() << "gained" << data.size() << "bytes";
+    }
+    else
+    {
+        qCritical() << "Cannot write to " << m_target_path << ", illegal status" << m_status;
+    }
+}
+
+}
+
+bool Net::Download::abort()
+{
+    if(m_reply)
+    {
+        m_reply->abort();
+    }
+    else
+    {
+        m_status = Job_Aborted;
+    }
+    return true;
+}
+
+bool Net::Download::canAbort()
+{
+    return true;
+}

src/net/Download.h

@@ -0,0 +1,75 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "NetAction.h"
+#include "Validator.h"
+#include "Sink.h"
+
+#include "QObjectPtr.h"
+
+namespace Net {
+class Download : public NetAction
+{
+    Q_OBJECT
+
+public: /* types */
+    typedef shared_qobject_ptr<class Download> Ptr;
+    enum class Option
+    {
+        NoOptions = 0,
+        AcceptLocalFiles = 1
+    };
+    Q_DECLARE_FLAGS(Options, Option)
+
+protected: /* con/des */
+    explicit Download();
+public:
+    virtual ~Download(){};
+    static Download::Ptr makeByteArray(QUrl url, QByteArray *output, Options options = Option::NoOptions);
+    static Download::Ptr makeFile(QUrl url, QString path, Options options = Option::NoOptions);
+
+public: /* methods */
+    QString getTargetFilepath()
+    {
+        return m_target_path;
+    }
+    void addValidator(Validator * v);
+    bool abort() override;
+    bool canAbort() override;
+
+private: /* methods */
+    bool handleRedirect();
+
+protected slots:
+    void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
+    void downloadError(QNetworkReply::NetworkError error) override;
+    void sslErrors(const QList<QSslError> & errors);
+    void downloadFinished() override;
+    void downloadReadyRead() override;
+
+public slots:
+    void startImpl() override;
+
+private: /* data */
+    // FIXME: remove this, it has no business being here.
+    QString m_target_path;
+    std::unique_ptr<Sink> m_sink;
+    Options m_options;
+};
+}
+
+Q_DECLARE_OPERATORS_FOR_FLAGS(Net::Download::Options)

src/net/FileSink.cpp

@@ -0,0 +1,114 @@
+#include "FileSink.h"
+#include <QFile>
+#include <QFileInfo>
+#include "FileSystem.h"
+
+namespace Net {
+
+FileSink::FileSink(QString filename)
+    :m_filename(filename)
+{
+    // nil
+}
+
+FileSink::~FileSink()
+{
+    // nil
+}
+
+JobStatus FileSink::init(QNetworkRequest& request)
+{
+    auto result = initCache(request);
+    if(result != Job_InProgress)
+    {
+        return result;
+    }
+    // create a new save file and open it for writing
+    if (!FS::ensureFilePathExists(m_filename))
+    {
+        qCritical() << "Could not create folder for " + m_filename;
+        return Job_Failed;
+    }
+    wroteAnyData = false;
+    m_output_file.reset(new QSaveFile(m_filename));
+    if (!m_output_file->open(QIODevice::WriteOnly))
+    {
+        qCritical() << "Could not open " + m_filename + " for writing";
+        return Job_Failed;
+    }
+
+    if(initAllValidators(request))
+        return Job_InProgress;
+    return Job_Failed;
+}
+
+JobStatus FileSink::initCache(QNetworkRequest &)
+{
+    return Job_InProgress;
+}
+
+JobStatus FileSink::write(QByteArray& data)
+{
+    if (!writeAllValidators(data) || m_output_file->write(data) != data.size())
+    {
+        qCritical() << "Failed writing into " + m_filename;
+        m_output_file->cancelWriting();
+        m_output_file.reset();
+        wroteAnyData = false;
+        return Job_Failed;
+    }
+    wroteAnyData = true;
+    return Job_InProgress;
+}
+
+JobStatus FileSink::abort()
+{
+    m_output_file->cancelWriting();
+    failAllValidators();
+    return Job_Failed;
+}
+
+JobStatus FileSink::finalize(QNetworkReply& reply)
+{
+    bool gotFile = false;
+    QVariant statusCodeV = reply.attribute(QNetworkRequest::HttpStatusCodeAttribute);
+    bool validStatus = false;
+    int statusCode = statusCodeV.toInt(&validStatus);
+    if(validStatus)
+    {
+        // this leaves out 304 Not Modified
+        gotFile = statusCode == 200 || statusCode == 203;
+    }
+    // if we wrote any data to the save file, we try to commit the data to the real file.
+    // if it actually got a proper file, we write it even if it was empty
+    if (gotFile || wroteAnyData)
+    {
+        // ask validators for data consistency
+        // we only do this for actual downloads, not 'your data is still the same' cache hits
+        if(!finalizeAllValidators(reply))
+            return Job_Failed;
+        // nothing went wrong...
+        if (!m_output_file->commit())
+        {
+            qCritical() << "Failed to commit changes to " << m_filename;
+            m_output_file->cancelWriting();
+            return Job_Failed;
+        }
+    }
+    // then get rid of the save file
+    m_output_file.reset();
+
+    return finalizeCache(reply);
+}
+
+JobStatus FileSink::finalizeCache(QNetworkReply &)
+{
+    return Job_Finished;
+}
+
+bool FileSink::hasLocalData()
+{
+    QFileInfo info(m_filename);
+    return info.exists() && info.size() != 0;
+}
+}

src/net/FileSink.h

@@ -0,0 +1,28 @@
+#pragma once
+#include "Sink.h"
+#include <QSaveFile>
+
+namespace Net {
+class FileSink : public Sink
+{
+public: /* con/des */
+    FileSink(QString filename);
+    virtual ~FileSink();
+
+public: /* methods */
+    JobStatus init(QNetworkRequest & request) override;
+    JobStatus write(QByteArray & data) override;
+    JobStatus abort() override;
+    JobStatus finalize(QNetworkReply & reply) override;
+    bool hasLocalData() override;
+
+protected: /* methods */
+    virtual JobStatus initCache(QNetworkRequest &);
+    virtual JobStatus finalizeCache(QNetworkReply &reply);
+
+protected: /* data */
+    QString m_filename;
+    bool wroteAnyData = false;
+    std::unique_ptr<QSaveFile> m_output_file;
+};
+}

src/net/Mode.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace Net
+{
+enum class Mode
+{
+    Offline,
+    Online
+};
+}

src/net/NetAction.h

@@ -0,0 +1,122 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <QObject>
+#include <QUrl>
+#include <memory>
+#include <QNetworkReply>
+#include <QObjectPtr.h>
+
+enum JobStatus
+{
+    Job_NotStarted,
+    Job_InProgress,
+    Job_Finished,
+    Job_Failed,
+    Job_Aborted,
+    /*
+     * FIXME: @NUKE this confuses the task failing with us having a fallback in the form of local data. Clear up the confusion.
+     * Same could be true for aborted task - the presence of pre-existing result is a separate concern
+     */
+    Job_Failed_Proceed
+};
+
+class NetAction : public QObject
+{
+    Q_OBJECT
+protected:
+    explicit NetAction() : QObject(nullptr) {};
+
+public:
+    using Ptr = shared_qobject_ptr<NetAction>;
+
+    virtual ~NetAction() {};
+
+    bool isRunning() const
+    {
+        return m_status == Job_InProgress;
+    }
+    bool isFinished() const
+    {
+        return m_status >= Job_Finished;
+    }
+    bool wasSuccessful() const
+    {
+        return m_status == Job_Finished || m_status == Job_Failed_Proceed;
+    }
+
+    qint64 totalProgress() const
+    {
+        return m_total_progress;
+    }
+    qint64 currentProgress() const
+    {
+        return m_progress;
+    }
+    virtual bool abort()
+    {
+        return false;
+    }
+    virtual bool canAbort()
+    {
+        return false;
+    }
+    QUrl url()
+    {
+        return m_url;
+    }
+
+signals:
+    void started(int index);
+    void netActionProgress(int index, qint64 current, qint64 total);
+    void succeeded(int index);
+    void failed(int index);
+    void aborted(int index);
+
+protected slots:
+    virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
+    virtual void downloadError(QNetworkReply::NetworkError error) = 0;
+    virtual void downloadFinished() = 0;
+    virtual void downloadReadyRead() = 0;
+
+public slots:
+    void start(shared_qobject_ptr<QNetworkAccessManager> network) {
+        m_network = network;
+        startImpl();
+    }
+
+protected:
+    virtual void startImpl() = 0;
+
+public:
+    shared_qobject_ptr<QNetworkAccessManager> m_network;
+
+    /// index within the parent job, FIXME: nuke
+    int m_index_within_job = 0;
+
+    /// the network reply
+    unique_qobject_ptr<QNetworkReply> m_reply;
+
+    /// source URL
+    QUrl m_url;
+
+    qint64 m_progress = 0;
+    qint64 m_total_progress = 1;
+
+protected:
+    JobStatus m_status = Job_NotStarted;
+};

src/net/NetJob.cpp

@@ -0,0 +1,218 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "NetJob.h"
+#include "Download.h"
+
+#include <QDebug>
+
+void NetJob::partSucceeded(int index)
+{
+    // do progress. all slots are 1 in size at least
+    auto &slot = parts_progress[index];
+    partProgress(index, slot.total_progress, slot.total_progress);
+
+    m_doing.remove(index);
+    m_done.insert(index);
+    downloads[index].get()->disconnect(this);
+    startMoreParts();
+}
+
+void NetJob::partFailed(int index)
+{
+    m_doing.remove(index);
+    auto &slot = parts_progress[index];
+    if (slot.failures == 3)
+    {
+        m_failed.insert(index);
+    }
+    else
+    {
+        slot.failures++;
+        m_todo.enqueue(index);
+    }
+    downloads[index].get()->disconnect(this);
+    startMoreParts();
+}
+
+void NetJob::partAborted(int index)
+{
+    m_aborted = true;
+    m_doing.remove(index);
+    m_failed.insert(index);
+    downloads[index].get()->disconnect(this);
+    startMoreParts();
+}
+
+void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
+{
+    auto &slot = parts_progress[index];
+    slot.current_progress = bytesReceived;
+    slot.total_progress = bytesTotal;
+
+    int done = m_done.size();
+    int doing = m_doing.size();
+    int all = parts_progress.size();
+
+    qint64 bytesAll = 0;
+    qint64 bytesTotalAll = 0;
+    for(auto & partIdx: m_doing)
+    {
+        auto part = parts_progress[partIdx];
+        // do not count parts with unknown/nonsensical total size
+        if(part.total_progress <= 0)
+        {
+            continue;
+        }
+        bytesAll += part.current_progress;
+        bytesTotalAll += part.total_progress;
+    }
+
+    qint64 inprogress = (bytesTotalAll == 0) ? 0 : (bytesAll * 1000) / bytesTotalAll;
+    auto current = done * 1000 + doing * inprogress;
+    auto current_total = all * 1000;
+    // HACK: make sure it never jumps backwards.
+    // FAIL: This breaks if the size is not known (or is it something else?) and jumps to 1000, so if it is 1000 reset it to inprogress
+    if(m_current_progress == 1000) {
+        m_current_progress = inprogress;
+    }
+    if(m_current_progress > current)
+    {
+        current = m_current_progress;
+    }
+    m_current_progress = current;
+    setProgress(current, current_total);
+}
+
+void NetJob::executeTask()
+{
+    // hack that delays early failures so they can be caught easier
+    QMetaObject::invokeMethod(this, "startMoreParts", Qt::QueuedConnection);
+}
+
+void NetJob::startMoreParts()
+{
+    if(!isRunning())
+    {
+        // this actually makes sense. You can put running downloads into a NetJob and then not start it until much later.
+        return;
+    }
+    // OK. We are actively processing tasks, proceed.
+    // Check for final conditions if there's nothing in the queue.
+    if(!m_todo.size())
+    {
+        if(!m_doing.size())
+        {
+            if(!m_failed.size())
+            {
+                emitSucceeded();
+            }
+            else if(m_aborted)
+            {
+                emitAborted();
+            }
+            else
+            {
+                emitFailed(tr("Job '%1' failed to process:\n%2").arg(objectName()).arg(getFailedFiles().join("\n")));
+            }
+        }
+        return;
+    }
+    // There's work to do, try to start more parts.
+    while (m_doing.size() < 6)
+    {
+        if(!m_todo.size())
+            return;
+        int doThis = m_todo.dequeue();
+        m_doing.insert(doThis);
+        auto part = downloads[doThis];
+        // connect signals :D
+        connect(part.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
+        connect(part.get(), SIGNAL(failed(int)), SLOT(partFailed(int)));
+        connect(part.get(), SIGNAL(aborted(int)), SLOT(partAborted(int)));
+        connect(part.get(), SIGNAL(netActionProgress(int, qint64, qint64)),
+                SLOT(partProgress(int, qint64, qint64)));
+        part->start(m_network);
+    }
+}
+
+
+QStringList NetJob::getFailedFiles()
+{
+    QStringList failed;
+    for (auto index: m_failed)
+    {
+        failed.push_back(downloads[index]->url().toString());
+    }
+    failed.sort();
+    return failed;
+}
+
+bool NetJob::canAbort() const
+{
+    bool canFullyAbort = true;
+    // can abort the waiting?
+    for(auto index: m_todo)
+    {
+        auto part = downloads[index];
+        canFullyAbort &= part->canAbort();
+    }
+    // can abort the active?
+    for(auto index: m_doing)
+    {
+        auto part = downloads[index];
+        canFullyAbort &= part->canAbort();
+    }
+    return canFullyAbort;
+}
+
+bool NetJob::abort()
+{
+    bool fullyAborted = true;
+    // fail all waiting
+    m_failed.unite(m_todo.toSet());
+    m_todo.clear();
+    // abort active
+    auto toKill = m_doing.toList();
+    for(auto index: toKill)
+    {
+        auto part = downloads[index];
+        fullyAborted &= part->abort();
+    }
+    return fullyAborted;
+}
+
+bool NetJob::addNetAction(NetAction::Ptr action)
+{
+    action->m_index_within_job = downloads.size();
+    downloads.append(action);
+    part_info pi;
+    parts_progress.append(pi);
+    partProgress(parts_progress.count() - 1, action->currentProgress(), action->totalProgress());
+
+    if(action->isRunning())
+    {
+        connect(action.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
+        connect(action.get(), SIGNAL(failed(int)), SLOT(partFailed(int)));
+        connect(action.get(), SIGNAL(netActionProgress(int, qint64, qint64)), SLOT(partProgress(int, qint64, qint64)));
+    }
+    else
+    {
+        m_todo.append(parts_progress.size() - 1);
+    }
+    return true;
+}
+
+NetJob::~NetJob() = default;

src/net/NetJob.h

@@ -0,0 +1,91 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <QtNetwork>
+#include "NetAction.h"
+#include "Download.h"
+#include "tasks/Task.h"
+#include "QObjectPtr.h"
+
+class NetJob;
+
+class NetJob : public Task
+{
+    Q_OBJECT
+public:
+    using Ptr = shared_qobject_ptr<NetJob>;
+
+    explicit NetJob(QString job_name, shared_qobject_ptr<QNetworkAccessManager> network) : Task(), m_network(network)
+    {
+        setObjectName(job_name);
+    }
+    virtual ~NetJob();
+
+    bool addNetAction(NetAction::Ptr action);
+
+    NetAction::Ptr operator[](int index)
+    {
+        return downloads[index];
+    }
+    const NetAction::Ptr at(const int index)
+    {
+        return downloads.at(index);
+    }
+    NetAction::Ptr first()
+    {
+        if (downloads.size())
+            return downloads[0];
+        return NetAction::Ptr();
+    }
+    int size() const
+    {
+        return downloads.size();
+    }
+    QStringList getFailedFiles();
+
+    bool canAbort() const override;
+
+private slots:
+    void startMoreParts();
+
+public slots:
+    virtual void executeTask() override;
+    virtual bool abort() override;
+
+private slots:
+    void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
+    void partSucceeded(int index);
+    void partFailed(int index);
+    void partAborted(int index);
+
+private:
+    shared_qobject_ptr<QNetworkAccessManager> m_network;
+
+    struct part_info
+    {
+        qint64 current_progress = 0;
+        qint64 total_progress = 1;
+        int failures = 0;
+    };
+    QList<NetAction::Ptr> downloads;
+    QList<part_info> parts_progress;
+    QQueue<int> m_todo;
+    QSet<int> m_doing;
+    QSet<int> m_done;
+    QSet<int> m_failed;
+    qint64 m_current_progress = 0;
+    bool m_aborted = false;
+};

src/net/Sink.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include "net/NetAction.h"
+
+#include "Validator.h"
+
+namespace Net {
+class Sink
+{
+public: /* con/des */
+    Sink() {};
+    virtual ~Sink() {};
+
+public: /* methods */
+    virtual JobStatus init(QNetworkRequest & request) = 0;
+    virtual JobStatus write(QByteArray & data) = 0;
+    virtual JobStatus abort() = 0;
+    virtual JobStatus finalize(QNetworkReply & reply) = 0;
+    virtual bool hasLocalData() = 0;
+
+    void addValidator(Validator * validator)
+    {
+        if(validator)
+        {
+            validators.push_back(std::shared_ptr<Validator>(validator));
+        }
+    }
+
+protected: /* methods */
+    bool finalizeAllValidators(QNetworkReply & reply)
+    {
+        for(auto & validator: validators)
+        {
+            if(!validator->validate(reply))
+                return false;
+        }
+        return true;
+    }
+    bool failAllValidators()
+    {
+        bool success = true;
+        for(auto & validator: validators)
+        {
+            success &= validator->abort();
+        }
+        return success;
+    }
+    bool initAllValidators(QNetworkRequest & request)
+    {
+        for(auto & validator: validators)
+        {
+            if(!validator->init(request))
+                return false;
+        }
+        return true;
+    }
+    bool writeAllValidators(QByteArray & data)
+    {
+        for(auto & validator: validators)
+        {
+            if(!validator->write(data))
+                return false;
+        }
+        return true;
+    }
+
+protected: /* data */
+    std::vector<std::shared_ptr<Validator>> validators;
+};
+}

src/net/Validator.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "net/NetAction.h"
+
+namespace Net {
+class Validator
+{
+public: /* con/des */
+    Validator() {};
+    virtual ~Validator() {};
+
+public: /* methods */
+    virtual bool init(QNetworkRequest & request) = 0;
+    virtual bool write(QByteArray & data) = 0;
+    virtual bool abort() = 0;
+    virtual bool validate(QNetworkReply & reply) = 0;
+};
+}

src/pathmatcher/FSTreeMatcher.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "IPathMatcher.h"
+#include <SeparatorPrefixTree.h>
+#include <QRegularExpression>
+
+class FSTreeMatcher : public IPathMatcher
+{
+public:
+    virtual ~FSTreeMatcher() {};
+    FSTreeMatcher(SeparatorPrefixTree<'/'> & tree) : m_fsTree(tree)
+    {
+    }
+
+    bool matches(const QString &string) const override
+    {
+        return m_fsTree.covers(string);
+    }
+
+    SeparatorPrefixTree<'/'> & m_fsTree;
+};

src/pathmatcher/IPathMatcher.h

@@ -0,0 +1,13 @@
+#pragma once
+#include <memory>
+#include <QString>
+
+class IPathMatcher
+{
+public:
+    typedef std::shared_ptr<IPathMatcher> Ptr;
+
+public:
+    virtual ~IPathMatcher(){};
+    virtual bool matches(const QString &string) const = 0;
+};

src/pathmatcher/MultiMatcher.h

@@ -0,0 +1,31 @@
+#include "IPathMatcher.h"
+#include <SeparatorPrefixTree.h>
+#include <QRegularExpression>
+
+class MultiMatcher : public IPathMatcher
+{
+public:
+    virtual ~MultiMatcher() {};
+    MultiMatcher()
+    {
+    }
+    MultiMatcher &add(Ptr add)
+    {
+        m_matchers.append(add);
+        return *this;
+    }
+
+    virtual bool matches(const QString &string) const override
+    {
+        for(auto iter: m_matchers)
+        {
+            if(iter->matches(string))
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    QList<Ptr> m_matchers;
+};

src/pathmatcher/RegexpMatcher.h

@@ -0,0 +1,42 @@
+#include "IPathMatcher.h"
+#include <QRegularExpression>
+
+class RegexpMatcher : public IPathMatcher
+{
+public:
+    virtual ~RegexpMatcher() {};
+    RegexpMatcher(const QString &regexp)
+    {
+        m_regexp.setPattern(regexp);
+        m_onlyFilenamePart = !regexp.contains('/');
+    }
+
+    RegexpMatcher &caseSensitive(bool cs = true)
+    {
+        if(cs)
+        {
+            m_regexp.setPatternOptions(QRegularExpression::CaseInsensitiveOption);
+        }
+        else
+        {
+            m_regexp.setPatternOptions(QRegularExpression::NoPatternOption);
+        }
+        return *this;
+    }
+
+    virtual bool matches(const QString &string) const override
+    {
+        if(m_onlyFilenamePart)
+        {
+            auto slash = string.lastIndexOf('/');
+            if(slash != -1)
+            {
+                auto part = string.mid(slash + 1);
+                return m_regexp.match(part).hasMatch();
+            }
+        }
+        return m_regexp.match(string).hasMatch();
+    }
+    QRegularExpression m_regexp;
+    bool m_onlyFilenamePart = false;
+};

src/tasks/SequentialTask.cpp

@@ -0,0 +1,55 @@
+#include "SequentialTask.h"
+
+SequentialTask::SequentialTask(QObject *parent) : Task(parent), m_currentIndex(-1)
+{
+}
+
+void SequentialTask::addTask(Task::Ptr task)
+{
+    m_queue.append(task);
+}
+
+void SequentialTask::executeTask()
+{
+    m_currentIndex = -1;
+    startNext();
+}
+
+void SequentialTask::startNext()
+{
+    if (m_currentIndex != -1)
+    {
+        Task::Ptr previous = m_queue[m_currentIndex];
+        disconnect(previous.get(), 0, this, 0);
+    }
+    m_currentIndex++;
+    if (m_queue.isEmpty() || m_currentIndex >= m_queue.size())
+    {
+        emitSucceeded();
+        return;
+    }
+    Task::Ptr next = m_queue[m_currentIndex];
+    connect(next.get(), SIGNAL(failed(QString)), this, SLOT(subTaskFailed(QString)));
+    connect(next.get(), SIGNAL(status(QString)), this, SLOT(subTaskStatus(QString)));
+    connect(next.get(), SIGNAL(progress(qint64, qint64)), this, SLOT(subTaskProgress(qint64, qint64)));
+    connect(next.get(), SIGNAL(succeeded()), this, SLOT(startNext()));
+    next->start();
+}
+
+void SequentialTask::subTaskFailed(const QString &msg)
+{
+    emitFailed(msg);
+}
+void SequentialTask::subTaskStatus(const QString &msg)
+{
+    setStatus(msg);
+}
+void SequentialTask::subTaskProgress(qint64 current, qint64 total)
+{
+    if(total == 0)
+    {
+        setProgress(0, 100);
+        return;
+    }
+    setProgress(current, total);
+}

src/tasks/SequentialTask.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "Task.h"
+#include "QObjectPtr.h"
+
+#include <QQueue>
+
+class SequentialTask : public Task
+{
+    Q_OBJECT
+public:
+    explicit SequentialTask(QObject *parent = 0);
+    virtual ~SequentialTask() {};
+
+    void addTask(Task::Ptr task);
+
+protected:
+    void executeTask();
+
+private
+slots:
+    void startNext();
+    void subTaskFailed(const QString &msg);
+    void subTaskStatus(const QString &msg);
+    void subTaskProgress(qint64 current, qint64 total);
+
+private:
+    QQueue<Task::Ptr > m_queue;
+    int m_currentIndex;
+};

src/tasks/Task.cpp

@@ -0,0 +1,168 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Task.h"
+
+#include <QDebug>
+
+Task::Task(QObject *parent) : QObject(parent)
+{
+}
+
+void Task::setStatus(const QString &new_status)
+{
+    if(m_status != new_status)
+    {
+        m_status = new_status;
+        emit status(m_status);
+    }
+}
+
+void Task::setProgress(qint64 current, qint64 total)
+{
+    m_progress = current;
+    m_progressTotal = total;
+    emit progress(m_progress, m_progressTotal);
+}
+
+void Task::start()
+{
+    switch(m_state)
+    {
+        case State::Inactive:
+        {
+            qDebug() << "Task" << describe() << "starting for the first time";
+            break;
+        }
+        case State::AbortedByUser:
+        {
+            qDebug() << "Task" << describe() << "restarting for after being aborted by user";
+            break;
+        }
+        case State::Failed:
+        {
+            qDebug() << "Task" << describe() << "restarting for after failing at first";
+            break;
+        }
+        case State::Succeeded:
+        {
+            qDebug() << "Task" << describe() << "restarting for after succeeding at first";
+            break;
+        }
+        case State::Running:
+        {
+            qWarning() << "The launcher tried to start task" << describe() << "while it was already running!";
+            return;
+        }
+    }
+    // NOTE: only fall thorugh to here in end states
+    m_state = State::Running;
+    emit started();
+    executeTask();
+}
+
+void Task::emitFailed(QString reason)
+{
+    // Don't fail twice.
+    if (!isRunning())
+    {
+        qCritical() << "Task" << describe() << "failed while not running!!!!: " << reason;
+        return;
+    }
+    m_state = State::Failed;
+    m_failReason = reason;
+    qCritical() << "Task" << describe() << "failed: " << reason;
+    emit failed(reason);
+    emit finished();
+}
+
+void Task::emitAborted()
+{
+    // Don't abort twice.
+    if (!isRunning())
+    {
+        qCritical() << "Task" << describe() << "aborted while not running!!!!";
+        return;
+    }
+    m_state = State::AbortedByUser;
+    m_failReason = "Aborted.";
+    qDebug() << "Task" << describe() << "aborted.";
+    emit failed(m_failReason);
+    emit finished();
+}
+
+void Task::emitSucceeded()
+{
+    // Don't succeed twice.
+    if (!isRunning())
+    {
+        qCritical() << "Task" << describe() << "succeeded while not running!!!!";
+        return;
+    }
+    m_state = State::Succeeded;
+    qDebug() << "Task" << describe() << "succeeded";
+    emit succeeded();
+    emit finished();
+}
+
+QString Task::describe()
+{
+    QString outStr;
+    QTextStream out(&outStr);
+    out << metaObject()->className() << QChar('(');
+    auto name = objectName();
+    if(name.isEmpty())
+    {
+        out << QString("0x%1").arg((quintptr)this, 0, 16);
+    }
+    else
+    {
+        out << name;
+    }
+    out << QChar(')');
+    out.flush();
+    return outStr;
+}
+
+bool Task::isRunning() const
+{
+    return m_state == State::Running;
+}
+
+bool Task::isFinished() const
+{
+    return m_state != State::Running && m_state != State::Inactive;
+}
+
+bool Task::wasSuccessful() const
+{
+    return m_state == State::Succeeded;
+}
+
+QString Task::failReason() const
+{
+    return m_failReason;
+}
+
+void Task::logWarning(const QString& line)
+{
+    qWarning() << line;
+    m_Warnings.append(line);
+}
+
+QStringList Task::warnings() const
+{
+    return m_Warnings;
+}

src/tasks/Task.h

@@ -0,0 +1,110 @@
+/* Copyright 2013-2021 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <QObject>
+#include <QString>
+#include <QStringList>
+
+#include "QObjectPtr.h"
+
+class Task : public QObject
+{
+    Q_OBJECT
+public:
+    using Ptr = shared_qobject_ptr<Task>;
+
+    enum class State
+    {
+        Inactive,
+        Running,
+        Succeeded,
+        Failed,
+        AbortedByUser
+    };
+
+public:
+    explicit Task(QObject *parent = 0);
+    virtual ~Task() {};
+
+    bool isRunning() const;
+    bool isFinished() const;
+    bool wasSuccessful() const;
+
+    /*!
+     * Returns the string that was passed to emitFailed as the error message when the task failed.
+     * If the task hasn't failed, returns an empty string.
+     */
+    QString failReason() const;
+
+    virtual QStringList warnings() const;
+
+    virtual bool canAbort() const { return false; }
+
+    QString getStatus()
+    {
+        return m_status;
+    }
+
+    qint64 getProgress()
+    {
+        return m_progress;
+    }
+
+    qint64 getTotalProgress()
+    {
+        return m_progressTotal;
+    }
+
+protected:
+    void logWarning(const QString & line);
+
+private:
+    QString describe();
+
+signals:
+    void started();
+    void progress(qint64 current, qint64 total);
+    void finished();
+    void succeeded();
+    void failed(QString reason);
+    void status(QString status);
+
+public slots:
+    virtual void start();
+    virtual bool abort() { return false; };
+
+protected:
+    virtual void executeTask() = 0;
+
+protected slots:
+    virtual void emitSucceeded();
+    virtual void emitAborted();
+    virtual void emitFailed(QString reason);
+
+public slots:
+    void setStatus(const QString &status);
+    void setProgress(qint64 current, qint64 total);
+
+private:
+    State m_state = State::Inactive;
+    QStringList m_Warnings;
+    QString m_failReason = "";
+    QString m_status;
+    int m_progress = 0;
+    int m_progressTotal = 100;
+};
+

src/teamcity/BuildIndex.cpp

@@ -0,0 +1,2 @@
+#include "BuildIndex.h"
+

src/teamcity/BuildIndex.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <QObject>
+
+class BuildIndex : public QObject {
+protected:
+    Q_OBJECT
+    Q_DISABLE_COPY(BuildIndex)
+public:
+    BuildIndex (QObject * parent = nullptr);
+    virtual ~BuildIndex() = default;
+};