ObsidianMC/nugget
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement importLibraries and other things for remapPatches

Browse source
This commit is contained in:
Kyle Wood 2020-11-15 23:23:21 -08:00
parent 54367a5067
commit 0625c87c9a
12 changed files with 135 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/main/kotlin/DownloadService.kt
View file

@ -24,6 +24,12 @@ package io.papermc.paperweight
import io.papermc.paperweight.util.convertToFile
import io.papermc.paperweight.util.convertToUrl
import java.io.File
import java.net.URL
import java.time.Instant
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
import java.util.concurrent.TimeUnit
import org.apache.http.HttpHost
import org.apache.http.HttpStatus
import org.apache.http.client.config.CookieSpecs
@ -35,12 +41,6 @@ import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClientBuilder
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import java.io.File
import java.net.URL
import java.time.Instant
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
import java.util.concurrent.TimeUnit
abstract class DownloadService : BuildService<BuildServiceParameters.None>, AutoCloseable {

9
src/main/kotlin/Paperweight.kt
View file

@ -185,6 +185,7 @@ class Paperweight : Plugin<Project> {
data class McpTasks(
val generateSrgs: TaskProvider<GenerateSrgs>,
val remapVanillaJarSrg: TaskProvider<RunSpecialSource>,
val downloadMcLibraries: TaskProvider<DownloadMcLibraries>,
val applyMcpPatches: TaskProvider<ApplyMcpPatches>
)
@ -317,6 +318,8 @@ class Paperweight : Plugin<Project> {
fieldsCsv.set(mcpRewrites.flatMap { it.paperFieldCsv })
extraNotchSrgMappings.set(extension.paper.extraNotchSrgMappings)
vanillaJar.set(generalTasks.filterVanillaJar.flatMap { it.outputJar })
notchToSrg.set(cache.resolve(Constants.NOTCH_TO_SRG))
notchToMcp.set(cache.resolve(Constants.NOTCH_TO_MCP))
srgToNotch.set(cache.resolve(Constants.SRG_TO_NOTCH))
@ -370,7 +373,7 @@ class Paperweight : Plugin<Project> {
configFile.set(cache.resolve(Constants.MCP_CONFIG_JSON))
}
return McpTasks(generateSrgs, remapVanillaJarSrg, applyMcpPatches)
return McpTasks(generateSrgs, remapVanillaJarSrg, downloadMcLibraries, applyMcpPatches)
}
private fun Project.createSpigotTasks(
@ -573,6 +576,10 @@ class Paperweight : Plugin<Project> {
spigotDecompJar.set(spigotTasks.decompileVanillaJarSpigot.flatMap { it.outputJar }.get())
constructors.set(initialTasks.extractMcp.flatMap { it.constructors }.get())
// library class imports
mcLibrariesDir.set(mcpTasks.downloadMcLibraries.flatMap { it.outputDir }.get())
libraryImports.set(extension.paper.libraryClassImports)
parameterNames.set(spigotTasks.remapSpigotSources.flatMap { it.parameterNames }.get())
outputPatchDir.set(extension.paper.remappedSpigotServerPatchDir)

1
src/main/kotlin/ext/PaperExtension.kt
View file

@ -44,6 +44,7 @@ open class PaperExtension(objects: ObjectFactory, layout: ProjectLayout) {
val missingMemberEntriesSrgFile: RegularFileProperty = objects.fileFrom(mcpDir, "missing-spigot-member-mappings.csrg")
val extraNotchSrgMappings: RegularFileProperty = objects.fileFrom(mcpDir, "extra-notch-srg.tsrg")
val extraSpigotSrgMappings: RegularFileProperty = objects.fileFrom(mcpDir, "extra-spigot-srg.tsrg")
val libraryClassImports: RegularFileProperty = objects.fileFrom(mcpDir, "library-imports.txt")
init {
spigotApiPatchDir.disallowUnsafeRead()

3
src/main/kotlin/tasks/GenerateSpigotSrgs.kt
View file

@ -26,6 +26,7 @@ import io.papermc.paperweight.util.file
import io.papermc.paperweight.util.fileOrNull
import io.papermc.paperweight.util.path
import io.papermc.paperweight.util.writeMappings
import java.nio.file.Files
import org.cadixdev.atlas.Atlas
import org.cadixdev.bombe.asm.jar.JarEntryRemappingTransformer
import org.cadixdev.lorenz.MappingSet
@ -47,7 +48,6 @@ import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
abstract class GenerateSpigotSrgs : DefaultTask() {
@ -109,6 +109,7 @@ abstract class GenerateSpigotSrgs : DefaultTask() {
.build()
).merge()
// TODO Not sure if this is still needed here since it's already ran in GenerateSrgs too now
// notch <-> spigot is incomplete here, it would result in inheritance issues to work with this incomplete set.
// so we use it once to remap some jar, which fills out the inheritance data
val atlasOut = Files.createTempFile("paperweight", "jar")

19
src/main/kotlin/tasks/GenerateSrgs.kt
View file

@ -26,9 +26,14 @@ import io.papermc.paperweight.util.ensureParentExists
import io.papermc.paperweight.util.file
import io.papermc.paperweight.util.fileOrNull
import io.papermc.paperweight.util.getCsvReader
import io.papermc.paperweight.util.path
import io.papermc.paperweight.util.writeMappings
import java.nio.file.Files
import org.cadixdev.atlas.Atlas
import org.cadixdev.bombe.asm.jar.JarEntryRemappingTransformer
import org.cadixdev.bombe.type.signature.MethodSignature
import org.cadixdev.lorenz.MappingSet
import org.cadixdev.lorenz.asm.LorenzRemapper
import org.cadixdev.lorenz.io.MappingFormats
import org.cadixdev.lorenz.merge.MappingSetMerger
import org.cadixdev.lorenz.model.ClassMapping
@ -53,6 +58,9 @@ abstract class GenerateSrgs : DefaultTask() {
@get:InputFile
abstract val inSrg: RegularFileProperty
@get:InputFile
abstract val vanillaJar: RegularFileProperty
@get:OutputFile
abstract val notchToSrg: RegularFileProperty
@get:OutputFile
@ -77,6 +85,17 @@ abstract class GenerateSrgs : DefaultTask() {
MappingFormats.TSRG.createReader(path).use { it.read(inSet) }
}
// Fill out any missing inheritance info in the mappings
val atlasOut = Files.createTempFile("paperweight", "jar")
try {
Atlas().use { atlas ->
atlas.install { ctx -> JarEntryRemappingTransformer(LorenzRemapper(inSet, ctx.inheritanceProvider())) }
atlas.run(vanillaJar.path, atlasOut)
}
} finally {
Files.deleteIfExists(atlasOut)
}
ensureParentExists(notchToSrg, notchToMcp, mcpToNotch, mcpToSrg, srgToNotch, srgToMcp)
createMappings(inSet, methods, fields)
}

2
src/main/kotlin/tasks/WriteLibrariesFile.kt
View file

@ -51,7 +51,7 @@ abstract class WriteLibrariesFile : BaseTask() {
outputFile.file.delete()
outputFile.file.bufferedWriter().use { writer ->
for (file in files) {
if (file.name.endsWith(".jar")) {
if (file.name.endsWith(".jar") && !file.name.endsWith("-sources.jar")) {
writer.appendln("-e=${file.absolutePath}")
}
}

11
src/main/kotlin/tasks/download-task.kt
View file

@ -319,12 +319,17 @@ abstract class DownloadWorker : WorkAction<DownloadParams> {
abstract val layout: ProjectLayout
override fun execute() {
val target = parameters.target.file
val artifact = MavenArtifact.parse(parameters.artifact.get())
val source = artifact.copy(classifier = "sources")
if (parameters.downloadToDir.get()) {
artifact.downloadToDir(parameters.downloader.get(), parameters.target.file, parameters.repos.get())
artifact.downloadToDir(parameters.downloader.get(), target, parameters.repos.get())
try {
source.downloadToDir(parameters.downloader.get(), target, parameters.repos.get())
} catch (ignored: Exception) {}
} else {
artifact.downloadToFile(parameters.downloader.get(), parameters.target.file, parameters.repos.get())
artifact.downloadToFile(parameters.downloader.get(), target, parameters.repos.get())
}
}
}

12
src/main/kotlin/tasks/patchremap/PatchApplier.kt
View file

@ -38,6 +38,8 @@ class PatchApplier(
private var commitAuthor: String? = null
private var commitTime: String? = null
private val remappedBaseTag: String = "remapped-base"
// fun initRepo() {
// println("Initializing patch remap repo")
// git("branch", unmappedBranch).executeSilently()
@ -66,6 +68,7 @@ class PatchApplier(
fun commitInitialRemappedSource() {
git("add", ".").executeSilently()
git("commit", "-m", "Initial Remapped Source", "--author=Initial <auto@mated.null>").executeSilently()
git("tag", remappedBaseTag)
}
fun recordCommit() {
@ -95,4 +98,13 @@ class PatchApplier(
throw RuntimeException("Patch failed to apply: $patch")
}
}
fun generatePatches(target: File) {
target.deleteRecursively()
target.mkdirs()
git(
"format-patch", "--zero-commit", "--full-index", "--no-signature", "--no-stat", "-N", "-o",
target.absolutePath, remappedBaseTag
).runOut()
}
}

1
src/main/kotlin/tasks/patchremap/PatchSourceRemapWorker.kt
View file

@ -24,7 +24,6 @@ package io.papermc.paperweight.tasks.patchremap
import io.papermc.paperweight.tasks.sourceremap.ConstructorsData
import io.papermc.paperweight.tasks.sourceremap.ParamNames
import io.papermc.paperweight.tasks.sourceremap.PatchParameterRemapper
import io.papermc.paperweight.tasks.sourceremap.SrgParameterRemapper
import java.nio.file.Files
import java.nio.file.Path

93
src/main/kotlin/tasks/patchremap/RemapPatches.kt
View file

@ -22,6 +22,7 @@
package io.papermc.paperweight.tasks.patchremap
import io.papermc.paperweight.PaperweightException
import io.papermc.paperweight.tasks.BaseTask
import io.papermc.paperweight.tasks.sourceremap.parseConstructors
import io.papermc.paperweight.tasks.sourceremap.parseParamNames
@ -35,11 +36,13 @@ import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFile
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Classpath
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.kotlin.dsl.get
abstract class RemapPatches : BaseTask() {
@ -64,6 +67,11 @@ abstract class RemapPatches : BaseTask() {
@get:InputFile
abstract val spigotDecompJar: RegularFileProperty
@get:InputDirectory
abstract val mcLibrariesDir: DirectoryProperty
@get:InputFile
abstract val libraryImports: RegularFileProperty
// For parameter name remapping
@get:InputFile
abstract val parameterNames: RegularFileProperty
@ -73,8 +81,17 @@ abstract class RemapPatches : BaseTask() {
@get:OutputDirectory
abstract val outputPatchDir: DirectoryProperty
@get:Option(option = "skip-patches", description = "For resuming, skip first # of patches (e.g. --skip-patches=300)")
abstract val skipPatches: Property<String>
override fun init() {
skipPatches.convention("0")
}
@TaskAction
fun run() {
val skip = skipPatches.get().toInt()
// Check patches
val patches = inputPatchDir.file.listFiles() ?: return run {
println("No input patches found")
@ -95,8 +112,8 @@ abstract class RemapPatches : BaseTask() {
// Remap output directory, after each output this directory will be re-named to the input directory below for
// the next remap operation
println("setting up repo")
val tempApiDir = createWorkDir("patch-remap-api", source = spigotApiDir.file)
val tempInputDir = createWorkDirByCloning("patch-remap-input", source = spigotServerDir.file)
val tempApiDir = createWorkDir("patch-remap-api", source = spigotApiDir.file, recreate = skip == 0)
val tempInputDir = createWorkDirByCloning("patch-remap-input", source = spigotServerDir.file, recreate = skip == 0)
val tempOutputDir = createWorkDir("patch-remap-output")
val sourceInputDir = tempInputDir.resolve("src/main/java")
@ -121,17 +138,19 @@ abstract class RemapPatches : BaseTask() {
// patchApplier.initRepo() // Create empty initial commit
// remapper.remap() // Remap to Spigot mappings
// We need to include any missing classes for the patches later on
importMcDev(patches, tempInputDir.resolve("src/main/java"))
patchApplier.commitInitialSource() // Initial commit of Spigot sources
patchApplier.checkoutRemapped() // Switch to remapped branch without checking out files
if (skip == 0) {
// We need to include any missing classes for the patches later on
importMcDev(patches, tempInputDir.resolve("src/main/java"))
patchApplier.commitInitialSource() // Initial commit of Spigot sources
patchApplier.checkoutRemapped() // Switch to remapped branch without checking out files
remapper.remap() // Remap to new mappings
patchApplier.commitInitialRemappedSource() // Initial commit of Spigot sources mapped to new mappings
patchApplier.checkoutOld() // Normal checkout back to Spigot mappings branch
remapper.remap() // Remap to new mappings
patchApplier.commitInitialRemappedSource() // Initial commit of Spigot sources mapped to new mappings
patchApplier.checkoutOld() // Normal checkout back to Spigot mappings branch
}
// Repo setup is done, we can begin the patch loop now
patches.forEach { patch ->
patches.asSequence().drop(skip).forEach { patch ->
println("===========================")
println("attempting to remap $patch")
println("===========================")
@ -145,6 +164,8 @@ abstract class RemapPatches : BaseTask() {
println("done remapping patch $patch")
println("===========================")
}
patchApplier.generatePatches(outputPatchDir.file)
}
}
@ -163,6 +184,38 @@ abstract class RemapPatches : BaseTask() {
}
}
}
// Import library classes
val libraryLines = libraryImports.file.readLines()
if (libraryLines.isEmpty()) {
return
}
val libDir = mcLibrariesDir.file
val libFiles = (libDir.listFiles() ?: emptyArray()).filter { it.name.endsWith("-sources.jar" )}
if (libFiles.isEmpty()) {
throw PaperweightException("No library files found")
}
for (line in libraryLines) {
val (libraryName, filePath) = line.split(" ")
val libFile = libFiles.firstOrNull { it.name.startsWith(libraryName) }
?: throw PaperweightException("Failed to find library: $libraryName for class $filePath")
val outputFile = inputDir.resolve(filePath)
if (outputFile.exists()) {
continue
}
outputFile.parentFile.mkdirs()
ZipFile(libFile).use { zipFile ->
val zipEntry = zipFile.getEntry(filePath)
zipFile.getInputStream(zipEntry).use { input ->
outputFile.outputStream().buffered().use { output ->
input.copyTo(output)
}
}
}
}
}
private fun readMcDevNames(patches: Array<File>): Set<String> {
@ -181,20 +234,24 @@ abstract class RemapPatches : BaseTask() {
return result
}
private fun createWorkDir(name: String, source: File? = null): File {
private fun createWorkDir(name: String, source: File? = null, recreate: Boolean = true): File {
return layout.cache.resolve("paperweight").resolve(name).apply {
deleteRecursively()
mkdirs()
source?.copyRecursively(this)
if (recreate) {
deleteRecursively()
mkdirs()
source?.copyRecursively(this)
}
}
}
private fun createWorkDirByCloning(name: String, source: File): File {
private fun createWorkDirByCloning(name: String, source: File, recreate: Boolean = true): File {
val workDir = layout.cache.resolve("paperweight")
return workDir.resolve(name).apply {
deleteRecursively()
mkdirs()
Git(workDir)("clone", source.absolutePath, this.absolutePath).executeSilently()
if (recreate) {
deleteRecursively()
mkdirs()
Git(workDir)("clone", source.absolutePath, this.absolutePath).executeSilently()
}
}
}
}

2
src/main/kotlin/tasks/sourceremap/RemapSources.kt
View file

@ -91,7 +91,7 @@ abstract class RemapSources : ZippedTask() {
classpath.add(vanillaJar.file)
classpath.add(vanillaRemappedSpigotJar.file)
classpath.add(spigotApiDir.dir("src/main/java").get().asFile)
classpath.addAll(spigotDeps.get().asFileTree.filter { it.name.endsWith(".jar") }.files)
classpath.addAll(spigotDeps.get().asFileTree.filter { it.name.endsWith(".jar") && !it.name.endsWith("-sources.jar") }.files)
mappings.set(this@RemapSources.mappings.file)
constructors.set(this@RemapSources.constructors.file)

4
src/main/kotlin/util/utils.kt
View file

@ -33,6 +33,8 @@ import io.papermc.paperweight.util.Constants.paperTaskOutput
import java.io.File
import java.io.InputStream
import java.io.OutputStream
import java.net.URI
import java.net.URL
import java.nio.file.Path
import java.util.Optional
import kotlin.reflect.KClass
@ -48,8 +50,6 @@ import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.TaskContainer
import org.gradle.api.tasks.TaskProvider
import java.net.URI
import java.net.URL
val gson: Gson = Gson()