feat(userdev): Add experimental (for now) option for shared userdev caches (#187)

.github/workflows/deploy.yml

@@ -20,3 +20,11 @@ jobs:
         env:
           GRADLE_PUBLISH_KEY: "${{ secrets.GRADLE_PLUGIN_PORTAL_KEY }}"
           GRADLE_PUBLISH_SECRET: "${{ secrets.GRADLE_PLUGIN_PORTAL_SECRET }}"
+      - name: Parse tag
+        id: vars
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      - name: Create release and changelog
+        uses: MC-Machinations/auto-release-changelog@v1.1.3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: paperweight ${{ steps.vars.outputs.tag }}

paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/durations.kt

@@ -0,0 +1,86 @@
+/*
+ * paperweight is a Gradle plugin for the PaperMC project.
+ *
+ * Copyright (c) 2023 Kyle Wood (DenWav)
+ *                    Contributors
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation;
+ * version 2.1 only, no later versions.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+package io.papermc.paperweight.util
+
+import java.time.Duration
+import java.time.temporal.ChronoUnit
+
+/**
+ * Map of accepted abbreviation [Char]s to [ChronoUnit].
+ */
+private val units = mapOf(
+    'd' to ChronoUnit.DAYS,
+    'h' to ChronoUnit.HOURS,
+    'm' to ChronoUnit.MINUTES,
+    's' to ChronoUnit.SECONDS
+)
+
+/**
+ * Parses a [Duration] from [input].
+ *
+ * Accepted format is a number followed by a unit abbreviation.
+ * See [units] for possible units.
+ * Example input strings: `["1d", "12h", "1m", "30s"]`
+ *
+ * @param input formatted input string
+ * @throws InvalidDurationException when [input] is improperly formatted
+ */
+@Throws(InvalidDurationException::class)
+fun parseDuration(input: String): Duration {
+    if (input.isBlank()) {
+        throw InvalidDurationException.noInput(input)
+    }
+    if (input.length < 2) {
+        throw InvalidDurationException.invalidInput(input)
+    }
+    val unitAbbreviation = input.last()
+
+    val unit = units[unitAbbreviation] ?: throw InvalidDurationException.invalidInput(input)
+
+    val length = try {
+        input.substring(0, input.length - 1).toLong()
+    } catch (ex: NumberFormatException) {
+        throw InvalidDurationException.invalidInput(input, ex)
+    }
+
+    return Duration.of(length, unit)
+}
+
+private class InvalidDurationException private constructor(
+    message: String,
+    cause: Throwable? = null
+) : IllegalArgumentException(message, cause) {
+    companion object {
+        private val infoMessage = """
+      Accepted format is a number followed by a unit abbreviation.
+      Possible units: $units
+      Example input strings: ["1d", "12h", "1m", "30s"]
+        """.trimIndent()
+
+        fun noInput(input: String): InvalidDurationException =
+            InvalidDurationException("Cannot parse a Duration from a blank input string '$input'.\n$infoMessage")
+
+        fun invalidInput(input: String, cause: Throwable? = null) =
+            InvalidDurationException("Cannot parse a Duration from input '$input'.\n$infoMessage", cause)
+    }
+}

paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt

@@ -28,12 +28,17 @@ import io.papermc.paperweight.tasks.*
 import io.papermc.paperweight.userdev.attribute.Obfuscation
 import io.papermc.paperweight.userdev.internal.setup.SetupHandler
 import io.papermc.paperweight.userdev.internal.setup.UserdevSetup
-import io.papermc.paperweight.userdev.internal.setup.util.genSources
+import io.papermc.paperweight.userdev.internal.setup.util.*
 import io.papermc.paperweight.util.*
 import io.papermc.paperweight.util.constants.*
+import java.nio.file.Path
 import javax.inject.Inject
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.artifacts.ModuleDependency
+import org.gradle.api.artifacts.ProjectDependency
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.gradle.api.artifacts.result.ResolvedDependencyResult
 import org.gradle.api.attributes.Bundling
 import org.gradle.api.attributes.Category
 import org.gradle.api.attributes.LibraryElements
@@ -54,32 +59,27 @@ abstract class PaperweightUser : Plugin<Project> {
     abstract val javaToolchainService: JavaToolchainService
 
     override fun apply(target: Project) {
+        val sharedCacheRootRoot = target.gradle.gradleUserHomeDir.toPath().resolve("caches/paperweight-userdev")
+        val sharedCacheRoot = if (target.sharedCaches) sharedCacheRootRoot.resolve(paperweightHash) else null
+
         target.gradle.sharedServices.registerIfAbsent("download", DownloadService::class) {}
 
+        val cleanAll = target.tasks.register<Delete>("cleanAllPaperweightUserdevCaches") {
+            group = "paperweight"
+            description = "Delete the project & all shared paperweight-userdev setup cache."
+            delete(target.layout.cache)
+            delete(sharedCacheRootRoot)
+        }
         val cleanCache by target.tasks.registering<Delete> {
             group = "paperweight"
-            description = "Delete the project setup cache and task outputs."
+            description = "Delete the project paperweight-userdev setup cache."
             delete(target.layout.cache)
         }
 
         target.configurations.register(DEV_BUNDLE_CONFIG)
 
-        // these must not be initialized until afterEvaluate, as they resolve the dev bundle
-        val userdevSetup by lazy {
-            val devBundleZip = target.configurations.named(DEV_BUNDLE_CONFIG).map { it.singleFile }.convertToPath()
-            val serviceName = "paperweight-userdev:setupService:${devBundleZip.sha256asHex()}"
-
-            target.gradle.sharedServices
-                .registerIfAbsent(serviceName, UserdevSetup::class) {
-                    parameters {
-                        cache.set(target.layout.cache)
-                        bundleZip.set(devBundleZip)
-                        downloadService.set(target.download)
-                        genSources.set(target.genSources)
-                    }
-                }
-                .get()
-        }
+        // must not be initialized until afterEvaluate, as it resolves the dev bundle
+        val userdevSetup by lazy { createSetup(target, sharedCacheRoot) }
 
         val userdev = target.extensions.create(
             PAPERWEIGHT_EXTENSION,
@@ -133,7 +133,7 @@ abstract class PaperweightUser : Plugin<Project> {
             val cleaningCache = gradle.startParameter.taskRequests
                 .any { req ->
                     req.args.any { arg ->
-                        NameMatcher().find(arg, tasks.names) == cleanCache.name
+                        NameMatcher().find(arg, tasks.names) in setOf(cleanCache.name, cleanAll.name)
                     }
                 }
             if (cleaningCache) {
@@ -141,10 +141,10 @@ abstract class PaperweightUser : Plugin<Project> {
             }
 
             userdev.reobfArtifactConfiguration.get()
-                .configure(target, reobfJar)
+                .configure(this, reobfJar)
 
             if (userdev.injectPaperRepository.get()) {
-                target.repositories.maven(PAPER_MAVEN_REPO_URL) {
+                repositories.maven(PAPER_MAVEN_REPO_URL) {
                     content { onlyForConfigurations(DEV_BUNDLE_CONFIG) }
                 }
             }
@@ -153,6 +153,8 @@ abstract class PaperweightUser : Plugin<Project> {
             checkForDevBundle()
 
             configureRepositories(userdevSetup)
+
+            cleanSharedCaches(this, sharedCacheRootRoot)
         }
     }
 
@@ -254,4 +256,40 @@ abstract class PaperweightUser : Plugin<Project> {
 
     private fun createContext(project: Project): SetupHandler.Context =
         SetupHandler.Context(project, workerExecutor, javaToolchainService)
+
+    private fun createSetup(target: Project, sharedCacheRoot: Path?): UserdevSetup {
+        val bundleConfig = target.configurations.named(DEV_BUNDLE_CONFIG)
+        val devBundleZip = bundleConfig.map { it.singleFile }.convertToPath()
+        val bundleHash = devBundleZip.sha256asHex()
+        val cacheDir = if (sharedCacheRoot == null) {
+            target.layout.cache
+        } else {
+            when (bundleConfig.get().dependencies.single()) {
+                is ProjectDependency -> {
+                    throw PaperweightException("Shared caches does not support the dev bundle being a ProjectDependency.")
+                }
+
+                is ModuleDependency -> {
+                    val resolved =
+                        bundleConfig.get().incoming.resolutionResult.rootComponent.get().dependencies.single() as ResolvedDependencyResult
+                    val resolvedId = resolved.selected.id as ModuleComponentIdentifier
+                    sharedCacheRoot.resolve("module/${resolvedId.group}/${resolvedId.module}/${resolvedId.version}")
+                }
+
+                else -> sharedCacheRoot.resolve("non-module/$bundleHash")
+            }
+        }
+
+        val serviceName = "paperweight-userdev:setupService:$bundleHash"
+        return target.gradle.sharedServices
+            .registerIfAbsent(serviceName, UserdevSetup::class) {
+                parameters {
+                    cache.set(cacheDir)
+                    bundleZip.set(devBundleZip)
+                    downloadService.set(target.download)
+                    genSources.set(target.genSources)
+                }
+            }
+            .get()
+    }
 }

paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/UserdevSetup.kt

@@ -23,10 +23,11 @@
 package io.papermc.paperweight.userdev.internal.setup
 
 import io.papermc.paperweight.DownloadService
-import io.papermc.paperweight.userdev.internal.setup.util.lockSetup
+import io.papermc.paperweight.userdev.internal.setup.util.*
 import io.papermc.paperweight.util.*
 import io.papermc.paperweight.util.constants.*
 import java.nio.file.Path
+import kotlin.io.path.*
 import org.gradle.api.Project
 import org.gradle.api.artifacts.DependencySet
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
@@ -52,10 +53,12 @@ abstract class UserdevSetup : BuildService<UserdevSetup.Parameters>, SetupHandle
     }
 
     private val extractDevBundle: ExtractedBundle<Any> = lockSetup(parameters.cache.path) {
-        extractDevBundle(
+        val extract = extractDevBundle(
             parameters.cache.path.resolve(paperSetupOutput("extractDevBundle", "dir")),
             parameters.bundleZip.path
         )
+        lastUsedFile(parameters.cache.path).writeText(System.currentTimeMillis().toString())
+        extract
     }
 
     private val setup = createSetup()

paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/util/utils.kt

@@ -26,17 +26,19 @@ import io.papermc.paperweight.DownloadService
 import io.papermc.paperweight.userdev.PaperweightUser
 import io.papermc.paperweight.userdev.internal.setup.UserdevSetup
 import io.papermc.paperweight.util.*
-import io.papermc.paperweight.util.constants.USERDEV_SETUP_LOCK
+import io.papermc.paperweight.util.constants.*
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.time.Duration
 import java.util.stream.Collectors
 import kotlin.io.path.*
+import kotlin.streams.asSequence
 import kotlin.system.measureTimeMillis
 import org.gradle.api.Project
 import org.gradle.api.provider.Provider
 
-private val paperweightHash: String by lazy { hashPaperweightJar() }
+val paperweightHash: String by lazy { hashPaperweightJar() }
 
 fun Path.siblingLogFile(): Path = withDifferentExtension("log")
 
@@ -154,7 +156,7 @@ fun interface HashFunction : () -> String {
 
 private fun hashPaperweightJar(): String {
     val userdevShadowJar = Paths.get(PaperweightUser::class.java.protectionDomain.codeSource.location.toURI())
-    return hash(userdevShadowJar)
+    return userdevShadowJar.sha256asHex()
 }
 
 fun <R> lockSetup(cache: Path, canBeNested: Boolean = false, action: () -> R): R {
@@ -175,9 +177,56 @@ val Project.ci: Provider<Boolean>
         .map { it.toBoolean() }
         .orElse(false)
 
+private fun experimentalProp(name: String) = "paperweight.experimental.$name"
+
 val Project.genSources: Boolean
     get() {
         val ci = ci.get()
-        val prop = providers.gradleProperty("paperweight.experimental.genSources").orNull?.toBoolean()
+        val prop = providers.gradleProperty(experimentalProp("genSources")).orNull?.toBoolean()
         return prop ?: !ci
     }
+
+val Project.sharedCaches: Boolean
+    get() = providers.gradleProperty(experimentalProp("sharedCaches")).orNull.toBoolean()
+
+private fun deleteUnusedAfter(target: Project): Long =
+    target.providers.gradleProperty(experimentalProp("sharedCaches.deleteUnusedAfter"))
+        .map { value -> parseDuration(value) }
+        .orElse(Duration.ofDays(7))
+        .map { duration -> duration.toMillis() }
+        .get()
+
+fun lastUsedFile(cacheDir: Path): Path = cacheDir.resolve(paperSetupOutput("last-used", "txt"))
+
+fun cleanSharedCaches(target: Project, root: Path) {
+    if (!root.exists()) {
+        return
+    }
+    val toDelete = Files.walk(root).use { stream ->
+        stream.asSequence()
+            .filter { it.name == "last-used.txt" }
+            .mapNotNull {
+                val dir = it.parent.parent // paperweight dir
+                val lastUsed = it.readText().toLong()
+                val since = System.currentTimeMillis() - lastUsed
+                val cutoff = deleteUnusedAfter(target)
+                if (since > cutoff) dir else null
+            }
+            .toList()
+    }
+    for (path in toDelete) {
+        path.deleteRecursively()
+
+        // clean up empty parent directories
+        var parent: Path = path.parent
+        while (true) {
+            val entries = parent.listDirectoryEntries()
+            if (entries.isEmpty()) {
+                parent.deleteIfExists()
+            } else {
+                break
+            }
+            parent = parent.parent
+        }
+    }
+}