From b56b118555f19ab88f7e225f67f12b8d00e755a3 Mon Sep 17 00:00:00 2001 From: Jason <11360596+jpenilla@users.noreply.github.com> Date: Tue, 4 Apr 2023 11:56:51 -0700 Subject: [PATCH] feat(userdev): Add experimental (for now) option for shared userdev caches (#187) --- .github/workflows/deploy.yml | 8 ++ .../io/papermc/paperweight/util/durations.kt | 86 +++++++++++++++++++ .../paperweight/userdev/PaperweightUser.kt | 80 ++++++++++++----- .../userdev/internal/setup/UserdevSetup.kt | 7 +- .../userdev/internal/setup/util/utils.kt | 57 +++++++++++- 5 files changed, 211 insertions(+), 27 deletions(-) create mode 100644 paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/durations.kt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7555773..c8f5faf 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,3 +20,11 @@ jobs: env: GRADLE_PUBLISH_KEY: "${{ secrets.GRADLE_PLUGIN_PORTAL_KEY }}" GRADLE_PUBLISH_SECRET: "${{ secrets.GRADLE_PLUGIN_PORTAL_SECRET }}" + - name: Parse tag + id: vars + run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} + - name: Create release and changelog + uses: MC-Machinations/auto-release-changelog@v1.1.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: paperweight ${{ steps.vars.outputs.tag }} diff --git a/paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/durations.kt b/paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/durations.kt new file mode 100644 index 0000000..0155d48 --- /dev/null +++ b/paperweight-lib/src/main/kotlin/io/papermc/paperweight/util/durations.kt @@ -0,0 +1,86 @@ +/* + * paperweight is a Gradle plugin for the PaperMC project. + * + * Copyright (c) 2023 Kyle Wood (DenWav) + * Contributors + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 only, no later versions. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +package io.papermc.paperweight.util + +import java.time.Duration +import java.time.temporal.ChronoUnit + +/** + * Map of accepted abbreviation [Char]s to [ChronoUnit]. + */ +private val units = mapOf( + 'd' to ChronoUnit.DAYS, + 'h' to ChronoUnit.HOURS, + 'm' to ChronoUnit.MINUTES, + 's' to ChronoUnit.SECONDS +) + +/** + * Parses a [Duration] from [input]. + * + * Accepted format is a number followed by a unit abbreviation. + * See [units] for possible units. + * Example input strings: `["1d", "12h", "1m", "30s"]` + * + * @param input formatted input string + * @throws InvalidDurationException when [input] is improperly formatted + */ +@Throws(InvalidDurationException::class) +fun parseDuration(input: String): Duration { + if (input.isBlank()) { + throw InvalidDurationException.noInput(input) + } + if (input.length < 2) { + throw InvalidDurationException.invalidInput(input) + } + val unitAbbreviation = input.last() + + val unit = units[unitAbbreviation] ?: throw InvalidDurationException.invalidInput(input) + + val length = try { + input.substring(0, input.length - 1).toLong() + } catch (ex: NumberFormatException) { + throw InvalidDurationException.invalidInput(input, ex) + } + + return Duration.of(length, unit) +} + +private class InvalidDurationException private constructor( + message: String, + cause: Throwable? = null +) : IllegalArgumentException(message, cause) { + companion object { + private val infoMessage = """ + Accepted format is a number followed by a unit abbreviation. + Possible units: $units + Example input strings: ["1d", "12h", "1m", "30s"] + """.trimIndent() + + fun noInput(input: String): InvalidDurationException = + InvalidDurationException("Cannot parse a Duration from a blank input string '$input'.\n$infoMessage") + + fun invalidInput(input: String, cause: Throwable? = null) = + InvalidDurationException("Cannot parse a Duration from input '$input'.\n$infoMessage", cause) + } +} diff --git a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt index 0c11f0e..60da059 100644 --- a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt +++ b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/PaperweightUser.kt @@ -28,12 +28,17 @@ import io.papermc.paperweight.tasks.* import io.papermc.paperweight.userdev.attribute.Obfuscation import io.papermc.paperweight.userdev.internal.setup.SetupHandler import io.papermc.paperweight.userdev.internal.setup.UserdevSetup -import io.papermc.paperweight.userdev.internal.setup.util.genSources +import io.papermc.paperweight.userdev.internal.setup.util.* import io.papermc.paperweight.util.* import io.papermc.paperweight.util.constants.* +import java.nio.file.Path import javax.inject.Inject import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.artifacts.ModuleDependency +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.gradle.api.artifacts.result.ResolvedDependencyResult import org.gradle.api.attributes.Bundling import org.gradle.api.attributes.Category import org.gradle.api.attributes.LibraryElements @@ -54,32 +59,27 @@ abstract class PaperweightUser : Plugin { abstract val javaToolchainService: JavaToolchainService override fun apply(target: Project) { + val sharedCacheRootRoot = target.gradle.gradleUserHomeDir.toPath().resolve("caches/paperweight-userdev") + val sharedCacheRoot = if (target.sharedCaches) sharedCacheRootRoot.resolve(paperweightHash) else null + target.gradle.sharedServices.registerIfAbsent("download", DownloadService::class) {} + val cleanAll = target.tasks.register("cleanAllPaperweightUserdevCaches") { + group = "paperweight" + description = "Delete the project & all shared paperweight-userdev setup cache." + delete(target.layout.cache) + delete(sharedCacheRootRoot) + } val cleanCache by target.tasks.registering { group = "paperweight" - description = "Delete the project setup cache and task outputs." + description = "Delete the project paperweight-userdev setup cache." delete(target.layout.cache) } target.configurations.register(DEV_BUNDLE_CONFIG) - // these must not be initialized until afterEvaluate, as they resolve the dev bundle - val userdevSetup by lazy { - val devBundleZip = target.configurations.named(DEV_BUNDLE_CONFIG).map { it.singleFile }.convertToPath() - val serviceName = "paperweight-userdev:setupService:${devBundleZip.sha256asHex()}" - - target.gradle.sharedServices - .registerIfAbsent(serviceName, UserdevSetup::class) { - parameters { - cache.set(target.layout.cache) - bundleZip.set(devBundleZip) - downloadService.set(target.download) - genSources.set(target.genSources) - } - } - .get() - } + // must not be initialized until afterEvaluate, as it resolves the dev bundle + val userdevSetup by lazy { createSetup(target, sharedCacheRoot) } val userdev = target.extensions.create( PAPERWEIGHT_EXTENSION, @@ -133,7 +133,7 @@ abstract class PaperweightUser : Plugin { val cleaningCache = gradle.startParameter.taskRequests .any { req -> req.args.any { arg -> - NameMatcher().find(arg, tasks.names) == cleanCache.name + NameMatcher().find(arg, tasks.names) in setOf(cleanCache.name, cleanAll.name) } } if (cleaningCache) { @@ -141,10 +141,10 @@ abstract class PaperweightUser : Plugin { } userdev.reobfArtifactConfiguration.get() - .configure(target, reobfJar) + .configure(this, reobfJar) if (userdev.injectPaperRepository.get()) { - target.repositories.maven(PAPER_MAVEN_REPO_URL) { + repositories.maven(PAPER_MAVEN_REPO_URL) { content { onlyForConfigurations(DEV_BUNDLE_CONFIG) } } } @@ -153,6 +153,8 @@ abstract class PaperweightUser : Plugin { checkForDevBundle() configureRepositories(userdevSetup) + + cleanSharedCaches(this, sharedCacheRootRoot) } } @@ -254,4 +256,40 @@ abstract class PaperweightUser : Plugin { private fun createContext(project: Project): SetupHandler.Context = SetupHandler.Context(project, workerExecutor, javaToolchainService) + + private fun createSetup(target: Project, sharedCacheRoot: Path?): UserdevSetup { + val bundleConfig = target.configurations.named(DEV_BUNDLE_CONFIG) + val devBundleZip = bundleConfig.map { it.singleFile }.convertToPath() + val bundleHash = devBundleZip.sha256asHex() + val cacheDir = if (sharedCacheRoot == null) { + target.layout.cache + } else { + when (bundleConfig.get().dependencies.single()) { + is ProjectDependency -> { + throw PaperweightException("Shared caches does not support the dev bundle being a ProjectDependency.") + } + + is ModuleDependency -> { + val resolved = + bundleConfig.get().incoming.resolutionResult.rootComponent.get().dependencies.single() as ResolvedDependencyResult + val resolvedId = resolved.selected.id as ModuleComponentIdentifier + sharedCacheRoot.resolve("module/${resolvedId.group}/${resolvedId.module}/${resolvedId.version}") + } + + else -> sharedCacheRoot.resolve("non-module/$bundleHash") + } + } + + val serviceName = "paperweight-userdev:setupService:$bundleHash" + return target.gradle.sharedServices + .registerIfAbsent(serviceName, UserdevSetup::class) { + parameters { + cache.set(cacheDir) + bundleZip.set(devBundleZip) + downloadService.set(target.download) + genSources.set(target.genSources) + } + } + .get() + } } diff --git a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/UserdevSetup.kt b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/UserdevSetup.kt index 0fcfb26..60c4707 100644 --- a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/UserdevSetup.kt +++ b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/UserdevSetup.kt @@ -23,10 +23,11 @@ package io.papermc.paperweight.userdev.internal.setup import io.papermc.paperweight.DownloadService -import io.papermc.paperweight.userdev.internal.setup.util.lockSetup +import io.papermc.paperweight.userdev.internal.setup.util.* import io.papermc.paperweight.util.* import io.papermc.paperweight.util.constants.* import java.nio.file.Path +import kotlin.io.path.* import org.gradle.api.Project import org.gradle.api.artifacts.DependencySet import org.gradle.api.artifacts.repositories.IvyArtifactRepository @@ -52,10 +53,12 @@ abstract class UserdevSetup : BuildService, SetupHandle } private val extractDevBundle: ExtractedBundle = lockSetup(parameters.cache.path) { - extractDevBundle( + val extract = extractDevBundle( parameters.cache.path.resolve(paperSetupOutput("extractDevBundle", "dir")), parameters.bundleZip.path ) + lastUsedFile(parameters.cache.path).writeText(System.currentTimeMillis().toString()) + extract } private val setup = createSetup() diff --git a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/util/utils.kt b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/util/utils.kt index ca699ad..0e1e54c 100644 --- a/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/util/utils.kt +++ b/paperweight-userdev/src/main/kotlin/io/papermc/paperweight/userdev/internal/setup/util/utils.kt @@ -26,17 +26,19 @@ import io.papermc.paperweight.DownloadService import io.papermc.paperweight.userdev.PaperweightUser import io.papermc.paperweight.userdev.internal.setup.UserdevSetup import io.papermc.paperweight.util.* -import io.papermc.paperweight.util.constants.USERDEV_SETUP_LOCK +import io.papermc.paperweight.util.constants.* import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.time.Duration import java.util.stream.Collectors import kotlin.io.path.* +import kotlin.streams.asSequence import kotlin.system.measureTimeMillis import org.gradle.api.Project import org.gradle.api.provider.Provider -private val paperweightHash: String by lazy { hashPaperweightJar() } +val paperweightHash: String by lazy { hashPaperweightJar() } fun Path.siblingLogFile(): Path = withDifferentExtension("log") @@ -154,7 +156,7 @@ fun interface HashFunction : () -> String { private fun hashPaperweightJar(): String { val userdevShadowJar = Paths.get(PaperweightUser::class.java.protectionDomain.codeSource.location.toURI()) - return hash(userdevShadowJar) + return userdevShadowJar.sha256asHex() } fun lockSetup(cache: Path, canBeNested: Boolean = false, action: () -> R): R { @@ -175,9 +177,56 @@ val Project.ci: Provider .map { it.toBoolean() } .orElse(false) +private fun experimentalProp(name: String) = "paperweight.experimental.$name" + val Project.genSources: Boolean get() { val ci = ci.get() - val prop = providers.gradleProperty("paperweight.experimental.genSources").orNull?.toBoolean() + val prop = providers.gradleProperty(experimentalProp("genSources")).orNull?.toBoolean() return prop ?: !ci } + +val Project.sharedCaches: Boolean + get() = providers.gradleProperty(experimentalProp("sharedCaches")).orNull.toBoolean() + +private fun deleteUnusedAfter(target: Project): Long = + target.providers.gradleProperty(experimentalProp("sharedCaches.deleteUnusedAfter")) + .map { value -> parseDuration(value) } + .orElse(Duration.ofDays(7)) + .map { duration -> duration.toMillis() } + .get() + +fun lastUsedFile(cacheDir: Path): Path = cacheDir.resolve(paperSetupOutput("last-used", "txt")) + +fun cleanSharedCaches(target: Project, root: Path) { + if (!root.exists()) { + return + } + val toDelete = Files.walk(root).use { stream -> + stream.asSequence() + .filter { it.name == "last-used.txt" } + .mapNotNull { + val dir = it.parent.parent // paperweight dir + val lastUsed = it.readText().toLong() + val since = System.currentTimeMillis() - lastUsed + val cutoff = deleteUnusedAfter(target) + if (since > cutoff) dir else null + } + .toList() + } + for (path in toDelete) { + path.deleteRecursively() + + // clean up empty parent directories + var parent: Path = path.parent + while (true) { + val entries = parent.listDirectoryEntries() + if (entries.isEmpty()) { + parent.deleteIfExists() + } else { + break + } + parent = parent.parent + } + } +}