using parchment: parameter names & javadoc; use vineflower for source generation #1

Merged
owlsys merged 4 commits from parchment into mistress 2024-05-18 07:05:46 -04:00
9 changed files with 56 additions and 372 deletions
Showing only changes of commit 9006dbf994 - Show all commits

View file

@ -21,7 +21,8 @@ repositories {
}
dependencies {
implementation("org.ecorous.esnesnon:nonsense-remapper:1.0.0-SNAPSHOT")
//implementation("org.ecorous.esnesnon:nonsense-remapper:1.0.0-SNAPSHOT")
implementation("org.ecorous.esnesnon:nonsense-remapper:1.0.0-20240517.184632-6")
implementation("com.google.code.gson:gson:2.10.1")
implementation("org.vineflower:vineflower:1.10.1")
testImplementation(kotlin("test"))

View file

@ -1,9 +1,8 @@
package org.ecorous.esnesnon.gradle
import net.fabricmc.fernflower.api.IFabricJavadocProvider
import org.ecorous.esnesnon.gradle.parchment.ParchmentProvider
import org.ecorous.esnesnon.gradle.vineflower.ParchmentJavadocProvider
import org.ecorous.esnesnon.gradle.vineflower.RenamingPlugin
import org.ecorous.esnesnon.nonsense_remapper.provider.ParchmentProvider
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.jetbrains.java.decompiler.main.Fernflower
@ -12,7 +11,6 @@ import org.jetbrains.java.decompiler.main.decompiler.SingleFileSaver
import org.jetbrains.java.decompiler.main.extern.IFernflowerPreferences
import java.io.PrintStream
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.deleteExisting
import kotlin.io.path.exists
@ -45,41 +43,34 @@ class NonsenseGradlePlugin : Plugin<Project> {
}
}
project.task("genSources").apply {
group = "nonsense"
doFirst {
val fileName = remappedGameJarPath.fileName.toString()
val output =
remappedGameJarPath.resolveSibling(fileName.substring(0, fileName.length - 4) + "-sources.jar")
remappedGameJarPath.resolveSibling(fileName.substring(0, fileName.length - 13) + "-sources.jar")
if (output.exists()) {
println("Output $output already exists, deleting!")
output.deleteExisting()
}
val options = mutableMapOf<String, Any>()
if (useParchment) {
println("Preparing Parchment...")
val parchment = ParchmentProvider.getParchment(
minecraftVersion,
parchmentVersion,
project.gradle.gradleUserHomeDir.resolve("caches/nonsense-gradle/").toPath()
project.gradle.gradleUserHomeDir.resolve("nonsense-gradle/org/parchmentmc/parchment/$minecraftVersion/$parchmentVersion/")
.toPath()
)
RenamingPlugin.parchment = parchment
/*options["variable-renaming"] = "parchment"
options["rename-parameters"] = "1"*/
options[IFabricJavadocProvider.PROPERTY_NAME] = ParchmentJavadocProvider(parchment)
}
println("Decompiling...")
val log = project.rootDir.resolve("vineflower.log").toPath() // TODO temporarily log VF output more verbosely to a separate file to have more debug information
Files.deleteIfExists(log)
val logger = PrintStreamLogger(PrintStream(Files.newOutputStream(log)))
val logger = PrintStreamLogger(PrintStream(System.out))
options.putAll(
mapOf(
IFernflowerPreferences.DECOMPILE_GENERIC_SIGNATURES to "1",
IFernflowerPreferences.BYTECODE_SOURCE_MAPPING to "1",
IFernflowerPreferences.REMOVE_SYNTHETIC to "1",
IFernflowerPreferences.LOG_LEVEL to "info", // TODO set to 'warn' before commit/push! (related to the comment above)
IFernflowerPreferences.LOG_LEVEL to "warn",
IFernflowerPreferences.THREADS to Runtime.getRuntime().availableProcessors().toString(),
IFernflowerPreferences.INDENT_STRING to "\t"
)
@ -92,8 +83,6 @@ class NonsenseGradlePlugin : Plugin<Project> {
decomp.addSource(remappedGameJarPath.toFile())
decomp.decompileContext()
println("Setting up nonsense...")
println("Imagine actually doing stuff")
}
}
@ -112,7 +101,6 @@ class NonsenseGradlePlugin : Plugin<Project> {
companion object {
lateinit var minecraftVersion: String
lateinit var remappedGameJarPath: Path
var useParchment = false
lateinit var parchmentVersion: String
}
}

View file

@ -28,14 +28,24 @@ object VersionChecker {
}
downloadDirectory.createDirectories()
val raw = rawDownload(clientData.url)
if (!downloadFile.exists()) {
downloadFile.createFile()
}
downloadFile.writeBytes(raw)
savePomFile(version, downloadDirectory)
println("Downloaded client to $downloadFile")
return downloadFile
}
private fun savePomFile(version: String, dir: Path){
val content = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<project xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\" xmlns=\"http://maven.apache.org/POM/4.0.0\"\n" +
"\t\t xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
"\t<modelVersion>4.0.0</modelVersion>\n" +
"\t<groupId>net.minecraft</groupId>\n" +
"\t<artifactId>client</artifactId>\n" +
"\t<version>$version</version>\n" +
"</project>"
dir.resolve("client-$version.pom").writeText(content)
}
fun getDependencies(version: String, action: (String) -> Unit) {
fetchVersionData(version).libraries.map { it.name }.forEach {
action.invoke(it)

View file

@ -2,14 +2,21 @@ package org.ecorous.esnesnon.gradle.ext
import org.ecorous.esnesnon.gradle.NonsenseGradlePlugin
import org.ecorous.esnesnon.gradle.VersionChecker
import org.ecorous.esnesnon.gradle.parchment.ParchmentProvider
import org.ecorous.esnesnon.nonsense_remapper.NonsenseRemapper
import org.ecorous.esnesnon.nonsense_remapper.api.Mapper
import org.ecorous.esnesnon.nonsense_remapper.parser.ProguardParser
import org.ecorous.esnesnon.nonsense_remapper.provider.MojmapProvider
import org.ecorous.esnesnon.nonsense_remapper.provider.ParchmentProvider
import org.gradle.api.Project
import kotlin.io.path.notExists
fun Project.minecraft(version: String): Project { // return self to allow for chaining
fun Project.minecraft(
version: String,
parchmentVersion: String = ParchmentProvider.findForMinecraftVersion(version)
): Project { // return self to allow for chaining
if (VersionChecker.validateVersion(version)) {
NonsenseGradlePlugin.minecraftVersion = version
NonsenseGradlePlugin.parchmentVersion = parchmentVersion
println("Valid version! $version")
val clientData = VersionChecker.fetchClientDownload(version)
println("Client data: ${clientData.url}")
@ -21,12 +28,19 @@ fun Project.minecraft(version: String): Project { // return self to allow for ch
NonsenseGradlePlugin.remappedGameJarPath = remappedJar
println("Time to setup Minecraft!")
if (remappedJar.notExists()) {
NonsenseRemapper.run(version, clientJar, remappedJar, true)
val mapper = Mapper(ProguardParser.read(MojmapProvider.get(version).orElseThrow()).reverse())
val paramMappings = ParchmentProvider.getParchment(
version, parchmentVersion,
gradle.gradleUserHomeDir.resolve("nonsense-gradle/org/parchmentmc/parchment/$version/$parchmentVersion")
.toPath()
)
NonsenseRemapper.remap(mapper, clientJar, remappedJar, true, paramMappings)
}
VersionChecker.getDependencies(version) {
dependencies.add("implementation", it)
}
dependencies.add("implementation", files(remappedJar))
dependencies.add("implementation", "net.minecraft:client:$version:remapped")
//dependencies.add("implementation", files(remappedJar))
} else {
println("Invalid version! $version")
error("Invalid minecraft version provided: $version")
@ -34,8 +48,3 @@ fun Project.minecraft(version: String): Project { // return self to allow for ch
println("Minecraft!")
return this
}
fun Project.parchment(version: String = ParchmentProvider.findLatestValidVersion()) {
NonsenseGradlePlugin.useParchment = true
NonsenseGradlePlugin.parchmentVersion = version
}

View file

@ -1,101 +0,0 @@
package org.ecorous.esnesnon.gradle.parchment
import com.google.gson.Gson
import org.ecorous.esnesnon.gradle.NonsenseGradlePlugin
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import javax.xml.parsers.DocumentBuilderFactory
import kotlin.io.path.createParentDirectories
import kotlin.io.path.notExists
object ParchmentProvider {
const val PARCHMENT_URL: String = "https://maven.parchmentmc.org/org/parchmentmc/data"
private fun getParchmentUrl(gameVersion: String): String {
return "$PARCHMENT_URL/parchment-$gameVersion"
}
private fun findParchmentVersion(gameVersion: String): String {
val url = getParchmentUrl(gameVersion) + "/maven-metadata.xml"
val stream = URI.create(url).toURL().openStream()
val document = DocumentBuilderFactory.newInstance()
.newDocumentBuilder().parse(stream)
stream?.close()
return document.getElementsByTagName("release").item(0).textContent
}
fun getParchment(gameVersion: String, cacheDir: Path): Parchment {
return getParchment(gameVersion, findParchmentVersion(gameVersion), cacheDir)
}
fun getParchment(gameVersion: String, parchmentVer: String, cacheDir: Path): Parchment {
val cachePath = cacheDir.resolve("parchment/$gameVersion/$parchmentVer/parchment-$gameVersion-$parchmentVer.zip")
cachePath.createParentDirectories()
if (cachePath.notExists()){
val url = "${getParchmentUrl(gameVersion)}/$parchmentVer/parchment-$gameVersion-$parchmentVer.zip"
Files.copy(URI.create(url).toURL().openStream(), cachePath)
}
// TODO hash verification
FileSystems.newFileSystem(cachePath).use {fs ->
val mappings = Files.readString(fs.getPath("parchment.json"))
return Gson().fromJson(mappings, Parchment::class.java)
}
}
fun findLatestValidVersion(): String {
return findForMinecraftVersion(NonsenseGradlePlugin.minecraftVersion)
}
fun findForMinecraftVersion(minecraftVersion: String): String {
return findParchmentVersion(minecraftVersion)
}
}
class Parchment(val version: String, val packages: List<Package>, val classes: List<Class>?){
fun getClass(name: String): Class? {
classes?.forEach {
if(it.name == name){
return it
}
}
return null
}
}
class Package(val name: String, val javadoc: List<String>?)
class Class(val name: String, val javadoc: List<String>?, val fields: List<Field>?, val methods: List<Method>?){
fun getField(name: String, descriptor: String): Field? {
fields?.forEach {
if(it.name == name && it.descriptor == descriptor){
return it
}
}
return null
}
fun getMethod(name: String, descriptor: String): Method? {
methods?.forEach {
if(it.name == name && it.descriptor == descriptor){
return it
}
}
return null
}
}
class Field(val name: String, val descriptor: String, val javadoc: List<String>?)
class Method(val name: String, val descriptor: String, val javadoc: List<String>?, val parameters: List<Parameter>?){
fun getParameter(index: Int): Parameter? {
parameters?.forEach {
if (it.index == index){
return it
}
}
return null
}
}
class Parameter(val index: Int, val name: String)

View file

@ -1,23 +1,27 @@
package org.ecorous.esnesnon.gradle.vineflower
import net.fabricmc.fernflower.api.IFabricJavadocProvider
import org.ecorous.esnesnon.gradle.parchment.Parchment
import org.ecorous.esnesnon.nonsense_remapper.api.data.Parchment
import org.jetbrains.java.decompiler.struct.StructClass
import org.jetbrains.java.decompiler.struct.StructField
import org.jetbrains.java.decompiler.struct.StructMethod
class ParchmentJavadocProvider(val parchment: Parchment) : IFabricJavadocProvider {
class ParchmentJavadocProvider(private val parchment: Parchment) : IFabricJavadocProvider {
override fun getClassDoc(structClass: StructClass): String? {
return parchment.getClass(structClass.qualifiedName)?.javadoc?.joinToString("\n")
return parchment.getClass(structClass.qualifiedName).map { it.javadoc }.map { it.joinToString { "\n" } }
Outdated
Review

might be nicer in kotlin to do .orElse(null) and use ?.
or at least combine the map calls into one as it seems unnecessary to split them

might be nicer in kotlin to do .orElse(null) and use ?. or at least combine the map calls into one as it seems unnecessary to split them
.orElse(null)
}
override fun getFieldDoc(structClass: StructClass, structField: StructField): String? {
return parchment.getClass(structClass.qualifiedName)
?.getField(structField.name, structField.descriptor)?.javadoc?.joinToString("\n")
.flatMap { it.getField(structField.name, structField.descriptor) }.map { it.javadoc }
.map { it.joinToString { "\n" } }
.orElse(null)
}
override fun getMethodDoc(structClass: StructClass, structMethod: StructMethod): String? {
return parchment.getClass(structClass.qualifiedName)
?.getMethod(structMethod.name, structMethod.descriptor)?.javadoc?.joinToString("\n")
.flatMap { it.getMethod(structMethod.name, structMethod.descriptor) }
.map { it.javadoc }.map { it.joinToString { "\n" } }.orElse(null)
}
}

View file

@ -1,203 +0,0 @@
package org.ecorous.esnesnon.gradle.vineflower
import org.ecorous.esnesnon.gradle.parchment.Parchment
import org.jetbrains.java.decompiler.code.CodeConstants
import org.jetbrains.java.decompiler.main.DecompilerContext
import org.jetbrains.java.decompiler.main.extern.IVariableNameProvider
import org.jetbrains.java.decompiler.main.extern.IVariableNamingFactory
import org.jetbrains.java.decompiler.modules.decompiler.ExprProcessor
import org.jetbrains.java.decompiler.modules.decompiler.vars.VarVersionPair
import org.jetbrains.java.decompiler.struct.StructMethod
import org.jetbrains.java.decompiler.struct.gen.CodeType
import org.jetbrains.java.decompiler.struct.gen.MethodDescriptor
import org.jetbrains.java.decompiler.struct.gen.VarType
import org.jetbrains.java.decompiler.util.Pair
import org.jetbrains.java.decompiler.util.TextUtil
// Adapted from https://github.com/Vineflower/vineflower/blob/ddf84710f8521ce62d1e3e55a39d300432d6b729/plugins/variable-renaming/src/main/java/org/vineflower/variablerenaming/TinyNameProvider.java
class ParchmentNameProvider(private val parchment: Parchment, val method: StructMethod) : IVariableNameProvider {
private val parameters: MutableMap<Int, String?> = mutableMapOf()
private val renameParameters = true
private val usedNames: MutableSet<String> = mutableSetOf()
override fun rename(entries: Map<VarVersionPair, Pair<VarType, String>>): Map<VarVersionPair, String?> {
var params = 0
if ((this.method.accessFlags and CodeConstants.ACC_STATIC) != CodeConstants.ACC_STATIC) {
params++
}
val md = MethodDescriptor.parseDescriptor(this.method.descriptor)
for (param in md.params) {
params += param.stackSize
}
val keys: MutableList<VarVersionPair> = ArrayList(entries.keys)
keys.sortWith { o1: VarVersionPair, o2: VarVersionPair -> if ((o1.`var` != o2.`var`)) o1.`var` - o2.`var` else o1.version - o2.version }
val result: MutableMap<VarVersionPair, String?> = LinkedHashMap()
for (ver in keys) {
// note: entries[ver]!!.a is currently null for references that are not parameter definitions. Figure this out. It currently causes errors further down the line
// and prevents VF from working correctly.
val type = cleanType(entries[ver]!!.b)
if (ver.`var` >= params) {
result[ver] = getNewName(
Pair.of(
entries[ver]!!.a, type
)
)
} else if (renameParameters) {
result[ver] = parameters.computeIfAbsent(
ver.`var`
) {
parchment.getClass(method.classQualifiedName)?.getMethod(method.name, method.descriptor)
?.getParameter(ver.`var`)?.name ?: getNewName(
Pair.of(
entries[ver]!!.a, type
)
)
}
}
}
return result
}
private fun getNewName(pair: Pair<VarType, String>): String? {
val type = pair.a
usedNames.add(pair.b)
var increment = true
var name: String
when (type.type) {
CodeType.BYTECHAR, CodeType.SHORTCHAR, CodeType.INT -> name = "i"
CodeType.LONG -> name = "l"
CodeType.BYTE -> name = "b"
CodeType.SHORT -> name = "s"
CodeType.CHAR -> name = "c"
CodeType.FLOAT -> name = "f"
CodeType.DOUBLE -> name = "d"
CodeType.BOOLEAN -> {
name = "bl"
increment = false
}
CodeType.OBJECT, CodeType.GENVAR -> {
name = pair.b
// Lowercase first letter
if (Character.isUpperCase(name[0])) {
name = name[0].lowercaseChar().toString() + name.substring(1)
}
if (type.arrayDim > 0 && !name.endsWith("s")) {
name += "s"
}
increment = false
}
else -> return null
}
if (increment) {
// Must be of length 1
var idxStart = name[0].code - 'a'.code
while (usedNames.contains(name)) {
name = convertToName(idxStart++)
}
} else {
// Increment via numbers
val oname = name
var idx = 0
while (usedNames.contains(name)) {
name = oname + (++idx)
}
}
if (TextUtil.isKeyword(name, method.bytecodeVersion, method)) {
name += "_"
}
usedNames.add(name)
return name
}
override fun renameParameter(flags: Int, type: VarType, name: String?, index: Int): String? {
var typeName: String
DecompilerContext.getImportCollector().lock().use {
typeName = ExprProcessor.getCastTypeName(type)
}
if (!this.renameParameters) {
return super.renameParameter(flags, type, name, index)
}
return parameters.computeIfAbsent(
index
) {
parchment.getClass(method.classQualifiedName)?.getMethod(method.name, method.descriptor)
?.getParameter(index)?.name ?: getNewName(
Pair.of(
type,
cleanType(typeName)
)
)
}
}
private fun convertToName(idx: Int): String {
// Convert to base 26
val str = idx.toString(26)
// Remap start of name from '0' to 'a'
val res = StringBuilder()
for (i in str.length - 1 downTo 0) {
var c = str[i]
// If we're numerical, remap to lowercase ascii range
if (c <= '9') {
c = ('a'.code + (c.code - '0'.code)).toChar()
} else {
// If we're not, simply shift up 10 ascii characters
c += 10
}
// TODO: idx 26 starts at 'ba', when it should start at 'aa'
res.insert(0, c)
}
return res.toString()
}
private fun cleanType(type: String): String {
var cleaned = type
if (cleaned.indexOf('<') != -1) {
cleaned = cleaned.substring(0, cleaned.indexOf('<'))
}
if (cleaned.indexOf('.') != -1) {
cleaned = cleaned.substring(cleaned.lastIndexOf('.') + 1)
}
if (cleaned.indexOf('$') != -1) {
cleaned = cleaned.substring(cleaned.lastIndexOf('$') + 1)
}
cleaned = cleaned.replace("\\[]".toRegex(), "")
return cleaned
}
override fun addParentContext(renamer: IVariableNameProvider) {
val prov = renamer as ParchmentNameProvider
usedNames.addAll(prov.usedNames)
}
class ParchmentNameProviderFactory(private val parchment: Parchment) : IVariableNamingFactory {
override fun createFactory(structMethod: StructMethod?): IVariableNameProvider {
return ParchmentNameProvider(parchment, structMethod!!)
}
}
}

View file

@ -1,23 +0,0 @@
package org.ecorous.esnesnon.gradle.vineflower
import org.ecorous.esnesnon.gradle.parchment.Parchment
import org.jetbrains.java.decompiler.api.plugin.Plugin
import org.jetbrains.java.decompiler.main.extern.IVariableNamingFactory
class RenamingPlugin : Plugin {
override fun id(): String {
return "NonsenseGradle"
}
override fun description(): String {
return "Renaming of parameters using the parchment mappings set"
}
override fun getRenamingFactory(): IVariableNamingFactory {
return ParchmentNameProvider.ParchmentNameProviderFactory(parchment)
}
companion object {
lateinit var parchment: Parchment
}
}

View file

@ -1 +0,0 @@
org.ecorous.esnesnon.gradle.vineflower.RenamingPlugin