add refmap generation and remapping

This commit is contained in:
moehreag 2024-10-02 16:17:05 +02:00
parent e25e598057
commit 54fb5fe91e
10 changed files with 408 additions and 34 deletions

View file

@ -24,7 +24,7 @@ repositories {
}
dependencies {
implementation("dev.frogmc:thyroxine:0.0.1-alpha.17")
implementation("dev.frogmc:thyroxine:0.0.1-alpha.18")
implementation("org.ow2.asm:asm:9.7")
implementation("org.ow2.asm:asm-commons:9.7")
implementation("org.ow2.asm:asm-tree:9.7")

View file

@ -0,0 +1,54 @@
package dev.frogmc.phytotelma.mixin.obfuscation;
import javax.annotation.processing.Filer;
import javax.annotation.processing.Messager;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.util.Map;
import java.util.stream.Collectors;
import dev.frogmc.thyroxine.api.data.MappingBundle;
import dev.frogmc.thyroxine.api.data.MappingData;
import dev.frogmc.thyroxine.parser.tiny.TinyV2Parser;
import org.spongepowered.asm.obfuscation.mapping.common.MappingField;
import org.spongepowered.asm.obfuscation.mapping.common.MappingMethod;
import org.spongepowered.tools.obfuscation.mapping.common.MappingProvider;
public class FrogMappingProvider extends MappingProvider {
private final Map<MappingMethod, MappingMethod> methodMap = getMap("methodMap");
private final Map<MappingField, MappingField> fieldMap = getMap("fieldMap");
private final Map<String, String> classMap = getMap("classMap");
public FrogMappingProvider(Messager messager, Filer filer) {
super(messager, filer);
}
@Override
public void read(File input) throws IOException {
MappingBundle bundle = TinyV2Parser.parse(Files.readString(input.toPath()));
MappingData data = bundle.data().getFirst();
classMap.putAll(data.classes());
fieldMap.putAll(data.fields().entrySet().stream().collect(Collectors.toMap(
e -> new MappingField(e.getKey().owner(), e.getKey().name(), e.getKey().descriptor()),
e -> new MappingField(e.getKey().owner(), e.getKey().name()))));
methodMap.putAll(data.methods().entrySet().stream().collect(Collectors.toMap(
e -> new MappingMethod(e.getKey().owner(), e.getKey().name(), e.getKey().descriptor()),
e -> new MappingMethod(e.getKey().owner(), e.getKey().name()))));
}
// Mixin has guava shadowed which means the actual type is not something we can import
@SuppressWarnings("unchecked")
private <K, V> Map<K, V> getMap(String name) {
try {
Field field = MappingProvider.class.getDeclaredField(name);
field.setAccessible(true);
return (Map<K, V>) field.get(this);
} catch (ReflectiveOperationException e) {
throw new RuntimeException(e);
}
}
}

View file

@ -0,0 +1,42 @@
package dev.frogmc.phytotelma.mixin.obfuscation;
import javax.annotation.processing.Filer;
import javax.annotation.processing.Messager;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collections;
import java.util.stream.Collectors;
import dev.frogmc.thyroxine.api.data.MappingBundle;
import dev.frogmc.thyroxine.api.data.MappingData;
import dev.frogmc.thyroxine.api.data.Member;
import dev.frogmc.thyroxine.writer.tiny.TinyV2Writer;
import org.spongepowered.asm.obfuscation.mapping.common.MappingField;
import org.spongepowered.asm.obfuscation.mapping.common.MappingMethod;
import org.spongepowered.tools.obfuscation.ObfuscationType;
import org.spongepowered.tools.obfuscation.mapping.IMappingConsumer;
import org.spongepowered.tools.obfuscation.mapping.common.MappingWriter;
public class FrogMappingWriter extends MappingWriter {
public FrogMappingWriter(Messager messager, Filer filer) {
super(messager, filer);
}
@Override
public void write(String output, ObfuscationType type, IMappingConsumer.MappingSet<MappingField> fields, IMappingConsumer.MappingSet<MappingMethod> methods) {
String[] namespaces = type.getKey().split(":");
MappingData data = new MappingData(namespaces[0], namespaces[1], Collections.emptyMap(),
methods.stream().collect(Collectors.<IMappingConsumer.MappingSet.Pair<MappingMethod>, Member, String>toMap(
p -> new Member(p.from.getOwner(), p.from.getName(), p.from.getDesc()), p -> p.to.getSimpleName())),
fields.stream().collect(Collectors.<IMappingConsumer.MappingSet.Pair<MappingField>, Member, String>toMap(
p -> new Member(p.from.getOwner(), p.from.getName(), p.from.getDesc()), p -> p.to.getSimpleName())),
Collections.emptyMap()
);
MappingBundle bundle = new MappingBundle(data);
try (PrintWriter writer = openFileWriter(output, type + " output mappings")) {
TinyV2Writer.write(bundle, writer);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

View file

@ -0,0 +1,25 @@
package dev.frogmc.phytotelma.mixin.obfuscation;
import javax.annotation.processing.Filer;
import javax.annotation.processing.Messager;
import org.spongepowered.tools.obfuscation.ObfuscationEnvironment;
import org.spongepowered.tools.obfuscation.ObfuscationType;
import org.spongepowered.tools.obfuscation.mapping.IMappingProvider;
import org.spongepowered.tools.obfuscation.mapping.IMappingWriter;
public class FrogObfuscationEnvironment extends ObfuscationEnvironment {
protected FrogObfuscationEnvironment(ObfuscationType type) {
super(type);
}
@Override
protected IMappingProvider getMappingProvider(Messager messager, Filer filer) {
return new FrogMappingProvider(messager, filer);
}
@Override
protected IMappingWriter getMappingWriter(Messager messager, Filer filer) {
return new FrogMappingWriter(messager, filer);
}
}

View file

@ -0,0 +1,60 @@
package dev.frogmc.phytotelma.mixin.obfuscation;
import java.util.*;
import org.spongepowered.tools.obfuscation.interfaces.IMixinAnnotationProcessor;
import org.spongepowered.tools.obfuscation.service.IObfuscationService;
import org.spongepowered.tools.obfuscation.service.ObfuscationTypeDescriptor;
public class FrogObfuscationService implements IObfuscationService {
private static final String IN_MAP_FILE = "inMapFile",
IN_MAP_EXTRA_FILES = "inMapExtraFiles",
OUT_MAP_FILE = "outMapFile";
private static String capitalize(String s) {
if (s.length() > 2) {
return Character.toUpperCase(s.charAt(0)) + s.substring(1);
} else if (!s.isEmpty()) {
return s.toUpperCase(Locale.ROOT);
}
return s;
}
private static String namespaced(String name, String from, String to) {
return name + capitalize(from) + capitalize(to);
}
private static void addOptions(Set<String> options, String from, String to) {
options.add(namespaced(IN_MAP_FILE, from, to));
options.add(namespaced(IN_MAP_EXTRA_FILES, from, to));
options.add(namespaced(OUT_MAP_FILE, from, to));
}
@Override
public Set<String> getSupportedOptions() {
Set<String> options = new HashSet<>();
addOptions(options, "moj", "dev");
addOptions(options, "dev", "moj");
addOptions(options, "moj", "moj");
return options;
}
private static ObfuscationTypeDescriptor createObfuscationType(String from, String to) {
return new ObfuscationTypeDescriptor(
from+":"+to,
namespaced(IN_MAP_FILE, from, to),
namespaced(IN_MAP_EXTRA_FILES, from, to),
namespaced(OUT_MAP_FILE, from, to),
FrogObfuscationEnvironment.class
);
}
@Override
public Collection<ObfuscationTypeDescriptor> getObfuscationTypes(IMixinAnnotationProcessor ap) {
return List.of(
createObfuscationType("moj", "dev"),
createObfuscationType("dev", "moj"),
createObfuscationType("moj", "moj")
);
}
}

View file

@ -0,0 +1 @@
dev.frogmc.phytotelma.mixin.obfuscation.FrogObfuscationService

View file

@ -2,4 +2,4 @@ plugins {
id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0"
}
rootProject.name = "phytotelma"
include("frog-mixin-obfuscation")

View file

@ -9,7 +9,7 @@ import dev.frogmc.phytotelma.common.Env
import dev.frogmc.phytotelma.ext.PhytotelmaGradleExtension
import dev.frogmc.phytotelma.ext.PhytotelmaGradleExtensionImpl
import dev.frogmc.phytotelma.mappings.renameDstNamespace
import dev.frogmc.phytotelma.mixin.remapper.MixinAnnotationRemapper
import dev.frogmc.phytotelma.mixin.remapper.RefmapRemapper
import dev.frogmc.phytotelma.nest.NestStripper
import dev.frogmc.phytotelma.run.AssetDownloader
import dev.frogmc.phytotelma.run.RunConfigGenerator
@ -229,7 +229,7 @@ class PhytotelmaPlugin : Plugin<Project> {
Built-For: Minecraft ${data.minecraftVersion}
Build-Date: ${LocalDateTime.now()}
""".trimIndent()
).plus(data.jarManifestProperties.entries.joinToString("\n") { it.key+": "+it.value})
).plus(data.jarManifestProperties.entries.joinToString("\n") { it.key + ": " + it.value })
manifest.writeLines(lines, StandardCharsets.UTF_8)
val metadata = fs.getPath(Constants.MOD_METADATA_FILE)
tomlParser.parse(metadata, FileNotFoundAction.READ_NOTHING)
@ -239,12 +239,15 @@ class PhytotelmaPlugin : Plugin<Project> {
remapAccesswidener(mappings, aw)
}
}
val fs = FileSystems.newFileSystem(ProjectStorage.get(project).remappedGameJarPath)
RefmapRemapper.remapRefmap(file.toPath(), Thyroxine.createMapper(mappings, listOf(fs)), temp)
fs.close()
Thyroxine.remap(
mappings,
temp,
file.toPath(),
false,
defaultRemappingSteps(ProjectStorage.get(project).remappedGameJarPath!!)
defaultRemappingSteps()
)
Files.deleteIfExists(temp)
}
@ -273,10 +276,9 @@ class PhytotelmaPlugin : Plugin<Project> {
}
}
private fun defaultRemappingSteps(vararg sourceNSJars: Path): MutableList<RemappingStep> {
private fun defaultRemappingSteps(): MutableList<RemappingStep> {
return mutableListOf(
RemappingStep(::ClassRemapper),
RemappingStep { cv, mapper -> MixinAnnotationRemapper(cv, mapper, *sourceNSJars) },
)
}
@ -298,15 +300,18 @@ class PhytotelmaPlugin : Plugin<Project> {
if (artifacts.isEmpty()) {
return
}
val mojmap = MojmapProvider.get(version,
val mojmap = MojmapProvider.get(
version,
globalCacheDir.resolve("net/minecraft/client/${version}/client-${version}.txt"),
globalCacheDir.resolve("net/minecraft/server/${version}/server-${version}.txt"))
globalCacheDir.resolve("net/minecraft/server/${version}/server-${version}.txt")
)
if (mojmapGameJar.notExists()) {
val remappedClient = officialClientJar.resolveSibling("client-$version-mojmap.jar")
val remappedServer = officialServerJar.resolveSibling("server-$version-mojmap.jar")
Thyroxine.remap(mojmap.data.first(), officialClientJar, remappedClient, true, false)
Thyroxine.remap(mojmap.data.first(), officialServerJar, remappedServer, true, false)
FileSystems.newFileSystem(mojmapGameJar, mutableMapOf<String, String>("create" to "true")).use { mergedFs ->
FileSystems.newFileSystem(mojmapGameJar, mutableMapOf<String, String>("create" to "true"))
.use { mergedFs ->
val consumer = object : Consumer<FileSystem> {
override fun accept(fs: FileSystem) {
Files.walkFileTree(fs.getPath("/"), object : SimpleFileVisitor<Path>() {
@ -375,17 +380,26 @@ class PhytotelmaPlugin : Plugin<Project> {
val temp = remappedPath.resolveSibling(remappedPath.fileName.toString() + ".tmp")
Thyroxine.remap(
mojOfficial, artifact.file.toPath(), temp, false, defaultRemappingSteps(
mojmapGameJar
), mojmapGameJar
mojOfficial, artifact.file.toPath(), temp, false, defaultRemappingSteps(), mojmapGameJar
)
Thyroxine.remap(
officialStore, temp, remappedPath, false, defaultRemappingSteps(
officialClientJar, officialServerJar
), officialClientJar, officialServerJar
officialStore, temp, remappedPath, false, defaultRemappingSteps(), officialClientJar, officialServerJar
)
Files.deleteIfExists(temp)
NestStripper.stripJij(remappedPath)
val filesystems = listOf(
FileSystems.newFileSystem(officialClientJar),
FileSystems.newFileSystem(officialServerJar)
)
RefmapRemapper.remapRefmap(
artifact.file.toPath(),
Thyroxine.createMapper(
MappingBundle(listOf(mojOfficial, officialStore), emptyList()).flattenData(),
filesystems
),
remappedPath
)
filesystems.forEach { it.close() }
FileSystems.newFileSystem(remappedPath).use { fs ->
val metadata = fs.getPath(Constants.MOD_METADATA_FILE)

View file

@ -1,6 +1,7 @@
package dev.frogmc.phytotelma.ext
import dev.frogmc.phytotelma.Constants
import dev.frogmc.phytotelma.PhytotelmaPlugin
import dev.frogmc.phytotelma.ProjectStorage
import dev.frogmc.phytotelma.VersionChecker
import dev.frogmc.phytotelma.accesswidener.AccessWidener
@ -13,17 +14,22 @@ import dev.frogmc.phytotelma.run.datagen.DatagenTask
import dev.frogmc.thyroxine.RemappingStep
import dev.frogmc.thyroxine.Thyroxine
import dev.frogmc.thyroxine.api.data.MappingData
import dev.frogmc.thyroxine.writer.tiny.TinyV2Writer
import org.gradle.api.Action
import org.gradle.api.Project
import org.gradle.api.model.ObjectFactory
import org.gradle.api.plugins.JavaPlugin
import org.gradle.internal.impldep.org.jsoup.helper.Consumer
import org.gradle.api.plugins.JavaPluginExtension
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.compile.JavaCompile
import java.nio.file.*
import java.nio.file.attribute.BasicFileAttributes
import java.util.function.Consumer
import javax.inject.Inject
import kotlin.io.path.copyTo
import kotlin.io.path.createParentDirectories
import kotlin.io.path.notExists
import kotlin.io.path.writer
abstract class PhytotelmaGradleExtensionImpl @Inject constructor(
private val project: Project,
@ -80,8 +86,8 @@ abstract class PhytotelmaGradleExtensionImpl @Inject constructor(
if (mergedJar.notExists() || applyAW || mcConf.mappingsName != projectData.mappingsName) {
projectData.mappingsName = mcConf.mappingsName
println("Merging game...")
FileSystems.newFileSystem(mergedJar, mutableMapOf<String, String>("create" to "true")).use { mergedFs ->
mergedJar.createParentDirectories()
FileSystems.newFileSystem(mergedJar, mapOf("create" to "true")).use { mergedFs ->
val consumer = object : Consumer<FileSystem> {
override fun accept(fs: FileSystem) {
Files.walkFileTree(fs.getPath("/"), object : SimpleFileVisitor<Path>() {
@ -99,6 +105,13 @@ abstract class PhytotelmaGradleExtensionImpl @Inject constructor(
FileSystems.newFileSystem(remappedClientJar).use { consumer.accept(it) }
FileSystems.newFileSystem(remappedServerJar).use { consumer.accept(it) }
}
VersionChecker.saveMergedPomFile(version, mergedJar.parent)
println("Writing mappings...")
TinyV2Writer.write(
mappings,
ProjectStorage.get(project).localCacheDir?.resolve("mappings.tiny")?.writer()
)
projectData.targetNamespace = mcConf.targetNamespace
@ -155,6 +168,46 @@ abstract class PhytotelmaGradleExtensionImpl @Inject constructor(
error("No loader version provided!")
}
project.dependencies.add(Constants.MINECRAFT_CONFIGURATION, "dev.frogmc:frogloader:${conf.version.get()}")
project.configurations.getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME)
.dependencies.find { it.group.equals("net.fabricmc") && it.name.equals("sponge-mixin") }
?.let { mixinDep ->
project.dependencies.add(
JavaPlugin.ANNOTATION_PROCESSOR_CONFIGURATION_NAME,
mixinDep.group + ":" + mixinDep.name + ":" + mixinDep.version
)
project.dependencies.add(
JavaPlugin.ANNOTATION_PROCESSOR_CONFIGURATION_NAME,
"dev.frogmc.phytotelma:frog-mixin-obfuscation:" + PhytotelmaPlugin::class.java.`package`.implementationVersion
)
val options = mapOf(
"defaultObfuscationEnv" to "dev:" + ProjectStorage.get(project).targetNamespace,
"quiet" to "true",
"inMapFileDevMoj" to ProjectStorage.get(project).localCacheDir!!.resolve("mappings.tiny"),
"outMapFileDevMoj" to project.layout.buildDirectory.file(
"mixin-out-" + ProjectStorage.get(project).mappingsName + "_" +
project.extensions.getByType(JavaPluginExtension::class.java).sourceSets.getByName(
SourceSet.MAIN_SOURCE_SET_NAME
)
)
.get().asFile,
"inMapExtraFilesDevMoj" to ProjectStorage.get(project).remappedGameJarPath,
)
val compileTasks = listOf(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaPlugin.COMPILE_TEST_JAVA_TASK_NAME)
compileTasks.forEach { name ->
project.tasks.getByName(name)
.takeIf { it is JavaCompile }
.let { it as JavaCompile }
.also { task ->
task.options.compilerArgs.addAll(options
.map { "-A" + it.key + "=" + it.value }
.plus(
"-AoutRefmapFile=" + task.destinationDirectory.asFile.get()
.resolve(project.name + "-refmap.json")
))
}
}
}
}
override fun froglib(action: Action<VersionConfiguration>) {

View file

@ -0,0 +1,125 @@
package dev.frogmc.phytotelma.mixin.remapper
import com.google.gson.Gson
import com.google.gson.JsonObject
import org.objectweb.asm.ClassReader
import org.objectweb.asm.Type
import org.objectweb.asm.commons.Remapper
import org.objectweb.asm.tree.ClassNode
import java.nio.charset.StandardCharsets
import java.nio.file.*
import kotlin.io.path.readBytes
import kotlin.io.path.reader
import kotlin.io.path.writer
object RefmapRemapper {
private val gson = Gson()
private const val MIXIN_ANNOTATION = "Lorg/spongepowered/asm/mixin/Mixin;"
fun remapRefmap(jar: Path, remapper: Remapper, outJar: Path) {
FileSystems.newFileSystem(jar).use { fs ->
Files.list(fs.getPath("/"))
.filter { it.fileName.toString().endsWith("-refmap.json") }
.forEach { refmap ->
val json = gson.fromJson(refmap.reader(), JsonObject::class.java)
val mappings = json.get("mappings").asJsonObject
val remapped = JsonObject()
val remappedMappings = JsonObject()
remapped.add("mappings", remappedMappings)
mappings.entrySet().forEach { mapping ->
val mixinName = mapping.key
val entries = mapping.value.asJsonObject
val remappedEntries = remapEntries(entries, fs, remapper)
remappedMappings.add(mixinName, remappedEntries)
}
val data = json.get("data").asJsonObject
val remappedData = JsonObject()
data.entrySet().forEach { entry ->
val namespaces = entry.key
val namespaceMappings = entry.value.asJsonObject
val remappedNS = JsonObject()
namespaceMappings.entrySet().forEach { mapping ->
val mixinName = mapping.key
val entries = mapping.value.asJsonObject
val remappedEntries = remapEntries(entries, fs, remapper)
remappedNS.add(mixinName, remappedEntries)
}
remappedData.add(namespaces, remappedNS)
}
remapped.add("data", remappedData)
FileSystems.newFileSystem(outJar).use { outFs ->
gson.toJson(
remapped, outFs.getPath(refmap.toString()).writer(
StandardCharsets.UTF_8,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
)
)
}
}
}
}
private fun remapEntries(entries: JsonObject, fs: FileSystem, remapper: Remapper): JsonObject {
val remappedEntries = JsonObject()
entries.entrySet().forEach {
val name = it.key
val value = it.value.asString
if (value.contains(":")) {
val field = value.split(":")
val target = getMixinTarget(fs.getPath("/$name.class"))
val fieldName = field.first()
val fieldDesc = field.last()
remappedEntries.addProperty(
name,
remapper.mapFieldName(target, fieldName, fieldDesc) + ":" + remapper.mapDesc(fieldDesc)
)
} else {
val method = value.split(";", limit = 2)
val owner = method.first() + ";"
val methodName = method.last().substringBefore("(")
val desc = method.last().substringAfter("(")
remappedEntries.addProperty(
methodName,
remapper.map(owner) + remapper.mapMethodName(owner, methodName, desc) + remapper.mapMethodDesc(desc)
)
}
}
return remappedEntries
}
private fun getMixinTarget(mixinClass: Path): String {
val reader = ClassReader(mixinClass.readBytes())
val node = ClassNode()
reader.accept(node, 0)
node.visibleAnnotations.forEach {
if (MIXIN_ANNOTATION == it.desc) {
it.values.forEach { value ->
when (value) {
is String -> return value
is Type -> return value.internalName
is List<*> -> value.forEach { c ->
when (c) {
is String -> return c
is Type -> return c.internalName
}
}
}
}
}
}
return ""
}
}