WIP Recompile

This commit is contained in:
Guus Waals 2024-09-16 01:42:27 +08:00
parent 5ac970ecdc
commit 556fa70207
16 changed files with 2115 additions and 197 deletions

9
.clangd Normal file
View File

@ -0,0 +1,9 @@
CompileFlags:
# CompilationDatabase: build\emscripten-mt\debug
CompilationDatabase: build\clang-x86_64-pc-windows-msvc\debug
Add:
- -IC:/Projects/R3/game_re/binders
# - -D_HAS_CXX20=1
# - -D_HAS_CXX17=1
# - -std=c++20
# - -DSH_USE_THREAD_FIBER=1

2
.gitignore vendored
View File

@ -14,5 +14,5 @@ windows_libs_other/
ACP_Ray2/
build/
game_re/
# game_re/
windows_libs/

View File

@ -1,4 +1,7 @@
cmake_minimum_required(VERSION 3.27.0)
project(reman3)
add_subdirectory(game_re)
add_subdirectory(game_re)
# Use highest possible C standard
set_target_properties(game_re PROPERTIES C_STANDARD 23)

17
game_re/CMakeLists.txt Normal file
View File

@ -0,0 +1,17 @@
add_executable(game_re
main.cpp
gh_global.cxx
binders/gh_static_mem.cxx)
target_include_directories(game_re PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/binders)
file(GLOB GH_AUTO_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gh_auto/*.cxx)
file(GLOB GH_FIX_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gh_fix/*.cxx)
target_sources(game_re PRIVATE
)
target_sources(game_re PRIVATE
${GH_AUTO_SOURCES}
${GH_FIX_SOURCES}
)

View File

@ -0,0 +1,8 @@
#ifndef DF40ED2B_5659_43AA_8A22_499A89C4BD62
#define DF40ED2B_5659_43AA_8A22_499A89C4BD62
#include "gh_types.h"
#include "gh_static_mem.h"
#include "../gh_global.h"
#endif /* DF40ED2B_5659_43AA_8A22_499A89C4BD62 */

View File

@ -0,0 +1,3 @@
#include "gh_static_mem.h"
unsigned char gh_static_mem[GH_STATIC_MEM_SIZE];

View File

@ -0,0 +1,12 @@
#ifndef A18E3E17_2A80_4DBD_96CD_1CE0120A164E
#define A18E3E17_2A80_4DBD_96CD_1CE0120A164E
#define GH_STATIC_MEM_START 0x005b6400
#define GH_STATIC_MEM_END 0x00843fff
#define GH_STATIC_MEM_SIZE (GH_STATIC_MEM_END - GH_STATIC_MEM_START)
extern unsigned char gh_static_mem[GH_STATIC_MEM_SIZE];
#define GH_STATIC(addr) (*(unsigned char *)(gh_static_mem + (addr - GH_STATIC_MEM_START)))
#endif /* A18E3E17_2A80_4DBD_96CD_1CE0120A164E */

View File

@ -0,0 +1,20 @@
#ifndef A523F6DB_0645_4DEB_8AEB_3792CB732B49
#define A523F6DB_0645_4DEB_8AEB_3792CB732B49
// Header included in all decompiled files to fix some common recompilation issues
#include <stdint.h>
#include <Windows.h>
#include <stdio.h>
// Fallback to int if type is not annotated
typedef int64_t undefined;
typedef uint32_t undefined4;
typedef uint16_t undefined2;
typedef unsigned char byte;
typedef byte undefined1;
typedef uint64_t uint;
typedef void* pointer;
typedef char *TerminatedCString;
typedef char *string;
#endif /* A523F6DB_0645_4DEB_8AEB_3792CB732B49 */

202
game_re/blacklist.txt Normal file
View File

@ -0,0 +1,202 @@
0055164b // FUN_0055164b
00404e5b // FUN_00404e5b
00403a5a // FUN_00403a5a
00550643 // __frnd
00407853 // FUN_00407853
0040924c // FUN_0040924c
0040744a // FUN_0040744a
0055265d // FUN_0055265d
00551451 // FUN_00551451
00550a57 // __startOneArgErrorHandling
00550655 // FUN_00550655
0040747b // FUN_0040747b
00409c7b // FUN_00409c7b
0054d26e // __fassign
00404073 // crt_0
00407672 // FUN_00407672
00407870 // FUN_00407870
00405270 // __aulldiv
0040926d // FUN_0040926d
00409e6d // FUN_00409e6d
0054d070 // __aullshr
00550077 // FUN_00550077
0040981f // FUN_0040981f
0040421d // FUN_0040421d
00409419 // FUN_00409419
0054c600 // ___timet_from_ft
0040a011 // FUN_0040a011
00552c04 // FUN_00552c04
00407e10 // FUN_00407e10
0040280d // FUN_0040280d
0040880c // FUN_0040880c
00404a0a // __abnormal_termination
00402c08 // FUN_00402c08
00551e13 // FUN_00551e13
00551810 // FUN_00551810
00551e16 // FUN_00551e16
00403a02 // FUN_00403a02
00407e3f // FUN_00407e3f
0040703f // FUN_0040703f
0055262f // FUN_0055262f
00404039 // FUN_00404039
00407439 // FUN_00407439
00408837 // crt_1
00404a36 // FUN_00404a36
0054be20 // _strncat
00402c31 // enterAllocCriticalSection
0054d424 // FUN_0054d424
0054ee3b // __ctrandisp2
00404a2d // __NLG_Notify1
00407628 // FUN_00407628
0055183d // FUN_0055183d
00406827 // FUN_00406827
0054f030 // __trandisp1
00409c23 // FUN_00409c23
00405a21 // leaveAllocCriticalSection
0054d4cb // FUN_0054d4cb
00403ade // FUN_00403ade
00404cce // __nh_malloc
00404ece // FUN_00404ece
004076ce // FUN_004076ce
005506d8 // FUN_005506d8
004040cb // crt_main1?
00407ec8 // FUN_00407ec8
004028c4 // c_call_static_init_fns
004026c0 // _strncpy
00402cc0 // trimCharFromStringEnd?
00403efb // crt_postMainExcept
00404cfa // FUN_00404cfa
004096f5 // FUN_004096f5
00407ef0 // _memset
00550ae5 // __fload_withFB
004068e9 // FUN_004068e9
005514f0 // FUN_005514f0
00405ee2 // malloc?
00409ce2 // FUN_00409ce2
004028e0 // FUN_004028e0
004052e0 // __aullrem
0055268a // FUN_0055268a
0040709e // FUN_0040709e
0054ee8a // FUN_0054ee8a
0040629c // FUN_0040629c
00409a9c // FUN_00409a9c
0054ee83 // FUN_0054ee83
00551286 // FUN_00551286
00402c92 // FUN_00402c92
00402a90 // _strrchr
00405490 // FUN_00405490
0055149a // FUN_0055149a
0040668d // FUN_0040668d
0054f097 // __trandisp2
00403a81 // FUN_00403a81
0051e090 // RtlUnwind
004046bf // crt_mainTls?
00404cbc // _malloc
004028bb // FUN_004028bb
00408abb // FUN_00408abb
0054d2ac // FUN_0054d2ac
004028b2 // FUN_004028b2
00406cb1 // FUN_00406cb1
00405aaa // FUN_00405aaa
004076a5 // FUN_004076a5
005518b4 // FUN_005518b4
004096a0 // __allmul
00550b4b // __math_exit
00551149 // FUN_00551149
00406d5a // FUN_00406d5a
00409559 // FUN_00409559
00551d43 // FUN_00551d43
0054af41 // crt_set_cfltcvt_etc
00407f50 // FUN_00407f50
0054d55e // __cfltcvt
00405b46 // FUN_00405b46
00552751 // FUN_00552751
00406940 // FUN_00406940
0054b354 // __ftol
00406f7e // FUN_00406f7e
00409772 // __mbsnbicoll
00404b69 // FUN_00404b69
0055157c // FUN_0055157c
00404d61 // FUN_00404d61
00403b61 // __freebuf
00404960 // __global_unwind2
0054b90e // operator_new
0054d30d // FUN_0054d30d
00404b15 // __seh_longjmp_unwind@4
00404713 // FUN_00404713
00551910 // FUN_00551910
00404503 // crt_initConsole
00404902 // crt_createProgramHeap
00408f01 // FUN_00408f01
0055032a // FUN_0055032a
0054cf20 // _longjmp
00404f33 // FUN_00404f33
00404b30 // FUN_00404b30
00405130 // _strlen
00408b30 // _strcmp
00404726 // initStaticUnk0
005505c9 // FUN_005505c9
0054d3cf // FUN_0054d3cf
005517cd // FUN_005517cd
00406dd6 // getFileTruncateStaticUnk1
0051e7c7 // `vector_constructor_iterator'
00551bc0 // __allshl
0054b3c0 // __alldiv
0054edc0 // __cintrindisp2
004067d1 // FUN_004067d1
004043d1 // crt_setupEnv
00407dd0 // _strncmp
00406dcd // getFileTruncateStaticUnk0
005507d9 // FUN_005507d9
004095c8 // FUN_004095c8
00408fc5 // FUN_00408fc5
005525d1 // ___add_12
00404dc0 // FUN_00404dc0
00406ffd // FUN_00406ffd
004027fc // __exit
004029fb // __fclose_lk
0054efee // __fload
005517e3 // FUN_005517e3
005529e4 // FUN_005529e4
004089ef // FUN_004089ef
004027eb // crt_postMain0
004061eb // FUN_004061eb
0054edfe // __cintrindisp1
004085e8 // FUN_004085e8
00402be4 // crt_exitWithFailure
0054ddf0 // __CallSettingFrame@12
0054b388 // _rand
00403998 // somethingWithEnvVar
00408f97 // FUN_00408f97
00406397 // FUN_00406397
00405b8e // FUN_00405b8e
00406b8d // FUN_00406b8d
0040478d // FUN_0040478d
00550799 // FUN_00550799
00405988 // allocateArray?
0054cf9c // __setjmp3
00406985 // FUN_00406985
00404184 // crt_main2?
00551597 // FUN_00551597
00408980 // FUN_00408980
00402bbf // __amsg_exit
004027be // c_static_init
00407bbc // FUN_00407bbc
00409fbb // FUN_00409fbb
0054d5af // FUN_0054d5af
00403bba // FUN_00403bba
004047ba // FUN_004047ba
0054d1ae // crt_unk?
00405bb9 // FUN_00405bb9
005515a3 // FUN_005515a3
005507a7 // FUN_005507a7
004097b1 // FUN_004097b1
004053b0 // FUN_004053b0
0054efbb // __ctrandisp1
005515be // FUN_005515be
00407ba5 // FUN_00407ba5
005525b0 // FUN_005525b0
005507b6 // FUN_005507b6
004049a2 // __local_unwind2
005517b7 // FUN_005517b7
004053a0 // FUN_004053a0

0
game_re/functions.txt Normal file
View File

2
game_re/gh_global.cxx Normal file
View File

@ -0,0 +1,2 @@
// AUTO-GENERATED FILE
#include <gh_auto_shared.h>

2
game_re/gh_global.h Normal file
View File

@ -0,0 +1,2 @@
// AUTO-GENERATED FILE
#include <gh_auto_shared.h>

8
game_re/main.cpp Normal file
View File

@ -0,0 +1,8 @@
#include <windows.h>
int r3_main(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR *cmdline,
int showCmd);
int main() {
r3_main(GetModuleHandle(NULL), NULL, NULL, SW_SHOW);
return 0;
}

12
game_re/readme.md Normal file
View File

@ -0,0 +1,12 @@
# Source structure
Since many source files are automatically generated from ghidra, the file names correspond to the function names inside ghidra, one source file per function.
For each required function in the program the file will be either in gh_auto or gh_fix
## gh_auto
Contains the decompiled ghidra source files
## gh_fix
Contains decompiled source functions, manually fixed up

File diff suppressed because it is too large Load Diff

View File

@ -16,14 +16,22 @@ import java.util.List;
import java.util.Scanner;
import java.util.Arrays;
import ghidra.app.cmd.label.AddLabelCmd;
import ghidra.app.decompiler.ClangMarkup;
import ghidra.app.decompiler.ClangNode;
import ghidra.app.decompiler.ClangTokenGroup;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileResults;
import ghidra.app.decompiler.DecompiledFunction;
import ghidra.app.script.GhidraScript;
import ghidra.docking.settings.Settings;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressFactory;
import ghidra.program.model.data.AbstractStringDataType;
import ghidra.program.model.data.BuiltInDataType;
import ghidra.program.model.data.DataOrganization;
import ghidra.program.model.data.DataOrganizationImpl;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.PointerDataType;
import ghidra.program.model.data.StringDataInstance;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.Variable;
@ -39,6 +47,7 @@ import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.symbol.SourceType;
import ghidra.program.model.symbol.Symbol;
import ghidra.program.model.symbol.SymbolTable;
import ghidra.program.model.symbol.SymbolType;
public class DecompileC extends GhidraScript {
public class PCallTracer {
@ -113,6 +122,10 @@ public class DecompileC extends GhidraScript {
private static final String OUTPUT_DIR = "game_re";
private static final int TIMEOUT = 10000;
// The static memory block
private Address staticMemoryBlockStart;
private Address staticMemoryBlockEnd;
// Auto rename invalid symbols
private static final boolean AUTO_RENAME_SYMBOLS = true;
@ -129,7 +142,6 @@ public class DecompileC extends GhidraScript {
void loadFunctionBlacklist() {
functionAddrBlackList.clear();
File blacklistFile = new File(outputDir, "blacklist.txt");
try (Scanner scanner = new Scanner(blacklistFile)) {
while (scanner.hasNextLine()) {
@ -261,93 +273,143 @@ public class DecompileC extends GhidraScript {
}
String escapeCString(String str) {
str = str.replaceAll("\"", "\\\"");
str = str.replaceAll("\n", "\\n");
str = str.replaceAll("\r", "\\r");
str = str.replaceAll("\t", "\\t");
str = str.replaceAll("\b", "\\b");
str = str.replaceAll("\f", "\\f");
str = str.replaceAll("\0", "\\0");
str = str.replace("\\", "\\\\");
str = str.replace("\"", "\\\"");
// str = str.replaceAll("\n", "\\n");
// str = str.replaceAll("\r", "\\r");
// str = str.replaceAll("\t", "\\t");
// str = str.replaceAll("\b", "\\b");
// str = str.replaceAll("\f", "\\f");
// str = str.replaceAll("\0", "\\0");
return str;
}
String readCString(Address addr, int maxLen) throws Exception {
StringBuilder sb = new StringBuilder();
int ofs = 0;
while (true) {
Address read = addr.add(ofs++);
// println("Reading: " + read);
byte b = currentProgram.getMemory().getByte(read);
// println("Read: " + b);
if (b == 0 || ofs >= maxLen) {
break;
}
sb.append((char) b);
}
if (sb.length() > 0) {
println("STR \"" + sb.toString() + "\"");
}
return sb.toString();
}
void decompileFunction(Hashtable<String, HighSymbol> outGlobalSymbols, DecompInterface decomp, Function function)
throws Exception {
String fileName = sanitizeFunctionName(function.getName()) + ".cxx";
File f1 = new File(dirDecompFix, fileName);
if (f1.exists()) {
println("Func " + function.getName() + " skipped (gh_fix)");
return;
}
File f0 = new File(dirDecompAuto, fileName);
if (f0.exists()) {
f0.delete();
}
println("Processing " + function.getName() + " => " + f0.toString());
DecompileResults decompRes = decomp.decompileFunction(function, TIMEOUT, monitor);
PrintWriter writer2 = new PrintWriter(f0, "UTF-8");
writer2.println("// AUTO-GENERATED FILE, MOVE TO 'gh_fix' FOLDER PREVENT OVERWRITING!!!!! ");
writer2.println("// " + function.getEntryPoint());
writer2.println();
writer2.println("#include <gh_auto_shared.h>");
writer2.println("#include \"../gh_global.h\"");
writer2.println();
HighFunction highFunction = decompRes.getHighFunction();
// ClangTokenGroup
// ClangNode.
// ClangTokenGroup ctg = decompRes.getCCodeMarkup();
// for (ClangTokenGroup it = ctg.groupIterator(); it.hasNext();) {
// }
writer2.println(cm.getCode());
writer2.close();
// Collect referenced global symbols
Iterator<HighSymbol> smyIt = decompRes.getHighFunction().getGlobalSymbolMap().getSymbols();
while (smyIt.hasNext()) {
HighSymbol gsym = smyIt.next();
if (outGlobalSymbols.containsKey(gsym.getName()))
continue;
outGlobalSymbols.put(gsym.getName(), gsym);
}
}
void decompileAll(DecompInterface decomp, List<Function> functions) throws Exception {
Hashtable<String, HighSymbol> globalSymbols = new Hashtable<>();
for (Function function : functions) {
String fileName = sanitizeFunctionName(function.getName()) + ".c";
File f1 = new File(dirDecompFix, fileName);
if (f1.exists()) {
println("Func " + function.getName() + " skipped (gh_fix)");
continue;
}
File f0 = new File(dirDecompAuto, fileName);
if (f0.exists()) {
f0.delete();
}
println("Processing " + function.getName() + " => " + f0.toString());
DecompileResults decompRes = decomp.decompileFunction(function, TIMEOUT, monitor);
PrintWriter writer2 = new PrintWriter(f0, "UTF-8");
writer2.println("// AUTO-GENERATED FILE, MOVE TO 'gh_fix' FOLDER PREVENT OVERWRITING!!!!! ");
writer2.println("// " + function.getEntryPoint());
writer2.println();
writer2.println("#include <gh_auto_shared.h>");
writer2.println();
writer2.println(decompRes.getDecompiledFunction().getC());
writer2.close();
// Collect referenced global symbols
Iterator<HighSymbol> smyIt = decompRes.getHighFunction().getGlobalSymbolMap().getSymbols();
while (smyIt.hasNext()) {
HighSymbol gsym = smyIt.next();
if (globalSymbols.containsKey(gsym.getName()))
continue;
globalSymbols.put(gsym.getName(), gsym);
}
decompileFunction(globalSymbols, decomp, function);
}
writer.close();
File globalSymbolsListH = new File(outputDir, "gh_global.h");
File globalSymbolsListC = new File(outputDir, "gh_global.c");
PrintWriter hwriter = new PrintWriter(globalSymbolsListH, "UTF-8");
hwriter.println("// AUTO-GENERATED FILE ");
hwriter.println("#include <gh_auto_shared.h>");
File globalSymbolsListC = new File(outputDir, "gh_global.cxx");
PrintWriter cwriter = new PrintWriter(globalSymbolsListC, "UTF-8");
for (HighSymbol sym : globalSymbols.values()) {
DataType dt = sym.getDataType();
String sanitizedName = sanitizeFunctionName(sym.getName());
if (!sanitizedName.equals(sym.getName())) {
println("Invalid global symbol name: " + sym.getName() + " - " + sym.getHighFunction().getFunction().getName());
cwriter.println("// AUTO-GENERATED FILE ");
cwriter.println("#include <gh_auto_shared.h>");
for (HighSymbol highSym : globalSymbols.values()) {
DataType dt = highSym.getDataType();
String dataType = dt.getDisplayName();
String name = highSym.getName();
String sanitizedName = sanitizeFunctionName(highSym.getName());
if (!sanitizedName.equals(highSym.getName())) {
println("Invalid global symbol name: " + highSym.getName() + " - "
+ highSym.getHighFunction().getFunction().getName());
} else {
Address address = sym.getStorage().getMinAddress();
MemoryBlock block = currentProgram.getMemory().getBlock(address);
String dataType = dt.toString();
String name = sym.getName();
if (block == null) {
println("Can not read variable " + name + " (" + dataType + ") at " + address);
continue;
Symbol symbol = highSym.getSymbol();
VariableStorage storage = highSym.getStorage();
Address addr = storage.getMinAddress();
int symSize = highSym.getSize();
if (addr == null) {
// Not sure why this is sometimes null
// also when it is not null, Symbol.getAddress() is not correct but very small
// like 00000056
// Not that storage will be <undefined> so maybe can check that
addr = symbol.getAddress();
}
if (dt instanceof AbstractStringDataType) {
// String type
hwriter.println("extern " + dataType + " " + name + "; // " + address);
String srcBlock = "";
// Read the actual string data from Ghidra
if (block != null && block.isInitialized()) {
byte[] bytes = new byte[dt.getLength()];
block.getBytes(address, bytes);
// Parse from UTF-8
String stringValue = new String(bytes, StandardCharsets.UTF_8);
srcBlock = dataType + " " + name + " = \"" + escapeCString(stringValue) + "\";";
println("Symbol: " + symbol + " Addr: " + addr + " Size:" + symSize + " " + storage.getSerializationString());
try {
String initBlk = " = ";
if (dt instanceof AbstractStringDataType) {
AbstractStringDataType sdt = (AbstractStringDataType) dt;
dataType = "const char*";
// String type
initBlk += "\"" + escapeCString(readCString(addr, 2048)) + "\"";
} else if (dt instanceof PointerDataType) {
PointerDataType pdt = (PointerDataType) dt;
DataType baseType = pdt.getDataType();
dataType = baseType.getDisplayName() + "*";
initBlk += "gh_ptr(0x" + addr + ")";
} else {
initBlk = " = 0";
}
cwriter.println(srcBlock + " // " + address);
cwriter.println(dataType + " " + name + initBlk + "; // " + addr);
} catch (Exception e) {
println("Error processing global symbol: " + e);
println("Symbol: " + highSym.getName() + " - " + addr + " - "
+ highSym.getHighFunction().getFunction().getName());
}
hwriter.println("extern " + dataType + " " + name + "; // " + addr);
}
}
hwriter.close();
@ -360,6 +422,9 @@ public class DecompileC extends GhidraScript {
return;
}
staticMemoryBlockStart = currentProgram.getAddressFactory().getAddress("005b6400");
staticMemoryBlockEnd = currentProgram.getAddressFactory().getAddress("00843fff");
// Make sure to create OUTPUT_PATH
rootDir = new File(sourceFile.getAbsolutePath()).getParentFile().getParentFile();
outputDir = new File(rootDir, OUTPUT_DIR);
@ -390,15 +455,11 @@ public class DecompileC extends GhidraScript {
functions.add(function);
}
// File functionList = new File(outputDir, "functions.txt");
// PrintWriter writer = new PrintWriter(functionList, "UTF-8");
int mode = 1;
if (mode == 0) { // Sanitize symbols
sanitizeGlobalSymbolsPass(decomp, functions);
} else if (mode == 1) { // Decompile all functions
decompileAll(decomp, functions);
}
}