From 7a7c907abb5ee7caa19a97ddfa944d1d21f63b1f Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Sun, 1 Jun 2025 21:59:37 +0800 Subject: [PATCH] Tool verify --- game_re/gh_stub/r3_windowProc.cxx | 13 -- game_re/scan_sources | 10 +- java/ghidra/Decompile.java | 184 ------------------------- java/ghidra/RedumpStubFunctions.java | 37 +++++ java/ghidra/re3lib/FunctionDumper.java | 162 ++++++++++++---------- tooling/cmd_verify.cpp | 9 ++ tooling/database.cpp | 62 +++++++++ tooling/tool.cpp | 4 +- tooling/tool.hpp | 2 + 9 files changed, 203 insertions(+), 280 deletions(-) delete mode 100644 game_re/gh_stub/r3_windowProc.cxx delete mode 100644 java/ghidra/Decompile.java create mode 100644 java/ghidra/RedumpStubFunctions.java diff --git a/game_re/gh_stub/r3_windowProc.cxx b/game_re/gh_stub/r3_windowProc.cxx deleted file mode 100644 index 09c14b90..00000000 --- a/game_re/gh_stub/r3_windowProc.cxx +++ /dev/null @@ -1,13 +0,0 @@ -// AUTO-GENERATED FILE!!!! -// This function has yet to be decompiled using 'Dump Current Function' in ghidra -// with possible manualy fixes - -#include -#include -#include - -// 004025e0 -// r3_windowProc -extern "C" long r3_windowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - return gh_stub_impl_stdcall((void*)0x004025e0, hwnd, uMsg, wParam, lParam); -} diff --git a/game_re/scan_sources b/game_re/scan_sources index 4a63656d..f32ec27a 100644 --- a/game_re/scan_sources +++ b/game_re/scan_sources @@ -1,8 +1,8 @@ #!/bin/bash script_dir=$(readlink -f $(dirname "$0")) -tool=$script_dir/../tooling/bin/r3_gh_tool +tool=$script_dir/../tooling/bin/gh_tool -set -e +set -ex shopt -s nullglob pushd $script_dir @@ -19,12 +19,12 @@ for type in "${types[@]}"; do any_files=true done if [ "$any_files" = true ]; then - $tool "@$file_list" -v --type=$type --log-file=logs/log-functions-${type}.txt + $tool -v --log-file=logs/log-functions-${type}.txt functions "@$file_list" --type=$type fi fi done -$tool gh_global.h -mglobals -v --log-file=logs/log-globals.txt -$tool -mduplicates -v --log-file=logs/log-duplicates.txt +$tool -v --log-file=logs/log-globals.txt globals gh_global.h +$tool -v --log-file=logs/log-verify.txt verify popd diff --git a/java/ghidra/Decompile.java b/java/ghidra/Decompile.java deleted file mode 100644 index d3389168..00000000 --- a/java/ghidra/Decompile.java +++ /dev/null @@ -1,184 +0,0 @@ -// Script to export decompiled C code from Ghidra -// @category _Reman3 -// @menupath Reman3.Decompile All - -import java.io.File; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Hashtable; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.Arrays; -import java.util.Dictionary; - -import ghidra.app.decompiler.ClangFieldToken; -import ghidra.app.decompiler.ClangLine; -import ghidra.app.decompiler.ClangSyntaxToken; -import ghidra.app.decompiler.ClangToken; -import ghidra.app.decompiler.DecompileResults; -import ghidra.app.decompiler.PrettyPrinter; -import ghidra.app.script.GhidraScript; -import ghidra.program.model.address.Address; -import ghidra.program.model.data.AbstractStringDataType; -import ghidra.program.model.data.Array; -import ghidra.program.model.data.ArrayDataType; -import ghidra.program.model.data.BitFieldDataType; -import ghidra.program.model.data.DataType; -import ghidra.program.model.data.DataTypeComponent; -import ghidra.program.model.data.DataTypeManager; -import ghidra.program.model.data.EnumDataType; -import ghidra.program.model.data.PointerDataType; -import ghidra.program.model.data.ProgramBasedDataTypeManager; -import ghidra.program.model.data.Structure; -import ghidra.program.model.data.TypedefDataType; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.VariableStorage; -import ghidra.program.model.pcode.HighFunction; -import ghidra.program.model.pcode.HighSymbol; -import ghidra.program.model.pcode.PcodeOp; -import ghidra.program.model.pcode.Varnode; -import ghidra.program.model.symbol.Symbol; -import ghidra.util.task.TaskMonitor; -import re3lib.*; - -public class Decompile extends GhidraScript { - - - // void headerGuardPre(PrintWriter writer, String tag) { - // writer.println("#ifndef GH_GENERATED_" + tag + "_H"); - // writer.println("#define GH_GENERATED_" + tag + "_H"); - // writer.println(); - // } - - // void headerGuardPost(PrintWriter writer, String tag) { - // writer.println("#endif // GH_GENERATED_" + tag + "_H"); - // } - - // void dumpGlobals(Hashtable globalSymbols) throws Exception { - // File globalSymbolsListH = new File(RecompileConfig.INSTANCE.outputDir, "gh_global.h"); - // PrintWriter hwriter = new PrintWriter(globalSymbolsListH, "UTF-8"); - // hwriter.println("// AUTO-GENERATED FILE "); - // headerGuardPre(hwriter, "GLOBALS"); - // hwriter.println("#include "); - // hwriter.println(); - - // File globalSymbolsListC = new File(RecompileConfig.INSTANCE.outputDir, "gh_global.cxx"); - // PrintWriter cwriter = new PrintWriter(globalSymbolsListC, "UTF-8"); - // cwriter.println("// AUTO-GENERATED FILE "); - // cwriter.println("#include "); - // hwriter.println(); - - // for (GlobalRec sym : globalSymbols.values()) { - // HighSymbol highSym = sym.highSymbol; - // DataType dt = highSym.getDataType(); - // String dataType = dt.getDisplayName(); - // String sanitizedName = sanitizeFunctionName(highSym.getName()); - // String name = highSym.getName(); - // if (!sanitizedName.equals(name)) { - // println("Invalid global symbol name: " + name); - // name = sanitizedName; - // } - // Address addr = sym.address; - // // println("Symbol: " + symbol + " Addr: " + addr + " Size:" + symSize + " " + - // // storage.getSerializationString()); - // try { - // String initBlk = " = "; - // boolean fullyDefinedType = false; - // if (dt instanceof AbstractStringDataType) { - // AbstractStringDataType sdt = (AbstractStringDataType) dt; - // dataType = "const char*"; - // // String type - // initBlk += "\"" + escapeCString(readCString(addr, 2048)) + "\""; - // fullyDefinedType = true; - // } else if (dt instanceof PointerDataType) { - // PointerDataType pdt = (PointerDataType) dt; - // DataType baseType = pdt.getDataType(); - // dataType = baseType.getDisplayName() + "*"; - // initBlk += "(" + dataType + ")&GH_MEM(0x" + addr + ")"; - // fullyDefinedType = true; - // } - // if (fullyDefinedType) { - // hwriter.println("extern " + dataType + " " + name + "; // " + addr); - // cwriter.println(dataType + " " + name + initBlk + "; // " + addr); - // } else { - // if (dt instanceof Array) { - // // println("Array: " + dt.getDisplayName() + " - " + addr + " - " + - // // dt.getClass().getSimpleName()); - // Array adt = (Array) dt; - // DataType baseType = adt.getDataType(); - // hwriter.println( - // "extern " + baseType.getDisplayName() + "(&" + name + ")[" + adt.getNumElements() + "]; // " + addr); - // cwriter.println( - // baseType.getDisplayName() + "(&" + name + ")[" + adt.getNumElements() + "] = *reinterpret_cast<" - // + baseType.getDisplayName() + "(*)[" + adt.getNumElements() + "]>(GH_MEM(0x" + addr + "));"); - // } else { - // String refTypeStr = dt.getDisplayName() + "&"; - // hwriter.println("extern " + refTypeStr + " " + name + "; // " + addr); - // cwriter.println(dataType + " " + name + "= (" + refTypeStr + ") GH_MEM(0x" + addr + ");"); - // } - // } - // } catch (Exception e) { - // println("Error processing global symbol: " + e); - // println("Symbol: " + highSym.getName() + " - " + addr + " - " - // + highSym.getHighFunction().getFunction().getName()); - // } - // } - - // headerGuardPost(hwriter, "GLOBALS"); - // hwriter.close(); - // cwriter.close(); - // } - - void decompileAll(List functions) throws Exception { - // Hashtable globalSymbols = new Hashtable<>(); - - // for (Function function : functions) { - // decompileFunction(globalSymbols, function); - // } - - // dumpStructureTypes(); - // dumpGlobals(globalSymbols); - } - - @Override - public void run() throws Exception { - if (currentProgram == null) { - return; - } - - RemanConfig.INSTANCE = new RemanConfig(this); - - if (!new File(RemanConfig.INSTANCE.outputDir).exists()) { - throw new Exception("Output directory does not exist: " + RemanConfig.INSTANCE.outputDir); - } - - // Make sure to create output folders - RemanConfig.INSTANCE.dirDecompFix.mkdirs(); - RemanConfig.INSTANCE.dirDecompAuto.mkdirs(); - RemanConfig.INSTANCE.dirDecompRef.mkdirs(); - - // buildFunctionBlacklist(); - - List functions = new ArrayList<>(); - - Iterator functionsIt = currentProgram.getFunctionManager().getFunctions(true).iterator(); - while (functionsIt.hasNext()) { - Function function = functionsIt.next(); - // if (!shouldDecompileFunction(function)) { - // continue; - // } - - functions.add(function); - } - - decompileAll(functions); - } - - String sanitizeFunctionName(String name) { - return name.replaceAll("[^a-zA-Z0-9_]", "_"); - } -} diff --git a/java/ghidra/RedumpStubFunctions.java b/java/ghidra/RedumpStubFunctions.java new file mode 100644 index 00000000..ed5141cf --- /dev/null +++ b/java/ghidra/RedumpStubFunctions.java @@ -0,0 +1,37 @@ +// Script to regenerate all dumped stub functions +// @category _Reman3 +// @menupath Reman3.Redump Stub Functions + +import java.util.List; + +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.Function; +import re3lib.GlobalDumper; +import re3lib.RemanConfig; +import re3lib.TypeDumper; +import re3lib.FunctionDatabase; +import re3lib.FunctionDumper; + +public class RedumpStubFunctions extends GhidraScript { + @Override + protected void run() throws Exception { + RemanConfig.INSTANCE = new RemanConfig(this); + RemanConfig.INSTANCE.createDirectories(); + + try (FunctionDatabase functionDatabase = new FunctionDatabase(this)) { + List entries = functionDatabase.loadAllEntries(); + FunctionDumper dumper = new FunctionDumper(this, functionDatabase, null); + for (FunctionDatabase.FunctionEntry entry : entries) { + if (entry.type == FunctionDatabase.Type.Stub) { + Function function = getFunctionAt(entry.address); + if (function == null) { + printerr("Function not found at address: " + entry.address); + continue; + } + println("Dumping stub function: " + function.getName()); + dumper.dumpStubFunction(function); + } + } + } + } +} diff --git a/java/ghidra/re3lib/FunctionDumper.java b/java/ghidra/re3lib/FunctionDumper.java index 122cc051..81e7a7bf 100644 --- a/java/ghidra/re3lib/FunctionDumper.java +++ b/java/ghidra/re3lib/FunctionDumper.java @@ -108,6 +108,91 @@ public class FunctionDumper { } } + public void dumpStubFunction(Function externalFunction) throws Exception { + String sanitizedExtFunctionName = Utils.sanitizeIdentifier(externalFunction.getName()); + + List entries1 = functionDatabase + .findEntriesByAddress(externalFunction.getEntryPoint()); + boolean needStub = true; + for (FunctionDatabase.FunctionEntry entry : entries1) { + if (entry.type == FunctionDatabase.Type.Auto || entry.type == FunctionDatabase.Type.Fix) { + needStub = false; + break; + } + } + if (!needStub) + return; + + String fileName = sanitizedExtFunctionName + ".cxx"; + File f4 = new File(RemanConfig.INSTANCE.dirDecompStub, fileName); + script.println("Generating function stub for " + externalFunction.getName() + " => " + f4.toString()); + + try (PrintWriter writer2 = new PrintWriter(f4, "UTF-8")) { + writer2.println("// AUTO-GENERATED FILE!!!!"); + writer2.println("// This function has yet to be decompiled using 'Dump Current Function' in ghidra"); + writer2.println("// with possible manualy fixes"); + writer2.println(); + writer2.println("#include "); + writer2.println("#include "); + writer2.println("#include "); + writer2.println(); + writer2.println("// " + externalFunction.getEntryPoint()); + writer2.println("// " + externalFunction.getName()); + + // Parse function signature to extract calling convention, return type, and + // parameters + String signature = externalFunction.getSignature().getPrototypeString(false); + signature = signature.replace(externalFunction.getName(), sanitizedExtFunctionName); + script.println("Santized Signature: " + signature); + String callingConvention = externalFunction.getCallingConventionName(); + String returnType = externalFunction.getReturnType().toString(); + + // Generate function stub using appropriate forwarding function + writer2.println("extern \"C\" " + signature + " {"); + + // Determine which stub function to use based on calling convention + String stubFunction; + if (callingConvention != null && callingConvention.equals("__stdcall")) { + stubFunction = "gh_stub_impl_stdcall"; + } else { + // Default to cdecl for most cases + stubFunction = "gh_stub_impl_cdecl"; + } + + // Generate parameter list for the call + StringBuilder paramList = new StringBuilder(); + var params = externalFunction.getParameters(); + for (int i = 0; i < params.length; i++) { + if (i > 0) + paramList.append(", "); + paramList.append(params[i].getName()); + } + + // Generate the stub call + String addrString = "0x" + + externalFunction.getEntryPoint().toString().replace("0x", ""); + if (returnType.equals("void")) { + writer2.println(" " + stubFunction + "<" + addrString + ", void>(" + + (paramList.length() > 0 ? ", " + paramList.toString() : "") + ");"); + } else { + writer2.println(" return " + stubFunction + "<" + addrString + ", " + returnType + ">(" + + externalFunction.getEntryPoint().toString().replace("0x", "") + + (paramList.length() > 0 ? ", " + paramList.toString() : "") + ");"); + } + + writer2.println("}"); + } + + if (!f4.exists()) { + createdFile = true; + } + + // Add stub function to database + FunctionDatabase.FunctionEntry newEntry = new FunctionDatabase.FunctionEntry(externalFunction.getEntryPoint(), + externalFunction.getName(), f4, FunctionDatabase.Type.Stub); + functionDatabase.addEntryAt(newEntry); + } + public void dump(Function function) throws Exception { String sanitizedFunctionName = Utils.sanitizeIdentifier(function.getName()); @@ -346,82 +431,7 @@ public class FunctionDumper { // Possibly generate stubs for external functions for (Function externalFunction : externalFunctionCalls) { - String sanitizedExtFunctionName = Utils.sanitizeIdentifier(externalFunction.getName()); - - List entries1 = functionDatabase - .findEntriesByAddress(externalFunction.getEntryPoint()); - boolean needStub = true; - for (FunctionDatabase.FunctionEntry entry : entries1) { - if (entry.type == FunctionDatabase.Type.Auto || entry.type == FunctionDatabase.Type.Fix) { - needStub = false; - break; - } - } - if (!needStub) - continue; - - fileName = sanitizedExtFunctionName + ".cxx"; - File f4 = new File(RemanConfig.INSTANCE.dirDecompStub, fileName); - script.println("Generating function stub for " + externalFunction.getName() + " => " + f4.toString()); - - try (PrintWriter writer2 = new PrintWriter(f4, "UTF-8")) { - writer2.println("// AUTO-GENERATED FILE!!!!"); - writer2.println("// This function has yet to be decompiled using 'Dump Current Function' in ghidra"); - writer2.println("// with possible manualy fixes"); - writer2.println(); - writer2.println("#include "); - writer2.println("#include "); - writer2.println("#include "); - writer2.println(); - writer2.println("// " + externalFunction.getEntryPoint()); - writer2.println("// " + externalFunction.getName()); - - // Parse function signature to extract calling convention, return type, and - // parameters - String signature = externalFunction.getSignature().getPrototypeString(false); - signature = signature.replace(externalFunction.getName(), sanitizedExtFunctionName); - script.println("Santiziaed Signature: " + signature); - String callingConvention = externalFunction.getCallingConventionName(); - String returnType = externalFunction.getReturnType().toString(); - - // Generate function stub using appropriate forwarding function - writer2.println("extern \"C\" " + signature + " {"); - - // Determine which stub function to use based on calling convention - String stubFunction; - if (callingConvention != null && callingConvention.equals("__stdcall")) { - stubFunction = "gh_stub_impl_stdcall"; - } else { - // Default to cdecl for most cases - stubFunction = "gh_stub_impl_cdecl"; - } - - // Generate parameter list for the call - StringBuilder paramList = new StringBuilder(); - var params = externalFunction.getParameters(); - for (int i = 0; i < params.length; i++) { - if (i > 0) - paramList.append(", "); - paramList.append(params[i].getName()); - } - - // Generate the stub call - if (returnType.equals("void")) { - writer2.println(" " + stubFunction + "((void*)0x" + - externalFunction.getEntryPoint().toString().replace("0x", "") + - (paramList.length() > 0 ? ", " + paramList.toString() : "") + ");"); - } else { - writer2.println(" return " + stubFunction + "<" + returnType + ">((void*)0x" + - externalFunction.getEntryPoint().toString().replace("0x", "") + - (paramList.length() > 0 ? ", " + paramList.toString() : "") + ");"); - } - - writer2.println("}"); - } - - if (!f4.exists()) { - createdFile = true; - } + dumpStubFunction(externalFunction); } } } diff --git a/tooling/cmd_verify.cpp b/tooling/cmd_verify.cpp index 99ffc6a0..8f9acc07 100644 --- a/tooling/cmd_verify.cpp +++ b/tooling/cmd_verify.cpp @@ -2,6 +2,15 @@ #include bool processDuplicates(DatabaseManager &db) { + // Scan all files in the database, and for non-existing files, remove them from the database + auto files = db.getAllFiles(); + for (auto &file : files) { + if (!std::filesystem::exists(file)) { + spdlog::warn("File not found, removing from database: {}", file); + db.removeFile(file); + } + } + spdlog::info("=== Checking for duplicate addresses ==="); bool found_address_duplicates = db.checkDuplicateAddresses(); if (found_address_duplicates) { diff --git a/tooling/database.cpp b/tooling/database.cpp index abeb26b3..40c1668b 100644 --- a/tooling/database.cpp +++ b/tooling/database.cpp @@ -12,6 +12,9 @@ public: sqlite3_stmt *insert_imports_stmt; sqlite3_stmt *delete_globals_stmt; sqlite3_stmt *insert_globals_stmt; + sqlite3_stmt *delete_file_functions_stmt; + sqlite3_stmt *delete_file_imports_stmt; + sqlite3_stmt *delete_file_globals_stmt; void prepareStatement(const char *sql, sqlite3_stmt **stmt, const std::string &error_msg); @@ -38,6 +41,15 @@ public: "VALUES (?, ?, ?)", &insert_globals_stmt, "Failed to prepare insert globals statement"); + prepareStatement("DELETE FROM Functions WHERE filepath = ?", + &delete_file_functions_stmt, + "Failed to prepare delete file functions statement"); + prepareStatement("DELETE FROM Imports WHERE filepath = ?", + &delete_file_imports_stmt, + "Failed to prepare delete file imports statement"); + prepareStatement("DELETE FROM Globals WHERE filepath = ?", + &delete_file_globals_stmt, + "Failed to prepare delete file globals statement"); } ~PreparedStatements() { @@ -47,6 +59,9 @@ public: sqlite3_finalize(insert_imports_stmt); sqlite3_finalize(delete_globals_stmt); sqlite3_finalize(insert_globals_stmt); + sqlite3_finalize(delete_file_functions_stmt); + sqlite3_finalize(delete_file_imports_stmt); + sqlite3_finalize(delete_file_globals_stmt); } }; @@ -253,3 +268,50 @@ std::vector DatabaseManager::getFunctionsByType(FileType type) { sqlite3_finalize(stmt); return functions; } + +std::vector DatabaseManager::getAllFiles() { + std::vector files; + + const char *sql = R"( + SELECT DISTINCT filepath FROM ( + SELECT filepath FROM Functions + UNION + SELECT filepath FROM Imports + UNION + SELECT filepath FROM Globals + ) ORDER BY filepath; + )"; + + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr) != SQLITE_OK) { + spdlog::error("Failed to prepare getAllFiles query: {}", sqlite3_errmsg(db)); + return files; + } + + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *filepath = (const char *)sqlite3_column_text(stmt, 0); + if (filepath) { + files.push_back(std::string(filepath)); + } + } + + sqlite3_finalize(stmt); + return files; +} + +void DatabaseManager::removeFile(const std::string &filepath) { + // Use prepared statements for efficient deletion + sqlite3_stmt *stmts[] = { + prepared_stmts->delete_file_functions_stmt, + prepared_stmts->delete_file_imports_stmt, + prepared_stmts->delete_file_globals_stmt + }; + + for (auto stmt : stmts) { + sqlite3_reset(stmt); + sqlite3_bind_text(stmt, 1, filepath.c_str(), -1, SQLITE_STATIC); + sqlite3_step(stmt); + } + + spdlog::debug("Removed all entries for file: {}", filepath); +} diff --git a/tooling/tool.cpp b/tooling/tool.cpp index dfbbab42..cd55e0c7 100644 --- a/tooling/tool.cpp +++ b/tooling/tool.cpp @@ -31,9 +31,9 @@ int main(int argc, char *argv[]) { } }); app.add_flag("--log-file", options.log_file, "Enable logging to file") - ->each([&](std::string) { + ->each([&](const std::string& arg) { auto log_sink = std::make_shared( - options.log_file, true); + arg, true); console->sinks().push_back(log_sink); }); app.require_subcommand(); diff --git a/tooling/tool.hpp b/tooling/tool.hpp index 4abda1b1..856e3a71 100644 --- a/tooling/tool.hpp +++ b/tooling/tool.hpp @@ -69,6 +69,8 @@ public: bool checkDuplicateAddresses(); bool checkDuplicateNames(); std::vector getFunctionsByType(FileType type); + std::vector getAllFiles(); + void removeFile(const std::string &filepath); }; // File processing functions