Tool verify

This commit is contained in:
Guus Waals 2025-06-01 21:59:37 +08:00
parent 6bce41bd2d
commit 7a7c907abb
9 changed files with 203 additions and 280 deletions

View File

@ -1,13 +0,0 @@
// AUTO-GENERATED FILE!!!!
// This function has yet to be decompiled using 'Dump Current Function' in ghidra
// with possible manualy fixes
#include <r3/binders/auto.h>
#include <r3/binders/stub.h>
#include <gh_global.h>
// 004025e0
// r3_windowProc
extern "C" long r3_windowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
return gh_stub_impl_stdcall<long>((void*)0x004025e0, hwnd, uMsg, wParam, lParam);
}

View File

@ -1,8 +1,8 @@
#!/bin/bash
script_dir=$(readlink -f $(dirname "$0"))
tool=$script_dir/../tooling/bin/r3_gh_tool
tool=$script_dir/../tooling/bin/gh_tool
set -e
set -ex
shopt -s nullglob
pushd $script_dir
@ -19,12 +19,12 @@ for type in "${types[@]}"; do
any_files=true
done
if [ "$any_files" = true ]; then
$tool "@$file_list" -v --type=$type --log-file=logs/log-functions-${type}.txt
$tool -v --log-file=logs/log-functions-${type}.txt functions "@$file_list" --type=$type
fi
fi
done
$tool gh_global.h -mglobals -v --log-file=logs/log-globals.txt
$tool -mduplicates -v --log-file=logs/log-duplicates.txt
$tool -v --log-file=logs/log-globals.txt globals gh_global.h
$tool -v --log-file=logs/log-verify.txt verify
popd

View File

@ -1,184 +0,0 @@
// Script to export decompiled C code from Ghidra
// @category _Reman3
// @menupath Reman3.Decompile All
import java.io.File;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Arrays;
import java.util.Dictionary;
import ghidra.app.decompiler.ClangFieldToken;
import ghidra.app.decompiler.ClangLine;
import ghidra.app.decompiler.ClangSyntaxToken;
import ghidra.app.decompiler.ClangToken;
import ghidra.app.decompiler.DecompileResults;
import ghidra.app.decompiler.PrettyPrinter;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.address.Address;
import ghidra.program.model.data.AbstractStringDataType;
import ghidra.program.model.data.Array;
import ghidra.program.model.data.ArrayDataType;
import ghidra.program.model.data.BitFieldDataType;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.DataTypeComponent;
import ghidra.program.model.data.DataTypeManager;
import ghidra.program.model.data.EnumDataType;
import ghidra.program.model.data.PointerDataType;
import ghidra.program.model.data.ProgramBasedDataTypeManager;
import ghidra.program.model.data.Structure;
import ghidra.program.model.data.TypedefDataType;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.VariableStorage;
import ghidra.program.model.pcode.HighFunction;
import ghidra.program.model.pcode.HighSymbol;
import ghidra.program.model.pcode.PcodeOp;
import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.symbol.Symbol;
import ghidra.util.task.TaskMonitor;
import re3lib.*;
public class Decompile extends GhidraScript {
// void headerGuardPre(PrintWriter writer, String tag) {
// writer.println("#ifndef GH_GENERATED_" + tag + "_H");
// writer.println("#define GH_GENERATED_" + tag + "_H");
// writer.println();
// }
// void headerGuardPost(PrintWriter writer, String tag) {
// writer.println("#endif // GH_GENERATED_" + tag + "_H");
// }
// void dumpGlobals(Hashtable<String, GlobalRec> globalSymbols) throws Exception {
// File globalSymbolsListH = new File(RecompileConfig.INSTANCE.outputDir, "gh_global.h");
// PrintWriter hwriter = new PrintWriter(globalSymbolsListH, "UTF-8");
// hwriter.println("// AUTO-GENERATED FILE ");
// headerGuardPre(hwriter, "GLOBALS");
// hwriter.println("#include <r3/binder/global.h>");
// hwriter.println();
// File globalSymbolsListC = new File(RecompileConfig.INSTANCE.outputDir, "gh_global.cxx");
// PrintWriter cwriter = new PrintWriter(globalSymbolsListC, "UTF-8");
// cwriter.println("// AUTO-GENERATED FILE ");
// cwriter.println("#include <r3/binder/global.h>");
// hwriter.println();
// for (GlobalRec sym : globalSymbols.values()) {
// HighSymbol highSym = sym.highSymbol;
// DataType dt = highSym.getDataType();
// String dataType = dt.getDisplayName();
// String sanitizedName = sanitizeFunctionName(highSym.getName());
// String name = highSym.getName();
// if (!sanitizedName.equals(name)) {
// println("Invalid global symbol name: " + name);
// name = sanitizedName;
// }
// Address addr = sym.address;
// // println("Symbol: " + symbol + " Addr: " + addr + " Size:" + symSize + " " +
// // storage.getSerializationString());
// try {
// String initBlk = " = ";
// boolean fullyDefinedType = false;
// if (dt instanceof AbstractStringDataType) {
// AbstractStringDataType sdt = (AbstractStringDataType) dt;
// dataType = "const char*";
// // String type
// initBlk += "\"" + escapeCString(readCString(addr, 2048)) + "\"";
// fullyDefinedType = true;
// } else if (dt instanceof PointerDataType) {
// PointerDataType pdt = (PointerDataType) dt;
// DataType baseType = pdt.getDataType();
// dataType = baseType.getDisplayName() + "*";
// initBlk += "(" + dataType + ")&GH_MEM(0x" + addr + ")";
// fullyDefinedType = true;
// }
// if (fullyDefinedType) {
// hwriter.println("extern " + dataType + " " + name + "; // " + addr);
// cwriter.println(dataType + " " + name + initBlk + "; // " + addr);
// } else {
// if (dt instanceof Array) {
// // println("Array: " + dt.getDisplayName() + " - " + addr + " - " +
// // dt.getClass().getSimpleName());
// Array adt = (Array) dt;
// DataType baseType = adt.getDataType();
// hwriter.println(
// "extern " + baseType.getDisplayName() + "(&" + name + ")[" + adt.getNumElements() + "]; // " + addr);
// cwriter.println(
// baseType.getDisplayName() + "(&" + name + ")[" + adt.getNumElements() + "] = *reinterpret_cast<"
// + baseType.getDisplayName() + "(*)[" + adt.getNumElements() + "]>(GH_MEM(0x" + addr + "));");
// } else {
// String refTypeStr = dt.getDisplayName() + "&";
// hwriter.println("extern " + refTypeStr + " " + name + "; // " + addr);
// cwriter.println(dataType + " " + name + "= (" + refTypeStr + ") GH_MEM(0x" + addr + ");");
// }
// }
// } catch (Exception e) {
// println("Error processing global symbol: " + e);
// println("Symbol: " + highSym.getName() + " - " + addr + " - "
// + highSym.getHighFunction().getFunction().getName());
// }
// }
// headerGuardPost(hwriter, "GLOBALS");
// hwriter.close();
// cwriter.close();
// }
void decompileAll(List<Function> functions) throws Exception {
// Hashtable<String, GlobalRec> globalSymbols = new Hashtable<>();
// for (Function function : functions) {
// decompileFunction(globalSymbols, function);
// }
// dumpStructureTypes();
// dumpGlobals(globalSymbols);
}
@Override
public void run() throws Exception {
if (currentProgram == null) {
return;
}
RemanConfig.INSTANCE = new RemanConfig(this);
if (!new File(RemanConfig.INSTANCE.outputDir).exists()) {
throw new Exception("Output directory does not exist: " + RemanConfig.INSTANCE.outputDir);
}
// Make sure to create output folders
RemanConfig.INSTANCE.dirDecompFix.mkdirs();
RemanConfig.INSTANCE.dirDecompAuto.mkdirs();
RemanConfig.INSTANCE.dirDecompRef.mkdirs();
// buildFunctionBlacklist();
List<Function> functions = new ArrayList<>();
Iterator<Function> functionsIt = currentProgram.getFunctionManager().getFunctions(true).iterator();
while (functionsIt.hasNext()) {
Function function = functionsIt.next();
// if (!shouldDecompileFunction(function)) {
// continue;
// }
functions.add(function);
}
decompileAll(functions);
}
String sanitizeFunctionName(String name) {
return name.replaceAll("[^a-zA-Z0-9_]", "_");
}
}

View File

@ -0,0 +1,37 @@
// Script to regenerate all dumped stub functions
// @category _Reman3
// @menupath Reman3.Redump Stub Functions
import java.util.List;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.listing.Function;
import re3lib.GlobalDumper;
import re3lib.RemanConfig;
import re3lib.TypeDumper;
import re3lib.FunctionDatabase;
import re3lib.FunctionDumper;
public class RedumpStubFunctions extends GhidraScript {
@Override
protected void run() throws Exception {
RemanConfig.INSTANCE = new RemanConfig(this);
RemanConfig.INSTANCE.createDirectories();
try (FunctionDatabase functionDatabase = new FunctionDatabase(this)) {
List<FunctionDatabase.FunctionEntry> entries = functionDatabase.loadAllEntries();
FunctionDumper dumper = new FunctionDumper(this, functionDatabase, null);
for (FunctionDatabase.FunctionEntry entry : entries) {
if (entry.type == FunctionDatabase.Type.Stub) {
Function function = getFunctionAt(entry.address);
if (function == null) {
printerr("Function not found at address: " + entry.address);
continue;
}
println("Dumping stub function: " + function.getName());
dumper.dumpStubFunction(function);
}
}
}
}
}

View File

@ -108,6 +108,91 @@ public class FunctionDumper {
}
}
public void dumpStubFunction(Function externalFunction) throws Exception {
String sanitizedExtFunctionName = Utils.sanitizeIdentifier(externalFunction.getName());
List<FunctionDatabase.FunctionEntry> entries1 = functionDatabase
.findEntriesByAddress(externalFunction.getEntryPoint());
boolean needStub = true;
for (FunctionDatabase.FunctionEntry entry : entries1) {
if (entry.type == FunctionDatabase.Type.Auto || entry.type == FunctionDatabase.Type.Fix) {
needStub = false;
break;
}
}
if (!needStub)
return;
String fileName = sanitizedExtFunctionName + ".cxx";
File f4 = new File(RemanConfig.INSTANCE.dirDecompStub, fileName);
script.println("Generating function stub for " + externalFunction.getName() + " => " + f4.toString());
try (PrintWriter writer2 = new PrintWriter(f4, "UTF-8")) {
writer2.println("// AUTO-GENERATED FILE!!!!");
writer2.println("// This function has yet to be decompiled using 'Dump Current Function' in ghidra");
writer2.println("// with possible manualy fixes");
writer2.println();
writer2.println("#include <r3/binders/auto.h>");
writer2.println("#include <r3/binders/stub.h>");
writer2.println("#include <gh_global.h>");
writer2.println();
writer2.println("// " + externalFunction.getEntryPoint());
writer2.println("// " + externalFunction.getName());
// Parse function signature to extract calling convention, return type, and
// parameters
String signature = externalFunction.getSignature().getPrototypeString(false);
signature = signature.replace(externalFunction.getName(), sanitizedExtFunctionName);
script.println("Santized Signature: " + signature);
String callingConvention = externalFunction.getCallingConventionName();
String returnType = externalFunction.getReturnType().toString();
// Generate function stub using appropriate forwarding function
writer2.println("extern \"C\" " + signature + " {");
// Determine which stub function to use based on calling convention
String stubFunction;
if (callingConvention != null && callingConvention.equals("__stdcall")) {
stubFunction = "gh_stub_impl_stdcall";
} else {
// Default to cdecl for most cases
stubFunction = "gh_stub_impl_cdecl";
}
// Generate parameter list for the call
StringBuilder paramList = new StringBuilder();
var params = externalFunction.getParameters();
for (int i = 0; i < params.length; i++) {
if (i > 0)
paramList.append(", ");
paramList.append(params[i].getName());
}
// Generate the stub call
String addrString = "0x" +
externalFunction.getEntryPoint().toString().replace("0x", "");
if (returnType.equals("void")) {
writer2.println(" " + stubFunction + "<" + addrString + ", void>(" +
(paramList.length() > 0 ? ", " + paramList.toString() : "") + ");");
} else {
writer2.println(" return " + stubFunction + "<" + addrString + ", " + returnType + ">(" +
externalFunction.getEntryPoint().toString().replace("0x", "") +
(paramList.length() > 0 ? ", " + paramList.toString() : "") + ");");
}
writer2.println("}");
}
if (!f4.exists()) {
createdFile = true;
}
// Add stub function to database
FunctionDatabase.FunctionEntry newEntry = new FunctionDatabase.FunctionEntry(externalFunction.getEntryPoint(),
externalFunction.getName(), f4, FunctionDatabase.Type.Stub);
functionDatabase.addEntryAt(newEntry);
}
public void dump(Function function)
throws Exception {
String sanitizedFunctionName = Utils.sanitizeIdentifier(function.getName());
@ -346,82 +431,7 @@ public class FunctionDumper {
// Possibly generate stubs for external functions
for (Function externalFunction : externalFunctionCalls) {
String sanitizedExtFunctionName = Utils.sanitizeIdentifier(externalFunction.getName());
List<FunctionDatabase.FunctionEntry> entries1 = functionDatabase
.findEntriesByAddress(externalFunction.getEntryPoint());
boolean needStub = true;
for (FunctionDatabase.FunctionEntry entry : entries1) {
if (entry.type == FunctionDatabase.Type.Auto || entry.type == FunctionDatabase.Type.Fix) {
needStub = false;
break;
}
}
if (!needStub)
continue;
fileName = sanitizedExtFunctionName + ".cxx";
File f4 = new File(RemanConfig.INSTANCE.dirDecompStub, fileName);
script.println("Generating function stub for " + externalFunction.getName() + " => " + f4.toString());
try (PrintWriter writer2 = new PrintWriter(f4, "UTF-8")) {
writer2.println("// AUTO-GENERATED FILE!!!!");
writer2.println("// This function has yet to be decompiled using 'Dump Current Function' in ghidra");
writer2.println("// with possible manualy fixes");
writer2.println();
writer2.println("#include <r3/binders/auto.h>");
writer2.println("#include <r3/binders/stub.h>");
writer2.println("#include <gh_global.h>");
writer2.println();
writer2.println("// " + externalFunction.getEntryPoint());
writer2.println("// " + externalFunction.getName());
// Parse function signature to extract calling convention, return type, and
// parameters
String signature = externalFunction.getSignature().getPrototypeString(false);
signature = signature.replace(externalFunction.getName(), sanitizedExtFunctionName);
script.println("Santiziaed Signature: " + signature);
String callingConvention = externalFunction.getCallingConventionName();
String returnType = externalFunction.getReturnType().toString();
// Generate function stub using appropriate forwarding function
writer2.println("extern \"C\" " + signature + " {");
// Determine which stub function to use based on calling convention
String stubFunction;
if (callingConvention != null && callingConvention.equals("__stdcall")) {
stubFunction = "gh_stub_impl_stdcall";
} else {
// Default to cdecl for most cases
stubFunction = "gh_stub_impl_cdecl";
}
// Generate parameter list for the call
StringBuilder paramList = new StringBuilder();
var params = externalFunction.getParameters();
for (int i = 0; i < params.length; i++) {
if (i > 0)
paramList.append(", ");
paramList.append(params[i].getName());
}
// Generate the stub call
if (returnType.equals("void")) {
writer2.println(" " + stubFunction + "<void>((void*)0x" +
externalFunction.getEntryPoint().toString().replace("0x", "") +
(paramList.length() > 0 ? ", " + paramList.toString() : "") + ");");
} else {
writer2.println(" return " + stubFunction + "<" + returnType + ">((void*)0x" +
externalFunction.getEntryPoint().toString().replace("0x", "") +
(paramList.length() > 0 ? ", " + paramList.toString() : "") + ");");
}
writer2.println("}");
}
if (!f4.exists()) {
createdFile = true;
}
dumpStubFunction(externalFunction);
}
}
}

View File

@ -2,6 +2,15 @@
#include <CLI11.hpp>
bool processDuplicates(DatabaseManager &db) {
// Scan all files in the database, and for non-existing files, remove them from the database
auto files = db.getAllFiles();
for (auto &file : files) {
if (!std::filesystem::exists(file)) {
spdlog::warn("File not found, removing from database: {}", file);
db.removeFile(file);
}
}
spdlog::info("=== Checking for duplicate addresses ===");
bool found_address_duplicates = db.checkDuplicateAddresses();
if (found_address_duplicates) {

View File

@ -12,6 +12,9 @@ public:
sqlite3_stmt *insert_imports_stmt;
sqlite3_stmt *delete_globals_stmt;
sqlite3_stmt *insert_globals_stmt;
sqlite3_stmt *delete_file_functions_stmt;
sqlite3_stmt *delete_file_imports_stmt;
sqlite3_stmt *delete_file_globals_stmt;
void prepareStatement(const char *sql, sqlite3_stmt **stmt,
const std::string &error_msg);
@ -38,6 +41,15 @@ public:
"VALUES (?, ?, ?)",
&insert_globals_stmt,
"Failed to prepare insert globals statement");
prepareStatement("DELETE FROM Functions WHERE filepath = ?",
&delete_file_functions_stmt,
"Failed to prepare delete file functions statement");
prepareStatement("DELETE FROM Imports WHERE filepath = ?",
&delete_file_imports_stmt,
"Failed to prepare delete file imports statement");
prepareStatement("DELETE FROM Globals WHERE filepath = ?",
&delete_file_globals_stmt,
"Failed to prepare delete file globals statement");
}
~PreparedStatements() {
@ -47,6 +59,9 @@ public:
sqlite3_finalize(insert_imports_stmt);
sqlite3_finalize(delete_globals_stmt);
sqlite3_finalize(insert_globals_stmt);
sqlite3_finalize(delete_file_functions_stmt);
sqlite3_finalize(delete_file_imports_stmt);
sqlite3_finalize(delete_file_globals_stmt);
}
};
@ -253,3 +268,50 @@ std::vector<FunctionInfo> DatabaseManager::getFunctionsByType(FileType type) {
sqlite3_finalize(stmt);
return functions;
}
std::vector<std::string> DatabaseManager::getAllFiles() {
std::vector<std::string> files;
const char *sql = R"(
SELECT DISTINCT filepath FROM (
SELECT filepath FROM Functions
UNION
SELECT filepath FROM Imports
UNION
SELECT filepath FROM Globals
) ORDER BY filepath;
)";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr) != SQLITE_OK) {
spdlog::error("Failed to prepare getAllFiles query: {}", sqlite3_errmsg(db));
return files;
}
while (sqlite3_step(stmt) == SQLITE_ROW) {
const char *filepath = (const char *)sqlite3_column_text(stmt, 0);
if (filepath) {
files.push_back(std::string(filepath));
}
}
sqlite3_finalize(stmt);
return files;
}
void DatabaseManager::removeFile(const std::string &filepath) {
// Use prepared statements for efficient deletion
sqlite3_stmt *stmts[] = {
prepared_stmts->delete_file_functions_stmt,
prepared_stmts->delete_file_imports_stmt,
prepared_stmts->delete_file_globals_stmt
};
for (auto stmt : stmts) {
sqlite3_reset(stmt);
sqlite3_bind_text(stmt, 1, filepath.c_str(), -1, SQLITE_STATIC);
sqlite3_step(stmt);
}
spdlog::debug("Removed all entries for file: {}", filepath);
}

View File

@ -31,9 +31,9 @@ int main(int argc, char *argv[]) {
}
});
app.add_flag("--log-file", options.log_file, "Enable logging to file")
->each([&](std::string) {
->each([&](const std::string& arg) {
auto log_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(
options.log_file, true);
arg, true);
console->sinks().push_back(log_sink);
});
app.require_subcommand();

View File

@ -69,6 +69,8 @@ public:
bool checkDuplicateAddresses();
bool checkDuplicateNames();
std::vector<FunctionInfo> getFunctionsByType(FileType type);
std::vector<std::string> getAllFiles();
void removeFile(const std::string &filepath);
};
// File processing functions