Add script to rebuild and regenerate function database

scripts/DumpCurrentFunctionRecursive.java

@@ -11,6 +11,7 @@ import re3lib.FunctionDumper;
 import re3lib.GlobalDumper;
 import re3lib.PCallTracer;
 import re3lib.RecompileConfig;
+import re3lib.TypeDumper;
 
 public class DumpCurrentFunctionRecursive extends GhidraScript {
   @Override
@@ -21,7 +22,10 @@ public class DumpCurrentFunctionRecursive extends GhidraScript {
     GlobalDumper globalDumper = new GlobalDumper(this);
     globalDumper.loadGlobalManifest();
 
+    FunctionDumper functionDumper = new FunctionDumper(this, globalDumper);
+
     PCallTracer tracer = new PCallTracer();
+    tracer.setBlacklist(functionDumper.functionAddrBlackList);
     tracer.traceCalls(getFunctionContaining(currentAddress));
 
     List<Function> functionsToDump = new ArrayList<>();
@@ -56,7 +60,6 @@ public class DumpCurrentFunctionRecursive extends GhidraScript {
         return;
       }
 
-      FunctionDumper functionDumper = new FunctionDumper(this, globalDumper);
       for (Function func : functionsToDump) {
         functionDumper.dump(func);
       }
@@ -67,5 +70,9 @@ public class DumpCurrentFunctionRecursive extends GhidraScript {
       globalDumper.dumpGlobals();
       globalDumper.saveGlobalManifest();
     }
+
+    // Dump types
+    TypeDumper dumper = new TypeDumper(this);
+    dumper.run();
   }
 }

scripts/RebuildFunctionDatabase.java

@@ -0,0 +1,114 @@
+// @category _Reman3
+// @menupath Reman3.Rebuild Function Database
+
+import ghidra.app.script.GhidraScript;
+import ghidra.program.model.address.Address;
+import re3lib.FunctionDatabase;
+import re3lib.RecompileConfig;
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RebuildFunctionDatabase extends GhidraScript {
+  FunctionDatabase functionDB;
+
+  @Override
+  public void run() throws Exception {
+    RecompileConfig.INSTANCE = new RecompileConfig(this);
+
+    functionDB = new FunctionDatabase(this);
+
+    scanDirectory(RecompileConfig.INSTANCE.dirDecompAuto);
+    scanDirectory(RecompileConfig.INSTANCE.dirDecompFix);
+    scanDirectory(RecompileConfig.INSTANCE.dirDecompStub);
+
+    println("Applying default filters...");
+    functionDB.applyDefaultFilters();
+
+    println("Saving function database...");
+    functionDB.save();
+
+    println("Function database rebuilt successfully.");
+
+    // for (FunctionDatabase.Entry entry : functionDB.entries) {
+    //   println(entry.address + " " + entry.name + " " + entry.file.getName());
+    //   for (FunctionDatabase.Dependency dependency : entry.dependencies) {
+    //     println("  " + dependency.address + " " + dependency.name);
+    //   }
+    // }
+  }
+
+  private void scanDirectory(File directory) throws Exception {
+    File[] files = directory.listFiles((dir, name) -> name.endsWith(".cxx"));
+    if (files == null)
+      return;
+
+    for (File file : files) {
+      scanFile(file);
+    }
+  }
+
+  private void scanFile(File file) throws Exception {
+    println("Scanning " + file);
+    try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
+      String line;
+      Pattern dependencyPattern = Pattern.compile("(\\w+)\\s+(\\w+)\\(.*\\);\\s*//\\s*([0-9A-Fa-f]{8})\\s*//\\s*(.*)");
+      Pattern addressPattern = Pattern.compile("//\\s*([0-9A-Fa-f]{8})");
+      Pattern functionNamePattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s*\\(");
+
+      List<FunctionDatabase.Dependency> dependencies = new ArrayList<>();
+      String address = null;
+      String functionName = null;
+
+      while ((line = reader.readLine()) != null) {
+        Matcher dependencyMatcher = dependencyPattern.matcher(line);
+        if (dependencyMatcher.find()) {
+          // println("Found dependency: " + dependencyMatcher.group(3));
+          Address depAddress = currentProgram.getAddressFactory().getAddress(dependencyMatcher.group(3));
+          String name = dependencyMatcher.group(2);
+          FunctionDatabase.Dependency dependency = functionDB.new Dependency(depAddress, name);
+          dependencies.add(dependency);
+          continue;
+        }
+
+        Matcher addressMatcher = addressPattern.matcher(line);
+        if (addressMatcher.find()) {
+          // println("Found address: " + addressMatcher.group(1));
+          address = addressMatcher.group(1);
+          // Skip any comments or newlines between address and function definition
+          while ((line = reader.readLine()) != null) {
+            line = line.trim();
+            // println("Line: " + line);
+            if (!line.isEmpty()) {
+              Matcher functionNameMatcher = functionNamePattern.matcher(line);
+              if (functionNameMatcher.find()) {
+                functionName = functionNameMatcher.group(2).trim();
+                break;
+              }
+            }
+          }
+          if (functionName != null) {
+            break;
+          }
+        }
+      }
+
+      if (address != null && functionName != null) {
+        Address functionAddress = currentProgram.getAddressFactory().getAddress(address);
+        FunctionDatabase.Entry entry = functionDB.new Entry();
+        entry.address = functionAddress;
+        entry.name = functionName;
+        entry.file = file;
+        entry.dependencies = dependencies;
+        functionDB.entries.add(entry);
+      } else {
+        throw new Exception("Failed to parse function at " + file.getName());
+      }
+    }
+  }
+}

scripts/re3lib/FunctionDatabase.java

@@ -0,0 +1,223 @@
+package re3lib;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import ghidra.app.script.GhidraScript;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.listing.Function;
+
+public class FunctionDatabase {
+  public class Dependency implements java.io.Serializable {
+    private static final long serialVersionUID = 1L;
+    public Address address;
+    public String name;
+
+    public Dependency(Address address, String name) {
+      this.address = address;
+      this.name = name;
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      out.writeObject(address != null ? address.toString() : null);
+      out.writeObject(name);
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      String addressString = (String) in.readObject();
+      if (addressString != null) {
+        address = RecompileConfig.INSTANCE.script.getCurrentProgram().getAddressFactory().getAddress(addressString);
+      }
+      name = (String) in.readObject();
+    }
+  }
+
+  public class Entry implements java.io.Serializable {
+    private static final long serialVersionUID = 1L;
+    public Address address;
+    public String name;
+    public File file;
+    public List<Dependency> dependencies = new ArrayList<>();
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      out.writeObject(address != null ? address.toString() : null);
+      out.writeObject(name);
+      out.writeObject(file != null ? file.toString() : null);
+      out.writeObject(dependencies);
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      String addressString = (String) in.readObject();
+      if (addressString != null) {
+        address = RecompileConfig.INSTANCE.script.getCurrentProgram().getAddressFactory().getAddress(addressString);
+      }
+      name = (String) in.readObject();
+      String fileString = (String) in.readObject();
+      if (fileString != null) {
+        file = new File(fileString);
+      }
+      dependencies = (List<Dependency>) in.readObject();
+    }
+  }
+
+  public List<Entry> entries = new ArrayList<>();
+  private File file;
+  private transient GhidraScript script;
+
+  public FunctionDatabase(GhidraScript script) {
+    this.script = script;
+    file = new File(RecompileConfig.INSTANCE.outputDir, "functions.dat");
+  }
+
+  public void load() throws Exception {
+    if (!file.exists()) {
+      return;
+    }
+
+    try (java.io.ObjectInputStream ois = new java.io.ObjectInputStream(new java.io.FileInputStream(file))) {
+      entries = (List<Entry>) ois.readObject();
+      script.println("Loaded " + entries.size() + " function entries from " + file);
+    } catch (java.io.IOException | ClassNotFoundException e) {
+      script.println("Error loading function database: " + e.getMessage());
+    }
+  }
+
+  public void save() throws Exception {
+    try (java.io.ObjectOutputStream oos = new java.io.ObjectOutputStream(new java.io.FileOutputStream(file))) {
+      oos.writeObject(entries);
+      script.println("Saved " + entries.size() + " function entries to " + file);
+    } catch (java.io.IOException e) {
+      script.println("Error saving function database: " + e.getMessage());
+    }
+  }
+
+  public void add(Entry entry) {
+    entries.add(entry);
+  }
+
+  public void applyDefaultFilters() throws Exception {
+    GlobalDumper globalDumper = new GlobalDumper(script);
+    FunctionDumper dumper = new FunctionDumper(script, globalDumper);
+
+    boolean madeAnyChanges = false;
+
+    // Create a hash map to store symbol names
+    Map<Address, String> symbolNames = new HashMap<>();
+    Map<String, File> exportedFunctionNames = new HashMap<>();
+    for (Entry entry : entries) {
+      Function function = script.getFunctionAt(entry.address);
+      if (function != null) {
+        String dirComponent = entry.file.getParent().toString();
+        boolean isAuto = dirComponent.startsWith(RecompileConfig.INSTANCE.dirDecompAuto.toString());
+        boolean isFix = dirComponent.startsWith(RecompileConfig.INSTANCE.dirDecompFix.toString());
+        // Get the actual symbol name and store it in the hash map
+        String symbolName = function.getName();
+        symbolNames.put(entry.address, symbolName);
+
+        if (isAuto && !exportedFunctionNames.containsKey(entry.name)) {
+          exportedFunctionNames.put(entry.name, entry.file);
+        } else if (isFix) {
+          exportedFunctionNames.replace(entry.name, entry.file);
+        }
+      }
+    }
+
+    // Print the number of symbol names collected
+    script.println("Collected " + symbolNames.size() + " symbol names");
+
+    boolean dryMode = true;
+
+    HashSet<Function> functionsToRegenerate = new HashSet<>();
+
+    Iterator<Entry> iterator = entries.iterator();
+    while (iterator.hasNext()) {
+      Entry entry = iterator.next();
+      Function function = script.getFunctionAt(entry.address);
+
+      boolean pendingDelete = false;
+      boolean pendingRegenerate = false;
+
+      // Remove CRT and other blacklisted functions
+      if (function == null || !dumper.isValidFunction(function)) {
+        // Remove the file
+        if (entry.file != null && entry.file.exists()) {
+          script.println("Removed file: " + entry.file.getAbsolutePath());
+          pendingDelete = true;
+        }
+
+        // Remove entry from the list
+        script.println("Removed invalid function entry: " + entry.name + " at " + entry.address);
+        function = null;
+      }
+
+      // Check if symbol name matches the symbol name parsed from the file
+      if (function != null) {
+        String actualSymbolName = symbolNames.get(entry.address);
+        if (actualSymbolName == null) {
+          throw new Exception(
+              "Symbol name not found for function at " + entry.address + " in file " + entry.file.getAbsolutePath());
+        }
+
+        if (actualSymbolName != null && !actualSymbolName.equals(entry.name)) {
+          File fnExportedFile = exportedFunctionNames.get(entry.name);
+          if (fnExportedFile != null && fnExportedFile != entry.file) {
+            // Already exists elsewhere, so remove this file
+            script.println("Removing duplicate function: " + entry.name + " at " + entry.address + " overridden by "
+                + fnExportedFile);
+            pendingDelete = true;
+          } else {
+            // Regeneral this function
+            script.println("Symbol name mismatch for function at " + entry.address + ": " +
+                "File name: " + entry.name + ", Actual symbol: " + actualSymbolName);
+            entry.name = actualSymbolName; // Update the entry name to match the actual symbol
+            pendingRegenerate = true;
+          }
+        }
+
+        // Check if dependencies are valid
+        for (Dependency dependency : entry.dependencies) {
+          Function depFunction = script.getFunctionAt(dependency.address);
+          if (depFunction == null) {
+            script.println("Dependency not found: " + dependency.name + " at " + dependency.address + " in " + entry.file);
+            pendingRegenerate = true;
+          } else if (!dumper.isValidFunction(depFunction) || !depFunction.getName().equals(dependency.name)) {
+            script
+                .println("Invalid dependency: " + dependency.name + " at " + dependency.address + " in " + entry.file + " should be " + dependency.name);
+            pendingRegenerate = true;
+          }
+        }
+
+        entry.name = actualSymbolName; // Update the entry name to match the actual symbol
+        madeAnyChanges = true;
+      }
+
+      if (pendingDelete) {
+        iterator.remove();
+        if (!dryMode) {
+          entry.file.delete();
+          madeAnyChanges = true;
+        }
+      } else if (pendingRegenerate) {
+        if (!dryMode) {
+          functionsToRegenerate.add(function);
+          madeAnyChanges = true;
+        }
+      }
+    }
+
+    for (Function function : functionsToRegenerate) {
+      script.println("Regenerating function: " + function.getName() + " at " + function.getEntryPoint());
+      dumper.dump(function);
+    }
+
+    if (madeAnyChanges) {
+      // Update CMake timestamp
+      RecompileConfig.INSTANCE.touchCMakeTimestamp();
+    }
+  }
+}

scripts/re3lib/FunctionDumper.java

@@ -28,7 +28,8 @@ import re3lib.GlobalDumper.GlobalRec;
 public class FunctionDumper {
   GhidraScript script;
   GlobalDumper globalDumper;
-  HashSet<Address> functionAddrBlackList = new HashSet<>();
+
+  public HashSet<Address> functionAddrBlackList = new HashSet<>();
 
   public boolean createdFile = false;
   // Collects functions called by the current function
@@ -188,7 +189,7 @@ public class FunctionDumper {
           ClangToken token = tokens.get(t);
 
           boolean thisDot = false;
-          // println("Token: " + token.toString());
+          script.println("Token: " + token.toString());
           if (token.toString().equals(".")) {
             // println("Found dot: " + token.toString() + " - " + token.getClass());
             thisDot = true;
@@ -232,10 +233,12 @@ public class FunctionDumper {
 
             // Check if it's a function pointer, otherwise add to globals
             if (address.isMemoryAddress()) {
+              // script.println("Address: " + address + " - " + sym.getName());
               Function maybeFunction = script.getFunctionAt(address);
               if (maybeFunction != null) {
                 externalFunctionCalls.add(maybeFunction);
               } else {
+                script.println("Add globals " + address + " - " + gsym.getName() + " - " + token.getText());
                 globalDumper.addGlobal(address, gsym);
               }
             }

scripts/re3lib/GlobalDumper.java

@@ -113,15 +113,11 @@ public class GlobalDumper {
         // script.println("DATA: " + addr + " - " + dt.getDisplayName());
       }
     }
-    // if(sym.getName().startsWith("s_SALIR")) {
-    // script.println("SALIR: " + addr + " - " + dt.getDisplayName() + " - " +
-    // dt.getClass().getName());
-    // // script.println("SALIR: " + sym.getdata());
-    // }
     if (dt == null) {
       script.println("WARNING: Missing type for global: " + sym.getName() + " at " + addr);
       return;
     }
+    // script.println("Global: " + addr + " - " + sym.getName() + " - " + dt.getDisplayName());
     globalAddrs.put(addr, new GlobalRec(addr, sym.getName(), dt));
   }