diff --git a/.gitmodules b/.gitmodules index 9f144335..e9ea04f7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "game_re/third_party/spdlog"] path = game_re/third_party/spdlog url = https://github.com/gabime/spdlog.git +[submodule "tooling2/third_party/tree-sitter"] + path = tooling2/third_party/tree-sitter + url = https://github.com/guusw/tree-sitter.git +[submodule "tooling2/third_party/tree-sitter-cpp"] + path = tooling2/third_party/tree-sitter-cpp + url = https://github.com/guusw/tree-sitter-cpp.git diff --git a/java/cparser/.gitignore b/java/cparser/.gitignore deleted file mode 100644 index 1ff28072..00000000 --- a/java/cparser/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.class -target/ \ No newline at end of file diff --git a/java/cparser/pom.xml b/java/cparser/pom.xml deleted file mode 100644 index fea56ca6..00000000 --- a/java/cparser/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - 4.0.0 - - cparser - cparser - 1.0-SNAPSHOT - - - - - junit - junit - 4.13.2 - test - - - - - - ./src/test/java - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 1.8 - 1.8 - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - - **/ParserTests.java - - - - - - - \ No newline at end of file diff --git a/java/cparser/src/main/java/cparser/AST.java b/java/cparser/src/main/java/cparser/AST.java deleted file mode 100644 index 1890d894..00000000 --- a/java/cparser/src/main/java/cparser/AST.java +++ /dev/null @@ -1,124 +0,0 @@ -package cparser; - -import java.util.List; - -public class AST { - - public static class Error { - public final Span span; - public final String message; - - public Error(Span span, String message) { - this.span = span; - this.message = message; - } - } - - public static class Span { - public final int startOffset; - public final int endOffset; - - public Span(int startOffset, int endOffset) { - this.startOffset = startOffset; - this.endOffset = endOffset; - } - - public Span() { - this.startOffset = -1; - this.endOffset = -1; - } - - public boolean isValid() { - return endOffset > startOffset; - } - } - - public static class PreprocessorExpression { - public final Span span; - - public PreprocessorExpression(Span span) { - this.span = span; - } - } - - public static class Type { - // The entire type definition, e.g. "byte**" - public final Span span; - // the base type name, e.g. byte in "byte* (&value)[20]" - public final Span baseSpan; - - public Type(Span span, Span baseSpan) { - this.span = span; - this.baseSpan = baseSpan; - } - } - - public static class Identifier { - public final Span span; - - public Identifier(Span span) { - this.span = span; - } - } - - public static class ArgumentList { - public final List arguments; - - public ArgumentList(List arguments) { - this.arguments = arguments; - } - } - - private static class Expr { - public final Span span; - - public Expr(Span span) { - this.span = span; - } - }; - - public static class VariableDeclaration { - public final String type; - public final String identifier; - public final boolean isPointer; - public final boolean isReference; - public final boolean isArray; - public final int arraySize; - public final boolean isFunction; - public final Span span; - - public VariableDeclaration(String type, String identifier, boolean isPointer, boolean isReference, - boolean isArray, int arraySize, boolean isFunction, Span span) { - this.type = type; - this.identifier = identifier; - this.isPointer = isPointer; - this.isReference = isReference; - this.isArray = isArray; - this.arraySize = arraySize; - this.isFunction = isFunction; - this.span = span; - } - } - - public static class FunctionDecl { - public final Identifier name; - public final Type returnValue; - public final ArgumentList args; - - public FunctionDecl(Identifier name, Type returnValue, ArgumentList args) { - this.name = name; - this.returnValue = returnValue; - this.args = args; - } - } - - public static class FunctionCall { - public final Identifier name; - public final ArgumentList args; - - public FunctionCall(Identifier name, ArgumentList args) { - this.name = name; - this.args = args; - } - } -} diff --git a/java/cparser/src/main/java/cparser/Log.java b/java/cparser/src/main/java/cparser/Log.java deleted file mode 100644 index a1abdf33..00000000 --- a/java/cparser/src/main/java/cparser/Log.java +++ /dev/null @@ -1,5 +0,0 @@ -package cparser; - -public interface Log { - public void log(String msg); -} diff --git a/java/cparser/src/main/java/cparser/Parser.java b/java/cparser/src/main/java/cparser/Parser.java deleted file mode 100644 index c733e85b..00000000 --- a/java/cparser/src/main/java/cparser/Parser.java +++ /dev/null @@ -1,358 +0,0 @@ -package cparser; - -import java.util.*; - -import cparser.Tokenizer.Token; -import cparser.Log; -import cparser.AST.VariableDeclaration; - -public class Parser { - private Tokenizer.TokenSet tokenSet; - private Log log; - private Tokenizer.Token[] tokens; - - private List statements = new ArrayList<>(); - private List variableDeclarations = new ArrayList<>(); - private List commentStack = new ArrayList<>(); - private List errors = new ArrayList<>(); - - public Parser(Tokenizer.TokenSet tokenSet, Log log) { - this.tokenSet = tokenSet; - this.statements = new ArrayList<>(); - this.tokens = tokenSet.getTokens(); - } - - void log(String msg) { - if (log != null) { - log.log(msg); - } - } - - int index = 0; - - public List getVariableDeclarations() { - return variableDeclarations; - } - - private static class Result { - public final T value; - public final AST.Span span; - - public Result(T value, AST.Span span) { - this.value = value; - this.span = span; - } - - public Result() { - this.value = null; - this.span = new AST.Span(); - } - - public boolean isValid() { - return span.isValid(); - } - - public static Result none() { - return new Result(); - } - - public Result toGeneric() { - return new Result(value, span); - } - } - - boolean applyResult(Result result) { - if (result.span.isValid()) { - index = result.span.endOffset + 1; - - if (result.value instanceof VariableDeclaration) { - variableDeclarations.add((VariableDeclaration) result.value); - } - - return true; - } - return false; - } - - private void addError(AST.Span span, String message) { - errors.add(new AST.Error(span, message)); - } - - private void addError(Tokenizer.Token token, String message) { - addError(token.getSpan(), message); - } - - public void parse() { - for (index = 0; index < tokens.length;) { - Tokenizer.Token token = tokens[index]; - if (token.type == Tokenizer.TokenType.BLOCK_COMMENT || token.type == Tokenizer.TokenType.COMMENT) { - commentStack.add(token.getSpan()); - index++; - continue; - } else if (token.type == Tokenizer.TokenType.HASH) { - Result result = parsePreprocessorExpression(); - if (result.span.isValid()) { - index = result.span.endOffset + 1; - } else { - addError(token, "Invalid preprocessor expression"); - index++; - } - } else { - Result stmt = parseStmt(); - if (applyResult(stmt)) - continue; - addError(token, "Invalid statement"); - index++; - } - } - } - - private Result parseVarDecl() { - int startIndex = index; - StringBuilder type = new StringBuilder(); - String identifier = null; - boolean isPointer = false; - boolean isReference = false; - boolean isArray = false; - boolean isFunction = false; - int arraySize = -1; - - boolean haveTypeIdentifier = false; - boolean haveIdentifier = false; - - int parenDepth = 0; - int idxStart = 0; - int idxDepth = 0; - - // Parse type - while (index < tokens.length) { - Token token = tokens[index]; - if (token.type == Tokenizer.TokenType.IDENTIFIER || token.type == Tokenizer.TokenType.OTHER) { - // Already have a type identifier, so this is most likely a variable name - if (!haveTypeIdentifier) { - type.append(tokenSet.getTextNoNewlines(token)); - haveTypeIdentifier = true; - } else if (!haveIdentifier) { - identifier = tokenSet.getTextNoNewlines(token); - haveIdentifier = true; - if (parenDepth == 0) - break; - } else { - // Unknown - addError(token, "Unknown token after identifier"); - } - index++; - } else if (token.type == Tokenizer.TokenType.STAR) { - isPointer = true; - type.append("*"); - index++; - } else if (token.type == Tokenizer.TokenType.AMPERSAND) { - isReference = true; - type.append("&"); - index++; - } else if (token.type == Tokenizer.TokenType.L_PAREN) { - parenDepth++; - index++; - } else if (token.type == Tokenizer.TokenType.R_PAREN) { - parenDepth--; - index++; - } else if (token.type == Tokenizer.TokenType.L_IDX) { - idxDepth++; - idxStart = token.ofs + 1; - if (haveIdentifier) { - // Parse function parameters? - } - break; - } else if (token.type == Tokenizer.TokenType.R_IDX) { - idxDepth--; - if (idxDepth == 0) { - String idxVal = tokenSet.getTextNoNewlines(idxStart, (token.ofs - 1) - idxStart); - type.append("[" + idxVal + "]"); - } - break; - } else { - break; - } - } - - AST.Span span = new AST.Span(tokens[startIndex].ofs, tokens[index].getEnd()); - VariableDeclaration varDecl = new VariableDeclaration( - type.toString().trim(), - identifier, - isPointer, - isReference, - isArray, - arraySize, - isFunction, - span); - - return new Result<>(varDecl, span); - } - - private Result parseStmt() { - int startIndex = index; - for (int i = startIndex; i < tokens.length; i++) { - Tokenizer.Token token = tokens[i]; - - Result varDeclResult = parseVarDecl(); - if (varDeclResult.isValid()) - return varDeclResult; - - // if (token.type == Tokenizer.TokenType.L_PAREN && idStack.size() > 0) { - // // Function call? - // } else if (token.type == Tokenizer.TokenType.SEMICOLON) { - // boolean isVarAssign = false; - // for (int j = startIndex; j < i; j++) { - // if (tokens[j].type == Tokenizer.TokenType.EQUALS) { - // isVarAssign = true; - // } - // } - // } else if (token.type == Tokenizer.TokenType.L_BRACE) { - // boolean isVarAssign = false; - // for (int j = startIndex; j < i; j++) { - // if (tokens[j].type == Tokenizer.TokenType.L_PAREN) { - // int endIndex = findClosingParenthesis(j); - // if (endIndex != -1) { - // index = endIndex; - // } - // } - // } - // } - } - - return Result.none(); - // if (index + 1 < tokens.length && tokens[index + 1].type == - // Tokenizer.TokenType.L_PAREN) { - // // Function call or declaration/definition - // if (index > 0 && (tokens[index - 1].type == Tokenizer.TokenType.IDENTIFIER || - // tokens[index - 1].type == Tokenizer.TokenType.OTHER)) { - // // Function declaration or definition - // index = parseFunctionDeclaration(); - // } else { - // // Function call - // index = parseFunctionCall(); - // } - // } else { - // // Variable reference - // index = parseVariableReference(); - // } - } - - private int parseVarDecl(int startIndex, int endIndex) { - if (tokens[startIndex].type == Tokenizer.TokenType.R_PAREN) { - return startIndex; - } - return startIndex + 1; - } - - // Try to parse prep expression - private Result parsePreprocessorExpression() { - int startIndex = index; - int index = startIndex; - if (tokenSet.tokens[index].type == Tokenizer.TokenType.HASH) { - int startLine = tokenSet.getLine(index); - while (index < tokenSet.tokens.length) { - if (tokenSet.getLine(index) > startLine) { - break; - } - index++; - } - // Find first next line token - index--; - } - - AST.Span span = new AST.Span(startIndex, index); - return new Result(new AST.PreprocessorExpression(span), span); - } - - // private ArgumentList parseArgumentList(int startIndex, int endIndex) { - // List arguments = new ArrayList<>(); - // for (int i = startIndex; i < endIndex; i++) { - // if (tokens[i].type == Tokenizer.TokenType.COMMA) { - - // } - // } - // return new ArgumentList(arguments); - // } - - // // Try to parse function declaration and return the ending token index - // private int parseFunctionDeclaration() { - // Tokenizer.Token[] tokens = tokenSet.getTokens(); - // String name = tokenSet.getTextNoNewlines(tokens[index]); - // int endIndex = findClosingParenthesis(index + 1); - - // if (endIndex == -1) - // return index; - - // boolean isDefinition = false; - // if (endIndex + 1 < tokens.length && tokens[endIndex + 1].type == - // Tokenizer.TokenType.L_BRACE) { - // isDefinition = true; - // endIndex = findClosingBrace(endIndex + 1); - // } - - // if (endIndex == -1) - // return index; - - // Function function = new Function(name, tokens[index].ofs, - // tokens[endIndex].ofs + tokens[endIndex].len, - // isDefinition); - // functions.add(function); - // return endIndex - 1; - // } - - // // Try to parse function call and return the ending token index - // private int parseFunctionCall() { - // Tokenizer.Token[] tokens = tokenSet.getTokens(); - // String name = tokenSet.getTextNoNewlines(tokens[index]); - // int endIndex = findClosingParenthesis(index + 1); - // if (endIndex == -1) - // return index; - - // FunctionCall functionCall = new FunctionCall(name, tokens[index].ofs, - // tokens[endIndex].ofs + tokens[endIndex].len); - // functionCalls.add(functionCall); - // return endIndex - 1; - // } - - // // Try to parse variable reference and add it to the list - // private int parseVariableReference() { - // Tokenizer.Token token = tokenSet.getTokens()[index]; - // String name = tokenSet.getTextNoNewlines(token); - // Variable variable = new Variable(name, token.ofs, token.ofs + token.len); - // variables.add(variable); - // return index + 1; - // } - - private int findClosingParenthesis(int startIndex) { - Tokenizer.Token[] tokens = tokenSet.getTokens(); - int parenCount = 1; - for (int i = startIndex + 1; i < tokens.length; i++) { - if (tokens[i].type == Tokenizer.TokenType.L_PAREN) { - parenCount++; - } else if (tokens[i].type == Tokenizer.TokenType.R_PAREN) { - parenCount--; - if (parenCount == 0) { - return i; - } - } - } - return -1; - } - - private int findClosingBrace(int startIndex) { - Tokenizer.Token[] tokens = tokenSet.getTokens(); - int braceCount = 1; - for (int i = startIndex + 1; i < tokens.length; i++) { - if (tokens[i].type == Tokenizer.TokenType.L_BRACE) { - braceCount++; - } else if (tokens[i].type == Tokenizer.TokenType.R_BRACE) { - braceCount--; - if (braceCount == 0) { - return i; - } - } - } - return -1; - } -} diff --git a/java/cparser/src/main/java/cparser/Tokenizer.java b/java/cparser/src/main/java/cparser/Tokenizer.java deleted file mode 100644 index 6df24880..00000000 --- a/java/cparser/src/main/java/cparser/Tokenizer.java +++ /dev/null @@ -1,392 +0,0 @@ -package cparser; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class Tokenizer { - public enum TokenType { - UNDEFINED, - HASH, - L_PAREN, - R_PAREN, - L_BRACE, - R_BRACE, - L_IDX, - R_IDX, - SEMICOLON, - EQUALS, - ARROW, - STAR, - AMPERSAND, - COMMA, - COMMENT, - BLOCK_COMMENT, - IDENTIFIER, - STRING_LITERAL, - NUMERIC_LITERAL, - NUMERIC_LITERAL_HEX, - OTHER, - KEYWORD, - } - - public class Token { - public int ofs; - public int len; - public TokenType type; - - public AST.Span getSpan() { - return new AST.Span(ofs, getEnd()); - } - - public int getEnd() { - return ofs + len; - } - } - - public static class TokenSet { - public final Token[] tokens; - public final String text; - private final TreeMap lineNumberTable; - - TokenSet(Token[] tokens, String text, TreeMap lineNumberTable) { - this.tokens = tokens; - this.text = text; - this.lineNumberTable = lineNumberTable; - } - - public Token[] getTokens() { - return this.tokens; - } - - public int getLine(int offset) { - Map.Entry entry = lineNumberTable.floorEntry(offset); - return entry != null ? entry.getValue() : -1; - } - - public String getText(Token token) { - return getText(token.ofs, token.len); - } - - public String getText(int ofs, int len) { - return text.substring(ofs, ofs + len); // Fixed recursion issue - } - - public String getTextNoNewlines(Token token) { - String text = getText(token); - return text.replace("\n", ""); - } - - - public String getTextNoNewlines(int ofs, int len) { - String text = getText(ofs, len); - return text.replace("\n", ""); - } - }; - - private final String text; - private TreeMap lineNumberTable; - private Log log; - - public Tokenizer(String text) { - this.text = text; - } - - public Tokenizer(String text, Log log) { - this.text = text; - this.log = log; - } - - void log(String msg) { - if (log != null) { - log.log(msg); - } - } - - String getText(Token token) { - return getText(token.ofs, token.len); - } - - String getText(int ofs, int len) { - return text.substring(ofs, ofs + len); // Fixed recursion issue - } - - TokenType lastTokenType = TokenType.UNDEFINED; - - /** - * Inserts a new token into the tokens list. - * - * @param tokens The list of tokens. - * @param tokenStart The starting index of the token. - * @param tokenEnd The current index in the text. - * @param currentType The type of the current token. - */ - private void insertToken(List tokens, int tokenStart, int tokenEnd, TokenType currentType) { - if (currentType != TokenType.UNDEFINED && tokenStart < tokenEnd) { - // Strip whitespace - for (int i = tokenStart; i < tokenEnd; i++) { - if (Character.isWhitespace(text.charAt(i))) { - tokenStart = i + 1; - } else { - break; - } - } - // Strip whitespace from end - for (int i = tokenEnd - 1; i >= tokenStart; i--) { - if (Character.isWhitespace(text.charAt(i))) { - tokenEnd = i; - } else { - break; - } - } - - if (tokenEnd - tokenStart > 0) { - Token token = new Token(); - token.ofs = tokenStart; - token.len = tokenEnd - tokenStart; - token.type = currentType; - if (currentType == TokenType.IDENTIFIER && isKeyword(getText(token))) { - token.type = TokenType.KEYWORD; - } - tokens.add(token); - } - - // Consume the token - currentType = TokenType.UNDEFINED; - } - } - - /** - * Handles the insertion of the last token after parsing is complete. - * - * @param tokens The list of tokens. - * @param tokenStart The starting index of the last token. - * @param currentType The type of the last token. - */ - private void handleLastToken(List tokens, int tokenStart, TokenType currentType) { - insertToken(tokens, tokenStart, text.length(), currentType); - } - - void buildLineNumberTable() { - this.lineNumberTable = new TreeMap<>(); - int lineNumber = 1; - lineNumberTable.put(0, 1); - for (int i = 0; i < text.length(); i++) { - if (text.charAt(i) == '\n') { - lineNumber++; - lineNumberTable.put(i + 1, lineNumber); - } - } - } - - List tokens = new ArrayList<>(); - - // Initialize tokenization state - int tokenStart = 0; - TokenType currentType = TokenType.UNDEFINED; - boolean inComment = false; - boolean inBlockComment = false; - boolean inString = false; - - class ScanRange { - int start; - int end; - TokenType type; - - ScanRange(int start, int end, TokenType type) { - this.start = start; - this.end = end; - this.type = type; - } - - // Invalid constructor - ScanRange() { - this.type = TokenType.UNDEFINED; - } - - boolean isValid() { - return this.type != TokenType.UNDEFINED; - } - }; - - // Add the following method to handle hexadecimal literals - private ScanRange tryParseHexadecimal(int currentIndex) { - if (text.charAt(currentIndex) == '0' && currentIndex + 1 < text.length()) { - char nextChar = text.charAt(currentIndex + 1); - if (nextChar == 'x' || nextChar == 'X') { - int tempIndex = currentIndex + 2; - while (tempIndex < text.length()) { - char c = text.charAt(tempIndex); - if (Character.digit(c, 16) == -1) { - break; - } - tempIndex++; - } - if (tempIndex > currentIndex + 2) { - return new ScanRange(currentIndex, tempIndex, TokenType.NUMERIC_LITERAL_HEX); - } - } - } - return new ScanRange(); - } - - // Identifier that starts with a letter or underscore, and can contain letters, - // digits, and underscores - private ScanRange tryParseIdentifier(int currentIndex) { - if (Character.isLetter(text.charAt(currentIndex)) || text.charAt(currentIndex) == '_') { - int tempIndex = currentIndex + 1; - while (tempIndex < text.length()) { - char c = text.charAt(tempIndex); - if (!(Character.isLetter(c) || Character.isDigit(c) || c == '_')) { - break; - } - tempIndex++; - } - return new ScanRange(currentIndex, tempIndex, TokenType.IDENTIFIER); - } - return new ScanRange(); - } - - private ScanRange tryParseWithLookahead(int currentIndex) { - ScanRange sr = tryParseHexadecimal(currentIndex); - if (!sr.isValid()) { - sr = tryParseIdentifier(currentIndex); - } - return sr; - } - - public boolean isKeyword(String text) { - return text.equals("while") || text.equals("for") || text.equals("if") || text.equals("else") || - text.equals("return") || text.equals("struct") || text.equals("typedef") || - text.equals("enum") || text.equals("union") || text.equals("const") || text.equals("static"); - } - - public TokenSet parse() { - this.buildLineNumberTable(); - - int index = 0; - while (index < text.length()) { - char currentChar = text.charAt(index); - TokenType newType = TokenType.OTHER; - - // Handle comments - if (inBlockComment) { - newType = TokenType.BLOCK_COMMENT; - if (currentChar == '*') { - if (index + 1 < text.length() && text.charAt(index + 1) == '/') { - inBlockComment = false; - index++; - } - } - } else if (inComment) { - newType = TokenType.COMMENT; - if (currentChar == '\n') { - inComment = false; - } - } - // Handle string literals - else if (inString) { - if (currentChar == '"') { - inString = false; - newType = TokenType.STRING_LITERAL; - } else { - newType = TokenType.STRING_LITERAL; - } - } - // Detect start of comments - else if (currentChar == '/' && index + 1 < text.length() && text.charAt(index + 1) == '*') { - inBlockComment = true; - newType = TokenType.BLOCK_COMMENT; - } else if (currentChar == '/' && index + 1 < text.length() && text.charAt(index + 1) == '/') { - inComment = true; - newType = TokenType.COMMENT; - } - // Detect start of string literals - else if (currentChar == '"') { - inString = true; - newType = TokenType.STRING_LITERAL; - } else { - ScanRange range = tryParseWithLookahead(index); - if (range.isValid()) { - // Insert the current token first - // script.println("Inserting current token: " + currentType + ", start: " + - // tokenStart + ", end: " + range.start); - insertToken(tokens, tokenStart, range.start, currentType); - - // Insert a ranged token - // script.println("Inserting ranged token: " + range.type + " start: " + - // range.start + ", end: " + range.end); - insertToken(tokens, range.start, range.end, range.type); - - // New start - currentType = TokenType.UNDEFINED; - tokenStart = range.end; - index = range.end; - } - // Detect numeric literals - else if (Character.isDigit(currentChar)) { - newType = TokenType.NUMERIC_LITERAL; - } - // Detect identifiers - else if (Character.isLetter(currentChar) || currentChar == '_') { - newType = TokenType.IDENTIFIER; - } - // Detect parentheses - else if (currentChar == '(') { - newType = TokenType.L_PAREN; - } else if (currentChar == ')') { - newType = TokenType.R_PAREN; - } - // Detect braces - else if (currentChar == '{') { - newType = TokenType.L_BRACE; - } else if (currentChar == '}') { - newType = TokenType.R_BRACE; - } - // Detect semicolon - else if (currentChar == ';') { - newType = TokenType.SEMICOLON; - } - // Detect comma - else if (currentChar == ',') { - newType = TokenType.COMMA; - } else if (currentChar == '#') { - newType = TokenType.HASH; - } else if (currentChar == '&') { - newType = TokenType.AMPERSAND; - } else if (currentChar == '[') { - newType = TokenType.L_IDX; - } else if (currentChar == ']') { - newType = TokenType.R_IDX; - } else if (currentChar == '=') { - newType = TokenType.EQUALS; - } else if (currentChar == '>' && index > 0 && text.charAt(index - 1) == '-') { - newType = TokenType.ARROW; - currentType = TokenType.ARROW; - tokenStart = index - 1; - } else if (currentChar == '*') { - newType = TokenType.STAR; - } - // Handle other characters - else { - newType = TokenType.OTHER; - } - } - - // Insert a new token if the type changes - if (newType != currentType) { - insertToken(tokens, tokenStart, index, currentType); - tokenStart = index; - currentType = newType; - } - - index++; - } - - // Handle the last token - handleLastToken(tokens, tokenStart, currentType); - - return new TokenSet(tokens.toArray(new Token[0]), text, lineNumberTable); - } -} diff --git a/java/cparser/src/test/java/cparser_tests/ParserTests.java b/java/cparser/src/test/java/cparser_tests/ParserTests.java deleted file mode 100644 index 51766c25..00000000 --- a/java/cparser/src/test/java/cparser_tests/ParserTests.java +++ /dev/null @@ -1,150 +0,0 @@ -package cparser_tests; - -import static org.junit.Assert.*; -import org.junit.Before; -import org.junit.Test; -import java.lang.String; - -import cparser.Parser; -import cparser.Tokenizer; -import cparser.AST; -import cparser.Log; - -import java.util.List; - -public class ParserTests { - private Parser parser; - private Tokenizer.TokenSet tokenSet; - private Log testLog; - - @Before - public void setUp() { - testLog = new Log() { - @Override - public void log(String msg) { - System.out.println(msg); - } - }; - } - - @Test - public void testParseVariableDecl() { - String[] testCases = { - "byte RVar1;", - "tdstLastErrorInfo *pdVar2;", - "undefined4 *puVar2;", - "CHAR pathToUbi_ini[260];", - "undefined& DAT_005a9ed4;", - "char(&s_Identifier_005b6420)[16];", - "void (*fnType)(int j, char, bool);", - "void** (*fnType)(int j, char, bool);" - }; - - for (String code : testCases) { - tokenSet = new Tokenizer(code).parse(); - parser = new Parser(tokenSet, testLog); - parser.parse(); - - List declarations = parser.getVariableDeclarations(); - assertEquals("Failed for case: " + code, 1, declarations.size()); - AST.VariableDeclaration decl = declarations.get(0); - assertNotNull(decl); - System.out.println("Parsed: " + code); - System.out.println(" Type: " + decl.type); - System.out.println(" Identifier: " + decl.identifier); - System.out.println(" Is Pointer: " + decl.isPointer); - System.out.println(" Is Reference: " + decl.isReference); - System.out.println(" Is Array: " + decl.isArray); - System.out.println(" Array Size: " + decl.arraySize); - System.out.println(" Is Function: " + decl.isFunction); - System.out.println(); - } - } - - /* - * @Test - * public void testParseFunctionDeclaration() { - * String code = "void foo(int a, int b);"; - * tokenSet = new Tokenizer(code).parse(); - * parser = new Parser(tokenSet, testLog); - * parser.parse(); - * - * List functions = parser.getFunctions(); - * assertEquals(1, functions.size()); - * assertEquals("foo", functions.get(0).name); - * assertFalse(functions.get(0).isDefinition); - * } - * - * @Test - * public void testParseFunctionDefinition() { - * String code = "int bar(int x) { return x + 1; }"; - * tokenSet = new Tokenizer(code).parse(); - * parser = new Parser(tokenSet, testLog); - * parser.parse(); - * - * List functions = parser.getFunctions(); - * assertEquals(1, functions.size()); - * assertEquals("bar", functions.get(0).name); - * assertTrue(functions.get(0).isDefinition); - * } - * - * @Test - * public void testParseFunctionCall() { - * String code = "result = calculate(5, 10);"; - * tokenSet = new Tokenizer(code).parse(); - * parser = new Parser(tokenSet, testLog); - * parser.parse(); - * - * List functionCalls = parser.getFunctionCalls(); - * assertEquals(1, functionCalls.size()); - * assertEquals("calculate", functionCalls.get(0).name); - * } - * - * @Test - * public void testParsePreprocessorDirective() { - * String code = "#include \nint main() { return 0; }"; - * tokenSet = new Tokenizer(code).parse(); - * parser = new Parser(tokenSet, testLog); - * parser.parse(); - * - * List functions = parser.getFunctions(); - * assertEquals(1, functions.size()); - * assertEquals("main", functions.get(0).name); - * } - * - * @Test - * public void testParseComplexCode() { - * String code = - * "#include \n" + - * "int globalVar = 10;\n" + - * "void helper(int x);\n" + - * "int main() {\n" + - * " int localVar = 5;\n" + - * " helper(localVar);\n" + - * " return 0;\n" + - * "}\n" + - * "void helper(int x) {\n" + - * " printf(\"%d\", x);\n" + - * "}"; - * - * tokenSet = new Tokenizer(code).parse(); - * parser = new Parser(tokenSet, testLog); - * parser.parse(); - * - * List variables = parser.getVariables(); - * List functions = parser.getFunctions(); - * List functionCalls = parser.getFunctionCalls(); - * - * assertEquals(2, variables.size()); - * assertEquals(2, functions.size()); - * assertEquals(2, functionCalls.size()); - * - * assertTrue(variables.stream().anyMatch(v -> v.name.equals("globalVar"))); - * assertTrue(variables.stream().anyMatch(v -> v.name.equals("localVar"))); - * assertTrue(functions.stream().anyMatch(f -> f.name.equals("main"))); - * assertTrue(functions.stream().anyMatch(f -> f.name.equals("helper"))); - * assertTrue(functionCalls.stream().anyMatch(fc -> fc.name.equals("helper"))); - * assertTrue(functionCalls.stream().anyMatch(fc -> fc.name.equals("printf"))); - * } - */ -} diff --git a/java/ghidra/Test.java b/java/ghidra/Test.java new file mode 100644 index 00000000..63137447 --- /dev/null +++ b/java/ghidra/Test.java @@ -0,0 +1,93 @@ +// @category _Reman3 +// @menupath Reman3.Test + +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.data.DataType; +import ghidra.program.model.data.StandAloneDataTypeManager; +import re3lib.RecompileConfig; + +import java.io.File; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +// SQLite imports +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class Test extends GhidraScript { + // Will rebuild all functions + public boolean rebuildAllGlobals = true; + FunctionDatabase functionDB; + + @Override + public void run() throws Exception { + RecompileConfig.INSTANCE = new RecompileConfig(this); + + // Example SQLite usage + testSQLite(); + } + + private void testSQLite() throws Exception { + String dbPath = "jdbc:sqlite:" + RecompileConfig.INSTANCE.outputDir + "/functions.db"; + + try (Connection conn = DriverManager.getConnection(dbPath)) { + println("Connected to SQLite database: " + dbPath); + + // Create a simple table + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE IF NOT EXISTS functions (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "address TEXT NOT NULL, " + + "name TEXT NOT NULL, " + + "file_path TEXT)"); + println("Functions table created/verified"); + } + + // Insert example data + String insertSQL = "INSERT INTO functions (address, name, file_path) VALUES (?, ?, ?)"; + try (PreparedStatement pstmt = conn.prepareStatement(insertSQL)) { + pstmt.setString(1, "0x00401000"); + pstmt.setString(2, "main"); + pstmt.setString(3, "/path/to/main.cxx"); + pstmt.executeUpdate(); + println("Inserted example function"); + } + + // Query data + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT * FROM functions")) { + while (rs.next()) { + println("Function: " + rs.getString("name") + + " at " + rs.getString("address") + + " in " + rs.getString("file_path")); + } + } + + } catch (SQLException e) { + println("SQLite error: " + e.getMessage()); + throw e; + } + } + + private void scanDirectory(File directory, FunctionDatabase.Type type) throws Exception { + File[] files = directory.listFiles((dir, name) -> name.endsWith(".cxx")); + if (files == null) + return; + + for (File file : files) { + scanFile(file, type); + } + } +} \ No newline at end of file diff --git a/java/ghidra/sqlite-jdbc-3.49.1.0.jar b/java/ghidra/sqlite-jdbc-3.49.1.0.jar new file mode 100644 index 00000000..071afa61 Binary files /dev/null and b/java/ghidra/sqlite-jdbc-3.49.1.0.jar differ diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 29105b38..818416ed 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -15,15 +15,24 @@ if(WIN32) add_custom_target(copy_binkw32 ALL DEPENDS ${BINK_DST}) add_dependencies(binkw32 copy_binkw32) - add_library(d3d8 SHARED IMPORTED) - target_include_directories(d3d8 PUBLIC ${CMAKE_CURRENT_LIST_DIR}/mssdk/include) - set_target_properties(d3d8 PROPERTIES - IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/mssdk/lib/d3d8.lib + set(SDK_LIB ${CMAKE_CURRENT_LIST_DIR}/mssdk/lib) + + add_library(d3d8_import SHARED IMPORTED) + set_target_properties(d3d8_import PROPERTIES + IMPORTED_IMPLIB ${SDK_LIB}/d3d8.lib ) - add_library(dinput8 SHARED IMPORTED) - target_include_directories(dinput8 PUBLIC ${CMAKE_CURRENT_LIST_DIR}/mssdk/include) - set_target_properties(dinput8 PROPERTIES - IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/mssdk/lib/dinput8.lib + add_library(dinput8_import SHARED IMPORTED) + set_target_properties(dinput8_import PROPERTIES + IMPORTED_IMPLIB ${SDK_LIB}/dinput8.lib ) + + set(SDK_INCLUDE ${CMAKE_CURRENT_LIST_DIR}/mssdk/include) + add_library(d3d8 INTERFACE) + target_include_directories(d3d8 INTERFACE ${SDK_INCLUDE}) + target_link_libraries(d3d8 INTERFACE d3d8_import) + + add_library(dinput8 INTERFACE) + target_include_directories(dinput8 INTERFACE ${SDK_INCLUDE}) + target_link_libraries(dinput8 INTERFACE dinput8_import) endif() \ No newline at end of file diff --git a/tooling2/third_party/tree-sitter b/tooling2/third_party/tree-sitter new file mode 160000 index 00000000..06537fda --- /dev/null +++ b/tooling2/third_party/tree-sitter @@ -0,0 +1 @@ +Subproject commit 06537fda832d02afed750a4b574fa5aa656ef6ed diff --git a/tooling2/third_party/tree-sitter-cpp b/tooling2/third_party/tree-sitter-cpp new file mode 160000 index 00000000..56455f42 --- /dev/null +++ b/tooling2/third_party/tree-sitter-cpp @@ -0,0 +1 @@ +Subproject commit 56455f4245baf4ea4e0881c5169de69d7edd5ae7