guus
/
reman3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix tool file categorization

This commit is contained in:
Guus Waals 2025-05-29 15:29:07 +08:00
parent 836e281b80
commit db228e64ec
2 changed files with 80 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
tooling/files.sh
View File

@ -3,19 +3,18 @@ set -e
tool=build/clang-x86_64-pc-windows-msvc/Release/r3_gh_tool
cmake --build build/clang-x86_64-pc-windows-msvc/Release --target r3_gh_tool
scan_dir=tmps/gh_auto
file_list=files.txt
rm -f $file_list
for file in tmps/gh_auto/*.cxx; do
echo $file >>$file_list
done
for file in tmps/gh_stub/*.cxx; do
echo $file >>$file_list
done
for file in tmps/gh_fix/*.cxx; do
echo $file >>$file_list
types=(auto ref fix stub)
for type in "${types[@]}"; do
file_list="files_${type}.txt"
rm -f "$file_list"
if [ -d "tmps/gh_${type}" ]; then
for file in "tmps/gh_${type}"/*.cxx; do
echo "$file" >>"$file_list"
done
$tool "@$file_list" -v --type=$type --log-file=log-functions.txt
fi
done
$tool @$file_list -v --log-file=log-functions.txt
$tool tmps/gh_global.h -mglobals -v --log-file=log-globals.txt
$tool -mduplicates -v --log-file=log-duplicates.txt

86
tooling/tool.cpp
View File

@ -18,6 +18,38 @@ extern "C" TSLanguage *tree_sitter_cpp();
// Global address regex pattern
const std::regex ADDRESS_REGEX(R"(//\s*([0-9a-fA-F]{8}))");
// Add enum for file types
enum class FileType { Auto, Fix, Stub, Ref };
// Helper function to convert string to FileType
FileType stringToFileType(const std::string &type_str) {
if (type_str == "auto")
return FileType::Auto;
if (type_str == "fix")
return FileType::Fix;
if (type_str == "stub")
return FileType::Stub;
if (type_str == "ref")
return FileType::Ref;
throw std::invalid_argument("Invalid file type: " + type_str);
}
// Helper function to convert FileType to string
std::string fileTypeToString(FileType type) {
switch (type) {
case FileType::Auto:
return "auto";
case FileType::Fix:
return "fix";
case FileType::Stub:
return "stub";
case FileType::Ref:
return "ref";
default:
throw std::logic_error(fmt::format("Invalid file type: {}", (int)type));
}
}
// Helper function to check if a comment contains an address
bool hasAddressPattern(const std::string &comment) {
return std::regex_search(comment, ADDRESS_REGEX);
@ -47,6 +79,7 @@ struct FunctionInfo {
std::string address;
std::string filepath;
bool is_import;
FileType type; // Add type field
};
struct GlobalInfo {
@ -81,11 +114,11 @@ public:
&delete_imports_stmt,
"Failed to prepare delete imports statement");
prepareStatement("INSERT OR REPLACE INTO Functions (filepath, name, "
"address) VALUES (?, ?, ?)",
"address, type) VALUES (?, ?, ?, ?)",
&insert_functions_stmt,
"Failed to prepare insert functions statement");
prepareStatement("INSERT OR REPLACE INTO Imports (filepath, name, address) "
"VALUES (?, ?, ?)",
prepareStatement("INSERT OR REPLACE INTO Imports (filepath, name, address, "
"type) VALUES (?, ?, ?, ?)",
&insert_imports_stmt,
"Failed to prepare insert imports statement");
prepareStatement("DELETE FROM Globals WHERE filepath = ?",
@ -128,6 +161,7 @@ public:
sqlite3_bind_text(stmt, 1, func.filepath.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, func.name.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 3, func.address.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 4, static_cast<int>(func.type));
sqlite3_step(stmt);
}
@ -157,8 +191,8 @@ public:
}
const char *create_tables = R"(
CREATE TABLE IF NOT EXISTS Functions (filepath TEXT, name TEXT, address TEXT, PRIMARY KEY (name, filepath));
CREATE TABLE IF NOT EXISTS Imports (filepath TEXT, name TEXT, address TEXT, PRIMARY KEY (name, filepath));
CREATE TABLE IF NOT EXISTS Functions (filepath TEXT, name TEXT, address TEXT, type INTEGER DEFAULT 0, PRIMARY KEY (name, filepath));
CREATE TABLE IF NOT EXISTS Imports (filepath TEXT, name TEXT, address TEXT, type INTEGER DEFAULT 0, PRIMARY KEY (name, filepath));
CREATE TABLE IF NOT EXISTS Globals (filepath TEXT, name TEXT, address TEXT);
)";
@ -390,7 +424,7 @@ bool hasFunctionBody(TSNode node) {
}
void findFunctions(TSNode node, const char *source_code, uint32_t source_length,
std::vector<FunctionInfo> &functions) {
std::vector<FunctionInfo> &functions, FileType file_type) {
const char *type = ts_node_type(node);
if (strcmp(type, "function_definition") == 0 ||
@ -409,7 +443,8 @@ void findFunctions(TSNode node, const char *source_code, uint32_t source_length,
FunctionInfo func{func_name, address, "",
strcmp(type, "function_definition") == 0
? !hasFunctionBody(node)
: true};
: true,
file_type}; // Add file_type parameter
functions.push_back(func);
}
// We'll never nest function declarations
@ -422,8 +457,8 @@ void findFunctions(TSNode node, const char *source_code, uint32_t source_length,
uint32_t child_count = ts_node_child_count(node);
for (uint32_t i = 0; i < child_count; i++) {
findFunctions(ts_node_child(node, i), source_code, source_length,
functions);
findFunctions(ts_node_child(node, i), source_code, source_length, functions,
file_type);
}
}
@ -454,7 +489,8 @@ std::vector<std::string> readFileList(const std::string &list_file) {
return files;
}
bool processFile(const std::string &filepath, DatabaseManager &db) {
bool processFile(const std::string &filepath, DatabaseManager &db,
FileType file_type) {
std::ifstream file(filepath);
if (!file.is_open()) {
spdlog::error("Could not open file {}", filepath);
@ -482,17 +518,18 @@ bool processFile(const std::string &filepath, DatabaseManager &db) {
std::vector<FunctionInfo> functions;
findFunctions(root_node, file_content.c_str(), file_content.length(),
functions);
functions, file_type);
for (auto &func : functions) {
func.filepath = filepath;
db.insertFunction(func);
spdlog::debug("{}: {} @ {} in {}", func.is_import ? "Import" : "Function",
func.name, func.address, filepath);
spdlog::debug("{}: {} @ {} in {} (type: {})",
func.is_import ? "Import" : "Function", func.name,
func.address, filepath, fileTypeToString(func.type));
}
spdlog::info("Processed {} functions/imports from {}", functions.size(),
filepath);
spdlog::info("Processed {} functions/imports from {} (type: {})",
functions.size(), filepath, fileTypeToString(file_type));
ts_tree_delete(tree);
ts_parser_delete(parser);
@ -741,6 +778,7 @@ int main(int argc, char *argv[]) {
std::string db_path = "gh.db";
std::string mode = "functions";
std::string log_file = "";
std::string type_str = "auto"; // Add type string variable
bool verbose = false;
app.add_option("files", input_files,
@ -755,11 +793,24 @@ int main(int argc, char *argv[]) {
->default_val("functions")
->check(
CLI::IsMember({"functions", "globals", "duplicates", "dump-tree"}));
app.add_option("-t,--type", type_str,
"File type: 'auto', 'fix', 'stub', or 'ref'")
->default_val("auto")
->check(CLI::IsMember({"auto", "fix", "stub", "ref"}));
app.add_flag("-v,--verbose", verbose, "Enable verbose logging (debug level)");
app.add_flag("--log-file", log_file, "Enable logging to file");
CLI11_PARSE(app, argc, argv);
// Convert string to FileType enum
FileType file_type;
try {
file_type = stringToFileType(type_str);
} catch (const std::invalid_argument &e) {
spdlog::error("Invalid file type: {}", type_str);
return 1;
}
// Set log level based on verbose flag
if (verbose) {
spdlog::set_level(spdlog::level::debug);
@ -830,10 +881,10 @@ int main(int argc, char *argv[]) {
db.beginTransaction();
for (const auto &filepath : files_to_process) {
spdlog::info("=== Processing: {} ===", filepath);
spdlog::info("=== Processing: {} (type: {}) ===", filepath, type_str);
bool success = false;
if (mode == "functions") {
success = processFile(filepath, db);
success = processFile(filepath, db, file_type);
} else if (mode == "globals") {
success = processGlobalsFile(filepath, db);
}
@ -860,6 +911,7 @@ int main(int argc, char *argv[]) {
spdlog::info("=== Summary ===");
spdlog::info("Processed {} files successfully", processed_count);
spdlog::info("Mode: {}", mode);
spdlog::info("File type: {}", type_str);
if (mode != "dump-tree") {
spdlog::info("Database saved to: {}", db_path);
}