From cc7964f03ebf29f27dd0da1ebd66df37d70c1066 Mon Sep 17 00:00:00 2001 From: Guus Waals <_@guusw.nl> Date: Tue, 27 May 2025 00:12:40 +0800 Subject: [PATCH] Updated --- tooling2/files.sh | 2 +- tooling2/tool.cpp | 140 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 114 insertions(+), 28 deletions(-) diff --git a/tooling2/files.sh b/tooling2/files.sh index a3096f16..7d33192e 100644 --- a/tooling2/files.sh +++ b/tooling2/files.sh @@ -13,7 +13,7 @@ for file in tmps/gh_stub/*.cxx do echo $file >> $file_list done -for file in tmps/gh_fix/*.h +for file in tmps/gh_fix/*.cxx do echo $file >> $file_list done diff --git a/tooling2/tool.cpp b/tooling2/tool.cpp index 9dadc16a..d999c2c5 100644 --- a/tooling2/tool.cpp +++ b/tooling2/tool.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include extern "C" TSLanguage *tree_sitter_cpp(); @@ -18,9 +19,81 @@ struct FunctionInfo { bool is_import; // true for extern declarations, false for definitions }; +class PreparedStatements { +private: + sqlite3* db; + sqlite3_stmt* delete_functions_stmt; + sqlite3_stmt* delete_imports_stmt; + sqlite3_stmt* insert_functions_stmt; + sqlite3_stmt* insert_imports_stmt; + +public: + PreparedStatements(sqlite3* database) : db(database) { + // Prepare delete statements + const char* delete_functions_sql = "DELETE FROM Functions WHERE filepath = ?"; + const char* delete_imports_sql = "DELETE FROM Imports WHERE filepath = ?"; + + // Prepare insert statements + const char* insert_functions_sql = "INSERT OR REPLACE INTO Functions (filepath, name, address) VALUES (?, ?, ?)"; + const char* insert_imports_sql = "INSERT OR REPLACE INTO Imports (filepath, name, address) VALUES (?, ?, ?)"; + + // Prepare all statements + int rc; + rc = sqlite3_prepare_v2(db, delete_functions_sql, -1, &delete_functions_stmt, nullptr); + if (rc != SQLITE_OK) { + throw std::runtime_error("Failed to prepare delete functions statement: " + std::string(sqlite3_errmsg(db))); + } + + rc = sqlite3_prepare_v2(db, delete_imports_sql, -1, &delete_imports_stmt, nullptr); + if (rc != SQLITE_OK) { + throw std::runtime_error("Failed to prepare delete imports statement: " + std::string(sqlite3_errmsg(db))); + } + + rc = sqlite3_prepare_v2(db, insert_functions_sql, -1, &insert_functions_stmt, nullptr); + if (rc != SQLITE_OK) { + throw std::runtime_error("Failed to prepare insert functions statement: " + std::string(sqlite3_errmsg(db))); + } + + rc = sqlite3_prepare_v2(db, insert_imports_sql, -1, &insert_imports_stmt, nullptr); + if (rc != SQLITE_OK) { + throw std::runtime_error("Failed to prepare insert imports statement: " + std::string(sqlite3_errmsg(db))); + } + } + + ~PreparedStatements() { + if (delete_functions_stmt) sqlite3_finalize(delete_functions_stmt); + if (delete_imports_stmt) sqlite3_finalize(delete_imports_stmt); + if (insert_functions_stmt) sqlite3_finalize(insert_functions_stmt); + if (insert_imports_stmt) sqlite3_finalize(insert_imports_stmt); + } + + void clearEntriesForFile(const std::string& filepath) { + // Clear functions + sqlite3_reset(delete_functions_stmt); + sqlite3_bind_text(delete_functions_stmt, 1, filepath.c_str(), -1, SQLITE_STATIC); + sqlite3_step(delete_functions_stmt); + + // Clear imports + sqlite3_reset(delete_imports_stmt); + sqlite3_bind_text(delete_imports_stmt, 1, filepath.c_str(), -1, SQLITE_STATIC); + sqlite3_step(delete_imports_stmt); + } + + void insertFunction(const FunctionInfo& func) { + sqlite3_stmt* stmt = func.is_import ? insert_imports_stmt : insert_functions_stmt; + + sqlite3_reset(stmt); + sqlite3_bind_text(stmt, 1, func.filepath.c_str(), -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, func.name.c_str(), -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, func.address.c_str(), -1, SQLITE_STATIC); + sqlite3_step(stmt); + } +}; + class DatabaseManager { private: sqlite3* db; + std::unique_ptr prepared_stmts; public: DatabaseManager(const std::string& db_path) : db(nullptr) { @@ -53,45 +126,36 @@ public: sqlite3_exec(db, create_functions_table, nullptr, nullptr, nullptr); sqlite3_exec(db, create_imports_table, nullptr, nullptr, nullptr); + + // Initialize prepared statements + prepared_stmts = std::make_unique(db); } ~DatabaseManager() { + // prepared_stmts will be destroyed automatically before db is closed if (db) { sqlite3_close(db); } } void clearEntriesForFile(const std::string& filepath) { - const char* delete_functions = "DELETE FROM Functions WHERE filepath = ?"; - const char* delete_imports = "DELETE FROM Imports WHERE filepath = ?"; - - sqlite3_stmt* stmt; - - // Clear functions - sqlite3_prepare_v2(db, delete_functions, -1, &stmt, nullptr); - sqlite3_bind_text(stmt, 1, filepath.c_str(), -1, SQLITE_STATIC); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - // Clear imports - sqlite3_prepare_v2(db, delete_imports, -1, &stmt, nullptr); - sqlite3_bind_text(stmt, 1, filepath.c_str(), -1, SQLITE_STATIC); - sqlite3_step(stmt); - sqlite3_finalize(stmt); + prepared_stmts->clearEntriesForFile(filepath); } void insertFunction(const FunctionInfo& func) { - const char* table = func.is_import ? "Imports" : "Functions"; - std::string sql = "INSERT OR REPLACE INTO " + std::string(table) + - " (filepath, name, address) VALUES (?, ?, ?)"; - - sqlite3_stmt* stmt; - sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, nullptr); - sqlite3_bind_text(stmt, 1, func.filepath.c_str(), -1, SQLITE_STATIC); - sqlite3_bind_text(stmt, 2, func.name.c_str(), -1, SQLITE_STATIC); - sqlite3_bind_text(stmt, 3, func.address.c_str(), -1, SQLITE_STATIC); - sqlite3_step(stmt); - sqlite3_finalize(stmt); + prepared_stmts->insertFunction(func); + } + + void beginTransaction() { + sqlite3_exec(db, "BEGIN TRANSACTION", nullptr, nullptr, nullptr); + } + + void commitTransaction() { + sqlite3_exec(db, "COMMIT", nullptr, nullptr, nullptr); + } + + void rollbackTransaction() { + sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr); } }; @@ -431,11 +495,33 @@ int main(int argc, char* argv[]) { int processed_count = 0; int total_functions = 0; + // Use transactions for better performance when processing multiple files + const size_t batch_size = 50; // Process files in batches for optimal performance + size_t current_batch = 0; + + db.beginTransaction(); + for (const auto& filepath : files_to_process) { std::cout << "\n=== Processing: " << filepath << " ===" << std::endl; if (processFile(filepath, db)) { processed_count++; } + + current_batch++; + + // Commit transaction every batch_size files to avoid long-running transactions + if (current_batch >= batch_size) { + db.commitTransaction(); + std::cout << "Committed batch of " << current_batch << " files to database" << std::endl; + db.beginTransaction(); + current_batch = 0; + } + } + + // Commit any remaining files in the final batch + if (current_batch > 0) { + db.commitTransaction(); + std::cout << "Committed final batch of " << current_batch << " files to database" << std::endl; } std::cout << "\n=== Summary ===" << std::endl;