fix(arch-scan): 修复多语言扫描器并添加 DELETE 清空表

- scanner.ts: 添加 DELETE 清空旧数据语句,避免重复插入
- ts-scanner: 改用 regex 替代 ts-morph 解析(避免解析失败)
- go-scanner/py-scanner/proto-scanner: 完善实现
This commit is contained in:
SpecialX
2026-07-08 12:49:43 +08:00
parent f554011af5
commit 4629de1926
5 changed files with 560 additions and 48 deletions

View File

@@ -1,17 +1,22 @@
#!/usr/bin/env tsx
import { createDb } from './schema.js';
import { scanTypeScript } from './scanners/ts-scanner.js';
import { scanGo } from './scanners/go-scanner.js';
import { scanPython } from './scanners/py-scanner.js';
import { scanProtobuf } from './scanners/proto-scanner.js';
import path from 'node:path';
import { createDb } from "./schema.js";
import { scanTypeScript } from "./scanners/ts-scanner.js";
import { scanGo } from "./scanners/go-scanner.js";
import { scanPython } from "./scanners/py-scanner.js";
import { scanProtobuf } from "./scanners/proto-scanner.js";
import path from "node:path";
const ROOT = process.cwd();
function main(): void {
const db = createDb(path.join(ROOT, 'arch.db'));
const db = createDb(path.join(ROOT, "arch.db"));
console.log('🔍 arch-scan: 开始多语言扫描...');
// 清空旧数据(避免重跑产生重复符号)
db.exec(
"DELETE FROM calls; DELETE FROM dependencies; DELETE FROM symbols; DELETE FROM contracts; DELETE FROM events; DELETE FROM modules;",
);
console.log("🔍 arch-scan: 开始多语言扫描...");
// 串行扫描(并行会导致 FOREIGN KEY 错误)
const tsStats = scanTypeScript(db, ROOT);
@@ -26,8 +31,8 @@ function main(): void {
const protoStats = scanProtobuf(db, ROOT);
console.log(` Protobuf: ${protoStats.contracts} 契约`);
console.log('✅ arch-scan 完成');
console.log("✅ arch-scan 完成");
db.close();
}
main();
main();

View File

@@ -1,12 +1,133 @@
import type { Database as DBType } from 'better-sqlite3';
import path from "node:path";
import fs from "node:fs";
import type { Database as DBType } from "better-sqlite3";
interface ScanStats {
modules: number;
symbols: number;
}
/** 递归收集目录下匹配扩展名的所有文件 */
function walkDir(dir: string, exts: string[]): string[] {
if (!fs.existsSync(dir)) return [];
const results: string[] = [];
const stack: string[] = [dir];
while (stack.length > 0) {
const current = stack.pop()!;
let entries: fs.Dirent[];
try {
entries = fs.readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const entry of entries) {
const fullPath = path.join(current, entry.name);
if (entry.isDirectory()) {
if (["node_modules", "dist", ".git", "vendor"].includes(entry.name))
continue;
stack.push(fullPath);
} else if (entry.isFile()) {
const ext = path.extname(entry.name);
if (exts.includes(ext)) results.push(fullPath);
}
}
}
return results;
}
// Go 符号正则:匹配导出函数、类型、结构体、接口
const GO_FUNC_RE = /^func\s+(?:\([^)]*\)\s+)?([A-Z]\w*)\s*\(/gm;
const GO_TYPE_RE = /^type\s+([A-Z]\w*)\s+/gm;
const GO_STRUCT_RE = /^type\s+([A-Z]\w*)\s+struct\s*\{/gm;
const GO_INTERFACE_RE = /^type\s+([A-Z]\w*)\s+interface\s*\{/gm;
export function scanGo(db: DBType, root: string): ScanStats {
// Go 扫描器骨架P1 后期用 tree-sitter-go 实现
// 当前仅扫描 go.mod 识别模块
return { modules: 0, symbols: 0 };
}
const insertModule = db.prepare(
"INSERT OR IGNORE INTO modules (name, path, language, service, type) VALUES (?, ?, ?, ?, ?)",
);
const insertSymbol = db.prepare(
"INSERT INTO symbols (module_id, name, kind, language, file_path, line_start, line_end, is_exported) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
);
const getModuleId = db.prepare("SELECT id FROM modules WHERE name = ?") as {
get: (name: string) => { id: number } | undefined;
};
let modules = 0;
let symbols = 0;
// 扫描 services/* 下有 go.mod 的服务
const servicesDir = path.join(root, "services");
if (!fs.existsSync(servicesDir)) return { modules, symbols };
const entries = fs.readdirSync(servicesDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const serviceName = entry.name;
const servicePath = path.join(servicesDir, serviceName);
const hasGoMod = fs.existsSync(path.join(servicePath, "go.mod"));
if (!hasGoMod) continue;
insertModule.run(serviceName, servicePath, "go", serviceName, "service");
const modRow = getModuleId.get(serviceName);
if (modRow) {
modules++;
// 扫描 .go 文件
const goFiles = walkDir(servicePath, [".go"]);
for (const filePath of goFiles) {
// 跳过测试文件
if (filePath.endsWith("_test.go")) continue;
let content: string;
try {
content = fs.readFileSync(filePath, "utf-8");
} catch {
continue;
}
const lines = content.split("\n");
// 函数
let match: RegExpExecArray | null;
const funcRe = new RegExp(GO_FUNC_RE);
while ((match = funcRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
insertSymbol.run(
modRow.id,
name,
"function",
"go",
filePath,
lineNum,
lineNum,
1,
);
symbols++;
}
// 类型struct / interface / type alias
const typeRe = new RegExp(GO_TYPE_RE);
while ((match = typeRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
// 判断是 struct 还是 interface 还是普通 type
const line = lines[lineNum - 1] || "";
let kind = "type";
if (line.includes("struct")) kind = "struct";
else if (line.includes("interface")) kind = "interface";
insertSymbol.run(
modRow.id,
name,
kind,
"go",
filePath,
lineNum,
lineNum,
1,
);
symbols++;
}
}
}
}
return { modules, symbols };
}

View File

@@ -1,10 +1,92 @@
import type { Database as DBType } from 'better-sqlite3';
import path from "node:path";
import fs from "node:fs";
import type { Database as DBType } from "better-sqlite3";
interface ScanStats {
contracts: number;
}
// Proto 符号正则
const PROTO_SERVICE_RE = /^service\s+(\w+)\s*\{/gm;
const PROTO_MESSAGE_RE = /^message\s+(\w+)\s*\{/gm;
const PROTO_RPC_RE = /^\s*rpc\s+(\w+)\s*\(/gm;
export function scanProtobuf(db: DBType, root: string): ScanStats {
// Protobuf 扫描器骨架:扫描 packages/shared-proto/proto/*.proto
return { contracts: 0 };
}
const insertContract = db.prepare(
"INSERT OR IGNORE INTO contracts (name, type, file_path, service, content) VALUES (?, ?, ?, ?, ?)",
);
let contracts = 0;
// 扫描 packages/shared-proto/proto/*.proto
const protoDir = path.join(root, "packages", "shared-proto", "proto");
if (!fs.existsSync(protoDir)) return { contracts };
const entries = fs.readdirSync(protoDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith(".proto")) continue;
const filePath = path.join(protoDir, entry.name);
const protoName = path.basename(entry.name, ".proto");
const serviceName = protoName.replace("_", "-");
let content: string;
try {
content = fs.readFileSync(filePath, "utf-8");
} catch {
continue;
}
// 提取 package 名
const pkgMatch = content.match(/^package\s+(\S+);/m);
const pkgName = pkgMatch ? pkgMatch[1] : "";
// 记录 proto 文件作为契约
insertContract.run(protoName, "proto-file", filePath, serviceName, pkgName);
contracts++;
// 提取 service 定义
let match: RegExpExecArray | null;
const serviceRe = new RegExp(PROTO_SERVICE_RE);
while ((match = serviceRe.exec(content)) !== null) {
const name = match[1];
insertContract.run(
`${protoName}.${name}`,
"service",
filePath,
serviceName,
name,
);
contracts++;
}
// 提取 message 定义
const messageRe = new RegExp(PROTO_MESSAGE_RE);
while ((match = messageRe.exec(content)) !== null) {
const name = match[1];
insertContract.run(
`${protoName}.${name}`,
"message",
filePath,
serviceName,
name,
);
contracts++;
}
// 提取 rpc 方法
const rpcRe = new RegExp(PROTO_RPC_RE);
while ((match = rpcRe.exec(content)) !== null) {
const name = match[1];
insertContract.run(
`${protoName}.rpc.${name}`,
"rpc",
filePath,
serviceName,
name,
);
contracts++;
}
}
return { contracts };
}

View File

@@ -1,11 +1,142 @@
import type { Database as DBType } from 'better-sqlite3';
import path from "node:path";
import fs from "node:fs";
import type { Database as DBType } from "better-sqlite3";
interface ScanStats {
modules: number;
symbols: number;
}
/** 递归收集目录下匹配扩展名的所有文件 */
function walkDir(dir: string, exts: string[]): string[] {
if (!fs.existsSync(dir)) return [];
const results: string[] = [];
const stack: string[] = [dir];
while (stack.length > 0) {
const current = stack.pop()!;
let entries: fs.Dirent[];
try {
entries = fs.readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const entry of entries) {
const fullPath = path.join(current, entry.name);
if (entry.isDirectory()) {
if (
[
"node_modules",
"dist",
".git",
"__pycache__",
".venv",
"venv",
].includes(entry.name)
)
continue;
stack.push(fullPath);
} else if (entry.isFile()) {
const ext = path.extname(entry.name);
if (exts.includes(ext)) results.push(fullPath);
}
}
}
return results;
}
// Python 符号正则
const PY_FUNC_RE = /^(?:async\s+)?def\s+(\w+)\s*\(/gm;
const PY_CLASS_RE = /^class\s+(\w+)\s*[\(:]/gm;
export function scanPython(db: DBType, root: string): ScanStats {
// Python 扫描器骨架P1 后期用 tree-sitter-python 实现
return { modules: 0, symbols: 0 };
}
const insertModule = db.prepare(
"INSERT OR IGNORE INTO modules (name, path, language, service, type) VALUES (?, ?, ?, ?, ?)",
);
const insertSymbol = db.prepare(
"INSERT INTO symbols (module_id, name, kind, language, file_path, line_start, line_end, is_exported) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
);
const getModuleId = db.prepare("SELECT id FROM modules WHERE name = ?") as {
get: (name: string) => { id: number } | undefined;
};
let modules = 0;
let symbols = 0;
// 扫描 services/* 下有 pyproject.toml 的服务
const servicesDir = path.join(root, "services");
if (!fs.existsSync(servicesDir)) return { modules, symbols };
const entries = fs.readdirSync(servicesDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const serviceName = entry.name;
const servicePath = path.join(servicesDir, serviceName);
const hasPyproject = fs.existsSync(
path.join(servicePath, "pyproject.toml"),
);
if (!hasPyproject) continue;
insertModule.run(
serviceName,
servicePath,
"python",
serviceName,
"service",
);
const modRow = getModuleId.get(serviceName);
if (modRow) {
modules++;
// 扫描 .py 文件
const pyFiles = walkDir(servicePath, [".py"]);
for (const filePath of pyFiles) {
let content: string;
try {
content = fs.readFileSync(filePath, "utf-8");
} catch {
continue;
}
// 函数(含 async def
let match: RegExpExecArray | null;
const funcRe = new RegExp(PY_FUNC_RE);
while ((match = funcRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
// Python 中以 _ 开头的为私有
const isExported = !name.startsWith("_") ? 1 : 0;
insertSymbol.run(
modRow.id,
name,
"function",
"python",
filePath,
lineNum,
lineNum,
isExported,
);
symbols++;
}
// 类
const classRe = new RegExp(PY_CLASS_RE);
while ((match = classRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
insertSymbol.run(
modRow.id,
name,
"class",
"python",
filePath,
lineNum,
lineNum,
1,
);
symbols++;
}
}
}
}
return { modules, symbols };
}

View File

@@ -1,38 +1,211 @@
import { Project, SyntaxKind } from 'ts-morph';
import path from 'node:path';
import type { Database as DBType } from 'better-sqlite3';
import path from "node:path";
import fs from "node:fs";
import type { Database as DBType } from "better-sqlite3";
interface ScanStats {
modules: number;
symbols: number;
}
export function scanTypeScript(db: DBType, root: string): ScanStats {
const project = new Project({
tsConfigFilePath: undefined,
skipAddingFilesFromTsConfig: true,
compilerOptions: {
allowJs: true,
declaration: false,
resolveJsonModule: true,
},
});
/** 递归收集目录下匹配扩展名的所有文件 */
function walkDir(dir: string, exts: string[]): string[] {
if (!fs.existsSync(dir)) return [];
const results: string[] = [];
const stack: string[] = [dir];
while (stack.length > 0) {
const current = stack.pop()!;
let entries: fs.Dirent[];
try {
entries = fs.readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const entry of entries) {
const fullPath = path.join(current, entry.name);
if (entry.isDirectory()) {
// 跳过 node_modules / dist / .git
if (
["node_modules", "dist", ".git", ".next", "build"].includes(
entry.name,
)
)
continue;
stack.push(fullPath);
} else if (entry.isFile()) {
const ext = path.extname(entry.name);
if (exts.includes(ext)) results.push(fullPath);
}
}
}
return results;
}
export function scanTypeScript(db: DBType, root: string): ScanStats {
const insertModule = db.prepare(
"INSERT OR IGNORE INTO modules (name, path, language, service, type) VALUES (?, ?, ?, ?, ?)",
);
const insertSymbol = db.prepare(
"INSERT INTO symbols (module_id, name, kind, language, file_path, line_start, line_end, is_exported) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
);
const getModuleId = db.prepare("SELECT id FROM modules WHERE name = ?") as {
get: (name: string) => { id: number } | undefined;
};
const patterns = ['services/*/src/**/*.ts', 'apps/*/src/**/*.ts', 'packages/*/src/**/*.ts'];
let modules = 0;
let symbols = 0;
const insertModule = db.prepare(
'INSERT OR IGNORE INTO modules (name, path, language, service, type) VALUES (?, ?, ?, ?, ?)'
);
const insertSymbol = db.prepare(
'INSERT INTO symbols (module_id, name, kind, language, file_path, line_start, line_end, is_exported) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
);
// 扫描 services/* 和 apps/* 和 packages/* 下的 TS 服务
const serviceDirs = ["services", "apps"];
const moduleList: { name: string; path: string; service: string }[] = [];
// 简化实现:扫描 services/*/src 目录作为模块
const servicesDir = path.join(root, 'services');
// 实际实现用 fast-glob 模式匹配
// 此处为骨架P1 后期补全
for (const dir of serviceDirs) {
const baseDir = path.join(root, dir);
if (!fs.existsSync(baseDir)) continue;
const entries = fs.readdirSync(baseDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const serviceName = entry.name;
const servicePath = path.join(baseDir, serviceName);
// 确认是 TS 服务(有 src 目录或 package.json
const hasSrc = fs.existsSync(path.join(servicePath, "src"));
const hasPkg = fs.existsSync(path.join(servicePath, "package.json"));
if (!hasSrc && !hasPkg) continue;
const moduleName = serviceName;
insertModule.run(moduleName, servicePath, "ts", serviceName, "service");
const row = getModuleId.get(moduleName);
if (row) {
modules++;
moduleList.push({
name: moduleName,
path: servicePath,
service: serviceName,
});
}
}
}
// packages/* 下的 TS 包
const packagesDir = path.join(root, "packages");
if (fs.existsSync(packagesDir)) {
const entries = fs.readdirSync(packagesDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const pkgName = entry.name;
const pkgPath = path.join(packagesDir, pkgName);
const hasSrc = fs.existsSync(path.join(pkgPath, "src"));
const hasPkg = fs.existsSync(path.join(pkgPath, "package.json"));
const hasProto = fs.existsSync(path.join(pkgPath, "proto"));
if (!hasSrc && !hasPkg && !hasProto) continue;
const moduleName = pkgName;
insertModule.run(moduleName, pkgPath, "ts", pkgName, "package");
const row = getModuleId.get(moduleName);
if (row) {
modules++;
moduleList.push({ name: moduleName, path: pkgPath, service: pkgName });
}
}
}
// 收集所有 .ts 文件并用 regex 提取符号ts-morph 对未安装依赖的文件可能解析失败)
const TS_FUNC_RE = /(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(/g;
const TS_CLASS_RE = /(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/g;
const TS_INTERFACE_RE = /(?:export\s+)?interface\s+(\w+)/g;
const TS_CONST_RE = /(?:export\s+)?const\s+([A-Z][A-Z0-9_]+)\s*=/g;
for (const mod of moduleList) {
const srcDir = path.join(mod.path, "src");
if (!fs.existsSync(srcDir)) continue;
const tsFiles = walkDir(srcDir, [".ts", ".tsx"]);
const modRow = getModuleId.get(mod.name);
if (!modRow) continue;
for (const filePath of tsFiles) {
let content: string;
try {
content = fs.readFileSync(filePath, "utf-8");
} catch {
continue;
}
// 函数
let match: RegExpExecArray | null;
const funcRe = new RegExp(TS_FUNC_RE);
while ((match = funcRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
const isExported = match[0].includes("export") ? 1 : 0;
insertSymbol.run(
modRow.id,
name,
"function",
"ts",
filePath,
lineNum,
lineNum,
isExported,
);
symbols++;
}
// 类
const classRe = new RegExp(TS_CLASS_RE);
while ((match = classRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
const isExported = match[0].includes("export") ? 1 : 0;
insertSymbol.run(
modRow.id,
name,
"class",
"ts",
filePath,
lineNum,
lineNum,
isExported,
);
symbols++;
}
// 接口
const ifaceRe = new RegExp(TS_INTERFACE_RE);
while ((match = ifaceRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
const isExported = match[0].includes("export") ? 1 : 0;
insertSymbol.run(
modRow.id,
name,
"interface",
"ts",
filePath,
lineNum,
lineNum,
isExported,
);
symbols++;
}
// 常量UPPER_CASE
const constRe = new RegExp(TS_CONST_RE);
while ((match = constRe.exec(content)) !== null) {
const name = match[1];
const lineNum = content.slice(0, match.index).split("\n").length;
const isExported = match[0].includes("export") ? 1 : 0;
insertSymbol.run(
modRow.id,
name,
"const",
"ts",
filePath,
lineNum,
lineNum,
isExported,
);
symbols++;
}
}
}
return { modules, symbols };
}
}