diff --git a/packages/rpa-core/package.json b/packages/rpa-core/package.json index 732e68d..1b23a74 100644 --- a/packages/rpa-core/package.json +++ b/packages/rpa-core/package.json @@ -15,7 +15,9 @@ "check:deps": "node scripts/check-deps.mjs" }, "dependencies": { + "@cantoo/pdf-lib": "^2.11.1", "@gutenye/ocr-node": "^1.4.9", + "@napi-rs/canvas": "^1.1.0", "adm-zip": "^0.5.16", "basic-ftp": "^6.2.3", "exceljs": "^4.4.0", @@ -25,6 +27,7 @@ "mysql2": "^3.24.6", "nodemailer": "^6.9.16", "pdf-lib": "^1.17.1", + "pdfjs-dist": "^4.10.38", "pg": "^8.23.1", "pngjs": "^7.0.0", "sm-crypto": "^0.5.8", diff --git a/packages/rpa-core/scripts/check-deps.mjs b/packages/rpa-core/scripts/check-deps.mjs index 3f3ccdc..4ad2cc9 100644 --- a/packages/rpa-core/scripts/check-deps.mjs +++ b/packages/rpa-core/scripts/check-deps.mjs @@ -18,6 +18,10 @@ const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") const srcRoot = path.join(pkgRoot, "src"); const IMPORT_RE = /(?:import|export)\s+(?:[\s\S]*?from\s*)?["']([^"']+)["']/g; +// 动态形式:import("x") / require("x") / createRequire 后的 require 调用。 +// 旧正则要求 import 后跟空白,`import("electron")` 完全不匹配(实测 0 命中, +// vision/cv-vision 曾靠这个盲区逃过检查——迁移阶段1-A9 修复)。 +const DYNAMIC_IMPORT_RE = /(?:\bimport\s*\(|\brequire\s*\()\s*["']([^"']+)["']/g; async function walk(dir) { const entries = await readdir(dir, { withFileTypes: true }); @@ -53,6 +57,17 @@ for (const file of await walk(srcRoot)) { } } } + for (const match of source.matchAll(DYNAMIC_IMPORT_RE)) { + const spec = match[1]; + const rel = path.relative(pkgRoot, file).replaceAll("\\", "/"); + if (spec === "electron" || spec.startsWith("electron/")) { + offenders.push(`${rel}: 禁止动态 import("${spec}")(宿主能力走 HostBridge 注入)`); + continue; + } + if (spec.includes("src/main")) { + offenders.push(`${rel}: 禁止动态引用宿主模块 "${spec}"`); + } + } } if (offenders.length) { diff --git a/packages/rpa-core/src/capabilities/cv-vision.test.ts b/packages/rpa-core/src/capabilities/cv-vision.test.ts index b765d6c..1b35bcc 100644 --- a/packages/rpa-core/src/capabilities/cv-vision.test.ts +++ b/packages/rpa-core/src/capabilities/cv-vision.test.ts @@ -10,30 +10,50 @@ const detectMock = vi.hoisted(() => vi.fn(async () => ({ { text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] } ] }))); -// desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面 +// 屏幕服务模拟: 默认无画面(走 GDI 回退), 多屏用例按显示器 id/请求宽度提供对应画面 const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> })); vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock })); vi.mock("@gutenye/ocr-node", () => ({ default: { create: vi.fn(async () => ({ detect: detectMock })) } })); -vi.mock("electron", () => ({ - desktopCapturer: { - getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) => - electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width)) - }, - screen: { - getAllDisplays: vi.fn(() => [ - { id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 }, - { id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 } - ]), - dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y })) - } -})); import { getHandler } from "../engine/registry"; +import { setHostBridge, type HostDisplay } from "../engine/host-bridge"; import "./cv-vision"; +const displayOf = (id: string, x: number, width: number, height: number, scaleFactor: number): HostDisplay => ({ + id, + bounds: { x, y: 0, width, height }, + scaleFactor +}); +const displays: HostDisplay[] = [ + displayOf("1", 0, 1920, 1080, 1), + displayOf("2", 1920, 1280, 720, 2) +]; + +setHostBridge({ + createFormWindow: () => { throw new Error("not used in cv-vision tests"); }, + dipToScreenPoint: (point) => ({ x: point.x, y: point.y }), + screenToDipPoint: (point) => ({ x: point.x, y: point.y }), + loadAiConfig: () => ({ baseUrl: "", apiKey: "", model: "", visionModel: "" }), + screen: { + listDisplays: () => displays, + primaryDisplay: () => displays[0], + dipToScreenPoint: (point) => ({ x: point.x, y: point.y }), + displayMatching: () => displays[0], + captureDisplay: async (displayId: string, wantWidth: number, wantHeight: number) => { + // 模拟真实 captureDisplay 的对位规则,但不做 sources[0] 兜底: + // 保持"主屏无画面 → 跳过该屏"的旧多屏语义 + const source = electronState.sources.find((item) => item.display_id === displayId && item.thumbnail.getSize().width === wantWidth) + || electronState.sources.find((item) => item.thumbnail.getSize().width === wantWidth); + if (!source || source.thumbnail.isEmpty()) throw new Error(`无法获取显示器 ${displayId} 的屏幕截图`); + void wantHeight; + return source.thumbnail.toPNG(); + } + } +}); + describe("CV vision capabilities", () => { beforeEach(() => { powershellMock.mockClear(); diff --git a/packages/rpa-core/src/capabilities/cv-vision.ts b/packages/rpa-core/src/capabilities/cv-vision.ts index e290fdb..5be8e43 100644 --- a/packages/rpa-core/src/capabilities/cv-vision.ts +++ b/packages/rpa-core/src/capabilities/cv-vision.ts @@ -1,5 +1,6 @@ import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; +import { requireScreenService } from "../engine/host-bridge"; import { powershell } from "../driver/windows/powershell"; import fs from "node:fs"; import os from "node:os"; @@ -44,30 +45,27 @@ async function gdiGrabScreen(imagePath: string): Promise { /** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */ interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number } -/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */ +/** 宿主屏幕服务逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */ async function grabScreenAll(baseDir: string): Promise { fs.mkdirSync(baseDir, { recursive: true }); try { - const { desktopCapturer, screen } = await import("electron"); - if (desktopCapturer && screen?.getAllDisplays) { - const shots: ScreenShot[] = []; - const displays = screen.getAllDisplays(); - for (const display of displays) { - const scale = display.scaleFactor || 1; - const sources = await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) } - }); - const source = sources.find(item => item.display_id === String(display.id)) - || sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale)) - || sources[0]; - if (!source || source.thumbnail.isEmpty()) continue; + const screenHost = requireScreenService(); + const shots: ScreenShot[] = []; + for (const display of screenHost.listDisplays()) { + const scale = display.scaleFactor || 1; + try { + const pngBytes = await screenHost.captureDisplay( + display.id, + Math.round(display.bounds.width * scale), + Math.round(display.bounds.height * scale) + ); + if (!pngBytes?.length) continue; const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`); - fs.writeFileSync(imagePath, source.thumbnail.toPNG()); + fs.writeFileSync(imagePath, Buffer.from(pngBytes)); shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale }); - } - if (shots.length) return shots; + } catch {} } + if (shots.length) return shots; } catch {} const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`); await gdiGrabScreen(imagePath); @@ -207,14 +205,11 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou return lines.join("\n"); } -/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */ +/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无宿主屏幕服务时按原值(缩放 100% 等价) */ async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> { try { - const { screen } = await import("electron"); - if (screen && typeof screen.dipToScreenPoint === "function") { - const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) }); - return { x: p.x, y: p.y }; - } + const point = requireScreenService().dipToScreenPoint({ x: Math.round(x), y: Math.round(y) }); + return { x: point.x, y: point.y }; } catch {} return { x: Math.round(x), y: Math.round(y) }; } diff --git a/packages/rpa-core/src/capabilities/vision.ts b/packages/rpa-core/src/capabilities/vision.ts index 62036da..ed01b08 100644 --- a/packages/rpa-core/src/capabilities/vision.ts +++ b/packages/rpa-core/src/capabilities/vision.ts @@ -3,6 +3,7 @@ import { promises as fs } from "node:fs"; import { PNG } from "pngjs"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; +import { requireScreenService, type HostDisplay } from "../engine/host-bridge"; import { powershell, psString } from "../driver/windows/powershell"; function text(args: Record, keys: string[], fallback = ""): string { @@ -15,28 +16,20 @@ function text(args: Record, keys: string[], fallback = ""): str * 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。 * 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标(副屏在主屏左侧时为负),用于结果坐标平移。 */ async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> { - const { desktopCapturer, screen } = await import("electron"); - const displays = screen.getAllDisplays(); + const screenHost = requireScreenService(); + const displays = screenHost.listDisplays(); if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息"); - // 每屏按自己的物理分辨率截图(display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位) - const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = []; + // 每屏按自己的物理分辨率截图(宿主按 displayId 对位 → 按物理原点排序对位) + const captures: { display: HostDisplay; origin: { x: number; y: number }; png: PNG }[] = []; for (const display of displays) { const scale = display.scaleFactor || 1; const wantW = Math.round(display.bounds.width * scale); const wantH = Math.round(display.bounds.height * scale); - const sources = await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: wantW, height: wantH } - }); - const source = sources.find(item => item.display_id === String(display.id)) - || sources.find(item => item.thumbnail.getSize().width === wantW) - || sources[0]; - if (!source || source.thumbnail.isEmpty()) { - throw new RuntimeError("desktop_error", "无法获取屏幕截图"); - } + const pngBytes = await screenHost.captureDisplay(display.id, wantW, wantH); + if (!pngBytes?.length) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); // 显示器 DIP 原点的物理坐标 = 该屏物理原点(屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点) - const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y }); - captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) }); + const origin = screenHost.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y }); + captures.push({ display, origin, png: PNG.sync.read(Buffer.from(pngBytes)) }); } const originX = Math.min(...captures.map(c => c.origin.x)); const originY = Math.min(...captures.map(c => c.origin.y)); @@ -63,9 +56,9 @@ async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; o /** DIP 区域 → 物理像素区域:左上角经 dipToScreenPoint 换算(天然支持副屏/负坐标),宽高乘所在屏缩放比 */ async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> { - const { screen } = await import("electron"); - const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) }); - const display = screen.getDisplayMatching({ + const screenHost = requireScreenService(); + const topLeft = screenHost.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) }); + const display = screenHost.displayMatching({ x: region.x, y: region.y, width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height)) }); @@ -429,7 +422,7 @@ async function locate(args: Record, evidenceDir: string, nodeId const rawAccuracy = Number(text(args, ["accuracy"], "0.8")); const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8)); // 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片 - const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1; + const scale = requireScreenService().primaryDisplay().scaleFactor || 1; const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []); if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像(要求相似度 ${(similarity * 100).toFixed(0)}%,未达到阈值)`, { nodeId, recommendation: "重新框选模板(框小一点、只框目标图案),或适当调低相似度(如 0.8);目标在副屏时确认其清晰可见" }); return { ...result, x: result.x + offsetX, y: result.y + offsetY }; diff --git a/packages/rpa-core/src/engine/host-bridge.ts b/packages/rpa-core/src/engine/host-bridge.ts index 854410d..8863aa2 100644 --- a/packages/rpa-core/src/engine/host-bridge.ts +++ b/packages/rpa-core/src/engine/host-bridge.ts @@ -1,4 +1,5 @@ import type { FlowDefinition } from "../types"; +import { RuntimeError } from "./types"; /** * 宿主桥(任务 2.1 正式化)。 @@ -46,7 +47,34 @@ export interface HostAiConfig { visionModel: string; } -/** 宿主能力桥最小面(决策点 4):dialog 窗口工厂 / DPI 坐标换算 / 配置读取 */ +/** 显示器信息(视觉能力的屏幕枚举用;id 为宿主侧稳定标识字符串) */ +export interface HostDisplay { + id: string; + bounds: { x: number; y: number; width: number; height: number }; + scaleFactor: number; +} + +/** + * 屏幕服务(迁移阶段1-A9):视觉能力原本经动态 import("electron") 拿 + * desktopCapturer/screen,现改为宿主注入,Core 回归零 Electron 依赖。 + */ +export interface HostScreenService { + /** 全部显示器 */ + listDisplays(): HostDisplay[]; + /** 主显示器 */ + primaryDisplay(): HostDisplay; + /** DIP 屏幕坐标 → 物理像素 */ + dipToScreenPoint(point: HostPoint): HostPoint; + /** 与该 DIP 矩形相交面积最大的显示器 */ + displayMatching(rect: { x: number; y: number; width: number; height: number }): HostDisplay; + /** + * 抓取指定显示器当前画面,像素尺寸尽量接近 want 值,返回 PNG 字节。 + * 无画面/抓取失败时抛错(调用方决定回退策略)。 + */ + captureDisplay(displayId: string, wantWidth: number, wantHeight: number): Promise; +} + +/** 宿主能力桥最小面(决策点 4):dialog 窗口工厂 / DPI 坐标换算 / 配置读取 / 屏幕服务 */ export interface HostBridge { /** 创建人机交互表单窗口(human-dialog 用) */ createFormWindow(options: { title: string }): HostDialogWindow; @@ -56,6 +84,8 @@ export interface HostBridge { screenToDipPoint(point: HostPoint): HostPoint; /** 读取 AI 服务配置(ai-service 用) */ loadAiConfig(): HostAiConfig; + /** 屏幕服务(vision/cv-vision 截屏与多屏几何);未注入时视觉能力报 host_unavailable */ + screen?: HostScreenService; } let hostBridge: HostBridge | undefined; @@ -67,3 +97,14 @@ export function setHostBridge(bridge: HostBridge): void { export function getHostBridge(): HostBridge | undefined { return hostBridge; } + +/** 视觉能力取屏幕服务的统一入口:未注入宿主桥/屏幕服务时抛稳定错误码 */ +export function requireScreenService(): HostScreenService { + const bridge = getHostBridge(); + if (!bridge?.screen) { + throw new RuntimeError("host_unavailable", "宿主未注入屏幕服务(HostBridge.screen),截屏/视觉能力不可用", { + recommendation: "宿主启动时需调用 setHostBridge 并提供 screen 服务(参考 demo 的 src/main/index.ts)" + }); + } + return bridge.screen; +} diff --git a/src/main/index.ts b/src/main/index.ts index ed0ff26..72d9163 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, dialog, ipcMain, screen, shell } from "electron"; +import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, screen, shell } from "electron"; import path from "node:path"; import type { AppInfo, AppSettings, FlowDefinition, FlowSaveOptions, FlowSaveResult, FlowSummary, FlowsImportResult } from "../shared/types"; import { @@ -24,7 +24,38 @@ import { logMain } from "./logger"; // 子流程仓库在宿主(flows.ts),包内能力经 host-bridge 取用(阶段 2 正式化 HostBridge) setSubflowSource({ listFlows: async () => listFlows(), getFlow: async (id) => getFlow(id) }); -// 宿主能力桥(决策点 4 最小面):人机交互窗口工厂 / DPI 坐标换算 / AI 配置读取 +// 宿主屏幕服务(迁移阶段1-A9):视觉能力经宿主桥拿 desktopCapturer/screen,Core 零 Electron 依赖 +const hostScreenService = { + listDisplays: () => + screen.getAllDisplays().map((display) => ({ + id: String(display.id), + bounds: display.bounds, + scaleFactor: display.scaleFactor || 1 + })), + primaryDisplay: () => { + const display = screen.getPrimaryDisplay(); + return { id: String(display.id), bounds: display.bounds, scaleFactor: display.scaleFactor || 1 }; + }, + dipToScreenPoint: (point: { x: number; y: number }) => screen.dipToScreenPoint(point), + displayMatching: (rect: { x: number; y: number; width: number; height: number }) => { + const display = screen.getDisplayMatching(rect); + return { id: String(display.id), bounds: display.bounds, scaleFactor: display.scaleFactor || 1 }; + }, + captureDisplay: async (displayId: string, wantWidth: number, wantHeight: number): Promise => { + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: wantWidth, height: wantHeight } + }); + const source = + sources.find((item) => item.display_id === displayId) || + sources.find((item) => item.thumbnail.getSize().width === wantWidth) || + sources[0]; + if (!source || source.thumbnail.isEmpty()) throw new Error(`无法获取显示器 ${displayId} 的屏幕截图`); + return source.thumbnail.toPNG(); + } +}; + +// 宿主能力桥(决策点 4 最小面):人机交互窗口工厂 / DPI 坐标换算 / AI 配置读取 / 屏幕服务 setHostBridge({ createFormWindow: ({ title }) => { const win = new BrowserWindow({ @@ -46,7 +77,8 @@ setHostBridge({ loadAiConfig: () => { const ai = loadSettings().aiConfig; return { baseUrl: ai.baseUrl ?? "", apiKey: ai.apiKey ?? "", model: ai.model ?? "", visionModel: ai.visionModel ?? "" }; - } + }, + screen: hostScreenService }); let mainWindow: BrowserWindow | null = null;