320 lines
15 KiB
PowerShell
320 lines
15 KiB
PowerShell
param(
|
||
[string]$Mode = "element",
|
||
[int]$CaptureX = 0,
|
||
[int]$CaptureY = 0,
|
||
[int]$CaptureWidth = 1000,
|
||
[int]$CaptureHeight = 700,
|
||
[string]$CapturePath = "",
|
||
[switch]$CaptureExact
|
||
)
|
||
$ErrorActionPreference = "Continue";
|
||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8;
|
||
$logPath = "D:/RPA/test-output/desktop-picker.log";
|
||
function Write-PickLog([string]$msg) {
|
||
try { $logDirectory = Split-Path -Parent $logPath; if (-not (Test-Path -LiteralPath $logDirectory)) { New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null }; Add-Content -Path $logPath -Value ((Get-Date -Format "HH:mm:ss.fff") + " | " + $msg) -Encoding UTF8 } catch {}
|
||
}
|
||
Write-PickLog ("=== start Mode=$Mode ===")
|
||
Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes, WindowsBase
|
||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name PickDpi -Namespace RpaPick | Out-Null
|
||
try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}
|
||
Write-PickLog "dpi aware set"
|
||
Add-Type -AssemblyName System.Windows.Forms
|
||
Write-PickLog "assemblies ok"
|
||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null
|
||
Write-PickLog "w32 ok"
|
||
if ($Mode -eq "capture") {
|
||
try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}
|
||
Add-Type -AssemblyName System.Drawing
|
||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);' -Name PickMetrics -Namespace RpaPickCapture | Out-Null
|
||
$vsX = [RpaPickCapture.PickMetrics]::GetSystemMetrics(76)
|
||
$vsY = [RpaPickCapture.PickMetrics]::GetSystemMetrics(77)
|
||
$vsW = [RpaPickCapture.PickMetrics]::GetSystemMetrics(78)
|
||
$vsH = [RpaPickCapture.PickMetrics]::GetSystemMetrics(79)
|
||
$cropW = [Math]::Min($CaptureWidth, $vsW)
|
||
$cropH = [Math]::Min($CaptureHeight, $vsH)
|
||
if ($CaptureExact) {
|
||
$cropX = [Math]::Max($vsX, [Math]::Min($CaptureX, ($vsX + $vsW - $cropW)))
|
||
$cropY = [Math]::Max($vsY, [Math]::Min($CaptureY, ($vsY + $vsH - $cropH)))
|
||
} else {
|
||
$cropX = [Math]::Max($vsX, [Math]::Min($CaptureX - [int]($cropW / 2), ($vsX + $vsW - $cropW)))
|
||
$cropY = [Math]::Max($vsY, [Math]::Min($CaptureY - [int]($cropH / 2), ($vsY + $vsH - $cropH)))
|
||
}
|
||
$markerX = $CaptureX - $cropX
|
||
$markerY = $CaptureY - $cropY
|
||
if ($CapturePath) { $imagePath = $CapturePath } else { $imagePath = Join-Path $env:TEMP ("ai-pick-" + [Guid]::NewGuid().ToString("N") + ".png") }
|
||
$imageDir = Split-Path -Parent $imagePath
|
||
if (-not (Test-Path -LiteralPath $imageDir)) { New-Item -ItemType Directory -Path $imageDir -Force | Out-Null }
|
||
$bitmap = New-Object System.Drawing.Bitmap $cropW, $cropH
|
||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||
$graphics.CopyFromScreen($cropX, $cropY, 0, 0, $bitmap.Size)
|
||
$pen = New-Object System.Drawing.Pen([System.Drawing.Color]::Red, 3)
|
||
try {
|
||
$graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
|
||
if (-not $CaptureExact) {
|
||
$graphics.DrawLine($pen, ($markerX - 20), $markerY, ($markerX + 20), $markerY)
|
||
$graphics.DrawLine($pen, $markerX, ($markerY - 20), $markerX, ($markerY + 20))
|
||
$graphics.DrawRectangle($pen, ($markerX - 9), ($markerY - 9), 18, 18)
|
||
}
|
||
} finally {
|
||
$pen.Dispose(); $graphics.Dispose()
|
||
}
|
||
$bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)
|
||
$bitmap.Dispose()
|
||
$windowTitle = ""
|
||
$uiaName = ""
|
||
$uiaType = ""
|
||
try {
|
||
$pt = New-Object System.Windows.Point($CaptureX, $CaptureY)
|
||
$ae = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||
if ($ae) {
|
||
$ac = $ae.Current
|
||
$uiaName = [string]$ac.Name
|
||
$uiaType = $ac.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||
$cur = $ae
|
||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||
$windowTitle = [string]$cur.Current.Name
|
||
$cur = $walker.GetParent($cur)
|
||
}
|
||
}
|
||
} catch {}
|
||
$result = @{
|
||
aiCapture = $true
|
||
cursorX = $CaptureX
|
||
cursorY = $CaptureY
|
||
cropX = $cropX
|
||
cropY = $cropY
|
||
cropW = $cropW
|
||
cropH = $cropH
|
||
markerX = $markerX
|
||
markerY = $markerY
|
||
window = $windowTitle
|
||
uiaName = $uiaName
|
||
uiaType = $uiaType
|
||
imagePath = $imagePath
|
||
}
|
||
Write-Output (ConvertTo-Json $result -Compress)
|
||
exit 0
|
||
}
|
||
if ($Mode -eq "windows") {
|
||
$wins = @()
|
||
foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } }
|
||
Write-PickLog "windows: $($wins.Count)"
|
||
Write-Output (ConvertTo-Json @($wins) -Compress)
|
||
exit 0
|
||
}
|
||
$info = New-Object System.Windows.Forms.Form
|
||
$info.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None
|
||
$info.TopMost = $true
|
||
$info.ShowInTaskbar = $false
|
||
$info.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual
|
||
$sw2 = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width
|
||
$info.Location = New-Object System.Drawing.Point(([int]($sw2 / 2) - 280), 10)
|
||
$info.Size = New-Object System.Drawing.Size(560, 64)
|
||
$info.BackColor = [System.Drawing.Color]::FromArgb(15, 40, 90)
|
||
$info.Opacity = 0.92
|
||
$infoLabel = New-Object System.Windows.Forms.Label
|
||
$infoLabel.Dock = [System.Windows.Forms.DockStyle]::Fill
|
||
$infoLabel.ForeColor = [System.Drawing.Color]::White
|
||
$infoLabel.Font = New-Object System.Drawing.Font("Microsoft YaHei", 11)
|
||
$infoLabel.TextAlign = [System.Drawing.ContentAlignment]::MiddleCenter
|
||
$infoLabel.Text = "移动鼠标到目标元素上"
|
||
$info.Controls.Add($infoLabel)
|
||
$hl = New-Object System.Windows.Forms.Form
|
||
$hl.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None
|
||
$hl.TopMost = $true
|
||
$hl.ShowInTaskbar = $false
|
||
$hl.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual
|
||
$hl.BackColor = [System.Drawing.Color]::Fuchsia
|
||
$hl.TransparencyKey = [System.Drawing.Color]::Fuchsia
|
||
$hl.Add_Paint({
|
||
param($sender, $e)
|
||
$pen = New-Object System.Drawing.Pen([System.Drawing.Color]::FromArgb(255, 255, 32, 32), 3)
|
||
try {
|
||
$size = $sender.ClientSize
|
||
$e.Graphics.DrawRectangle($pen, 1, 1, ($size.Width - 3), ($size.Height - 3))
|
||
} finally {
|
||
$pen.Dispose()
|
||
}
|
||
})
|
||
function Set-ClickThrough($form) {
|
||
if ($form.IsHandleCreated) {
|
||
$GWL_EXSTYLE = -20
|
||
$WS_EX_LAYERED = 0x80000
|
||
$WS_EX_TRANSPARENT = 0x20
|
||
$WS_EX_NOACTIVATE = 0x08000000
|
||
$WS_EX_TOOLWINDOW = 0x80
|
||
$style = [RpaPick.PickW32]::GetWindowLong($form.Handle, $GWL_EXSTYLE)
|
||
[void][RpaPick.PickW32]::SetWindowLong($form.Handle, $GWL_EXSTYLE, $style -bor $WS_EX_LAYERED -bor $WS_EX_TRANSPARENT -bor $WS_EX_NOACTIVATE -bor $WS_EX_TOOLWINDOW)
|
||
}
|
||
}
|
||
|
||
function Set-HighlightBox([int]$x, [int]$y, [int]$w, [int]$h) {
|
||
$bounds = New-Object System.Drawing.Rectangle(($x - 2), ($y - 2), ($w + 4), ($h + 4))
|
||
if (-not $hl.Visible) {
|
||
$hl.Bounds = $bounds
|
||
$hl.Show()
|
||
Set-ClickThrough $hl
|
||
} elseif ($hl.Bounds.X -ne $bounds.X -or $hl.Bounds.Y -ne $bounds.Y -or $hl.Bounds.Width -ne $bounds.Width -or $hl.Bounds.Height -ne $bounds.Height) {
|
||
# 不重设隐藏；同一元素内移动时不重复改窗口，避免 WinForms 分层窗体重绘闪烁
|
||
$hl.Bounds = $bounds
|
||
}
|
||
}
|
||
function Hide-Highlight { $hl.Hide() }
|
||
function Get-DeepestAtPoint([double]$px, [double]$py) {
|
||
$point = New-Object System.Windows.Point($px, $py)
|
||
$cur = [System.Windows.Automation.AutomationElement]::FromPoint($point)
|
||
if (-not $cur) { return $null }
|
||
for ($depth = 0; $depth -lt 25; $depth++) {
|
||
$children = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition)
|
||
$next = $null
|
||
$nextArea = [double]::MaxValue
|
||
foreach ($c in $children) {
|
||
try {
|
||
$r = $c.Current.BoundingRectangle
|
||
if ($r.Width -gt 1 -and $r.Height -gt 1 -and $px -ge $r.X -and $px -le ($r.X + $r.Width) -and $py -ge $r.Y -and $py -le ($r.Y + $r.Height)) {
|
||
$area = $r.Width * $r.Height
|
||
if ($area -lt $nextArea) { $next = $c; $nextArea = $area }
|
||
}
|
||
} catch {}
|
||
}
|
||
if (-not $next) { break }
|
||
$cur = $next
|
||
}
|
||
return $cur
|
||
}
|
||
$info.Show()
|
||
Set-ClickThrough $info
|
||
$stage = "hover"
|
||
$chain = New-Object System.Collections.ArrayList
|
||
$selIdx = -1
|
||
$prevSpace = $false
|
||
$prevPgUp = $false
|
||
$prevPgDn = $false
|
||
$prevF2 = $false
|
||
$cacheX = [int]::MinValue
|
||
$cacheY = [int]::MinValue
|
||
$cacheEl = $null
|
||
$cacheCtrlName = ""
|
||
$cacheName = ""
|
||
$cachePaneX = 0
|
||
$cachePaneY = 0
|
||
$cachePaneW = 0
|
||
$cachePaneH = 0
|
||
$cacheAt = [DateTime]::MinValue
|
||
$lastQueryAt = [DateTime]::MinValue
|
||
$queryGapMs = 45
|
||
$cacheTtlMs = 1800
|
||
Write-PickLog "loop start"
|
||
while ($true) {
|
||
try {
|
||
$space = (([RpaPick.PickW32]::GetAsyncKeyState(32) -band 0x8001) -ne 0)
|
||
$esc = (([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8001) -ne 0)
|
||
$pgUp = (([RpaPick.PickW32]::GetAsyncKeyState(33) -band 0x8001) -ne 0)
|
||
$pgDn = (([RpaPick.PickW32]::GetAsyncKeyState(34) -band 0x8001) -ne 0)
|
||
$f2 = (([RpaPick.PickW32]::GetAsyncKeyState(113) -band 0x8001) -ne 0)
|
||
if ($esc) { Write-PickLog "esc"; break }
|
||
if ($stage -eq "hover") {
|
||
$ptX = [System.Windows.Forms.Cursor]::Position.X
|
||
$ptY = [System.Windows.Forms.Cursor]::Position.Y
|
||
$now = [DateTime]::UtcNow
|
||
$samePoint = ($ptX -eq $cacheX -and $ptY -eq $cacheY)
|
||
$insideCache = $cacheEl -ne $null -and $ptX -ge $cachePaneX -and $ptX -lt ($cachePaneX + $cachePaneW) -and $ptY -ge $cachePaneY -and $ptY -lt ($cachePaneY + $cachePaneH)
|
||
$cacheAlive = ($null -ne $cacheEl -and ($now - $cacheAt).TotalMilliseconds -le $cacheTtlMs)
|
||
# 鼠标还在当前元素矩形内部时不丢弃缓存；UIA 查询完成后才切换，避免移动过程先隐藏再显示
|
||
if (-not $samePoint -and -not $insideCache) { $cacheEl = $null }
|
||
if (-not $cacheAlive -and ($now - $lastQueryAt).TotalMilliseconds -ge $queryGapMs) {
|
||
$queryX = $ptX
|
||
$queryY = $ptY
|
||
$lastQueryAt = [DateTime]::UtcNow
|
||
try { $queried = Get-DeepestAtPoint $queryX $queryY } catch { $queried = $null; Write-PickLog ("QUERY ERR: " + $_.Exception.Message) }
|
||
$afterX = [System.Windows.Forms.Cursor]::Position.X
|
||
$afterY = [System.Windows.Forms.Cursor]::Position.Y
|
||
if ($afterX -ne $queryX -or $afterY -ne $queryY) {
|
||
$queried = $null
|
||
Write-PickLog "query stale: cursor moved"
|
||
}
|
||
if ($null -ne $queried) {
|
||
$qc = $queried.Current
|
||
$qr = $qc.BoundingRectangle
|
||
$cacheEl = $queried
|
||
$cacheX = $queryX
|
||
$cacheY = $queryY
|
||
$cacheAt = [DateTime]::UtcNow
|
||
$cacheCtrlName = $qc.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||
$cacheName = [string]$qc.Name
|
||
$cachePaneX = [int]$qr.X
|
||
$cachePaneY = [int]$qr.Y
|
||
$cachePaneW = [int]$qr.Width
|
||
$cachePaneH = [int]$qr.Height
|
||
} else {
|
||
$cacheEl = $null
|
||
$cacheCtrlName = ""
|
||
$cacheName = ""
|
||
}
|
||
}
|
||
$el = $cacheEl
|
||
if ($el) {
|
||
Set-HighlightBox $cachePaneX $cachePaneY $cachePaneW $cachePaneH
|
||
$aiHint = if ($cacheCtrlName -eq "Pane") { " [F2]AI识别" } else { "" }
|
||
$infoLabel.Text = "名称: " + $cacheName + " 类型: " + $cacheCtrlName + " [空格]锁定" + $aiHint + " [Esc]取消"
|
||
} else { Hide-Highlight }
|
||
if ($f2 -and -not $prevF2 -and $el -and $cacheCtrlName -eq "Pane") {
|
||
Write-PickLog "f2 ai"
|
||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||
$cur = $el
|
||
$topTitle = ""
|
||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||
$topTitle = [string]$cur.Current.Name
|
||
$cur = $walker.GetParent($cur)
|
||
}
|
||
$result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = $cacheName; controlType = $cacheCtrlName; paneX = $cachePaneX; paneY = $cachePaneY; paneW = $cachePaneW; paneH = $cachePaneH }
|
||
break
|
||
}
|
||
if ($space -and -not $prevSpace -and $el) {
|
||
Write-PickLog "space lock"
|
||
$stage = "locked"
|
||
$chain = New-Object System.Collections.ArrayList
|
||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||
$cur = $cacheEl
|
||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||
$c = $cur.Current
|
||
$r = $c.BoundingRectangle
|
||
[void]$chain.Insert(0, @{ el = $cur; name = [string]$c.Name; automationId = [string]$c.AutomationId; controlType = $c.ControlType.ProgrammaticName.Replace("ControlType.", ""); x = [int]$r.X; y = [int]$r.Y; w = [int]$r.Width; h = [int]$r.Height; topLevel = $walker.GetParent($cur).Equals([System.Windows.Automation.AutomationElement]::RootElement) })
|
||
$cur = $walker.GetParent($cur)
|
||
}
|
||
$selIdx = $chain.Count - 1
|
||
Write-PickLog ("locked chain=" + $chain.Count)
|
||
$infoLabel.Text = "已锁定 [" + ($selIdx + 1) + "/" + $chain.Count + "] [PageUp/PageDown]调层级 [空格]确认回填 [Esc]取消"
|
||
}
|
||
}
|
||
if ($stage -eq "locked") {
|
||
$item = $chain[$selIdx]
|
||
if ($item) {
|
||
Set-HighlightBox $item.x $item.y $item.w $item.h
|
||
$infoLabel.Text = "层级 " + ($selIdx + 1) + "/" + $chain.Count + " 名称: " + $item.name + " 类型: " + $item.controlType + " [PageUp/PageDown]调层级 [空格]确认 [Esc]取消"
|
||
}
|
||
if ($pgUp -and -not $prevPgUp -and $selIdx -gt 0) { $selIdx -= 1 }
|
||
if ($pgDn -and -not $prevPgDn -and $selIdx -lt ($chain.Count - 1)) { $selIdx += 1 }
|
||
if ($space -and -not $prevSpace) {
|
||
$item = $chain[$selIdx]
|
||
$rootItem = $chain | Where-Object { $_.topLevel -and $_.name } | Select-Object -First 1
|
||
if ($rootItem) { $rootTitle = $rootItem.name } else { foreach ($it in $chain) { if ($it.name) { $rootTitle = $it.name; break } } }
|
||
$result = @{ selected = $selIdx; window = $rootTitle; name = $item.name; automationId = $item.automationId; controlType = $item.controlType; x = $item.x; y = $item.y; w = $item.w; h = $item.h }
|
||
break
|
||
}
|
||
}
|
||
} catch {
|
||
Write-PickLog ("LOOP ERR: " + $_.Exception.Message)
|
||
}
|
||
[System.Windows.Forms.Application]::DoEvents()
|
||
$prevSpace = $space
|
||
$prevPgUp = $pgUp
|
||
$prevPgDn = $pgDn
|
||
$prevF2 = $f2
|
||
Start-Sleep -Milliseconds 60
|
||
}
|
||
$info.Close()
|
||
$hl.Close()
|
||
if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) }
|