feat(android): 阶段 2b——连接界面 + IME 文本通道 + 控制键扫描码
两态 UI：连接态是 host/port/code 表单（原生 View 零依赖，intent extras 保留且 优先，构建脚本不受影响），成功后切远控态。只存 host/port；验证码不落盘（用户裁决）。 返回键分态：IME 显示时先收键盘，否则断开回表单——否则一次返回关键盘又杀会话。 IME 走 P1：onCheckIsTextEditor + BaseInputConnection 子类。commitText 整段 经 v3 Text 通道上行（char[]→GetCharArrayRegion，UTF-16 无损）；控制键映射 Set-1 扫描码（值与 viewer/keymap.rs 同源，右 Shift 按官方文档不置 extended）。 composing 不上远端（U4），getText* 回空串——本地没有也不伪造远端缓冲区。 键盘按钮显式 showSoftInput，不自动弹（vivo 行为不一致 + 挡画面）。 RemoteSurfaceView 补 isFocusableInTouchMode：没有它触摸态拿不到焦点， InputConnection 根本不会建立（2a 弹不出键盘的根因之一）。
This commit is contained in:
郑博文
parent
55cc8edbc1
commit
6c28a3f7e3
@ -1,29 +1,50 @@
|
||||
package com.verify.remote
|
||||
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.os.Bundle
|
||||
import android.text.InputType
|
||||
import android.util.Log
|
||||
import android.view.Gravity
|
||||
import android.view.KeyEvent
|
||||
import android.view.SurfaceHolder
|
||||
import android.view.View
|
||||
import android.view.inputmethod.InputMethodManager
|
||||
import android.widget.Button
|
||||
import android.widget.EditText
|
||||
import android.widget.FrameLayout
|
||||
import android.widget.LinearLayout
|
||||
import android.widget.LinearLayout.LayoutParams
|
||||
import android.widget.TextView
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
/**
|
||||
* 安卓主控端（阶段 2a）：连接被控端 → MediaCodec 解码 → SurfaceView 显示 + 触摸操作。
|
||||
* 安卓主控端（阶段 2b）：正式连接界面 + 远控画面 + 触摸操作 + 软键盘文本输入。
|
||||
*
|
||||
* 两态：
|
||||
* - 连接态：host / port / code 表单 + 状态文本（阶段 1/2a 只能靠 `am start --es host`
|
||||
* 传参，本界面对它补位；intent extras 仍保留且优先级更高，构建脚本继续可用）。
|
||||
* - 远控态：letterbox 画面 + 底部状态/操作条（帧数、RTT、诊断、键盘、断开）。
|
||||
*
|
||||
* 线程模型（与 android-ffi 的锁纪律对应）：
|
||||
* - UI 线程：触摸/按键 → nativeSend*（入队不阻塞）；连接/断开
|
||||
* - UI 线程：触摸/按键/IME 提交 → nativeSend*（入队不阻塞）；连接/断开
|
||||
* - 解码线程：nativeNextFrame 阻塞取帧 → H264Decoder.queue
|
||||
* - 断开顺序：**先置 stopDecoding 并 join 解码线程，再 destroy**
|
||||
* （Rust 侧明确：解码线程存活期间 destroy 是 UB）
|
||||
*
|
||||
* 布局：SurfaceView 按视频宽高比 letterbox 居中（RemoteSurfaceView），
|
||||
* 触摸坐标只在画面区域内产生，与远端归一化坐标一一对应。
|
||||
* 持久化：只存 host/port（连接成功后落盘）；**验证码不落盘**（2026-10-30 用户裁决）。
|
||||
*/
|
||||
class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
|
||||
private lateinit var surfaceView: RemoteSurfaceView
|
||||
private lateinit var statusView: TextView
|
||||
private lateinit var connectPanel: LinearLayout
|
||||
private lateinit var remotePanel: LinearLayout
|
||||
private lateinit var hostInput: EditText
|
||||
private lateinit var portInput: EditText
|
||||
private lateinit var codeInput: EditText
|
||||
private lateinit var connectButton: Button
|
||||
private lateinit var connectStatus: TextView
|
||||
|
||||
@Volatile
|
||||
private var handle = 0L
|
||||
@ -32,12 +53,16 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
private var stopDecoding = false
|
||||
private var decodeThread: Thread? = null
|
||||
|
||||
/** 连接中标志：防重复点击起出第二个会话（句柄未回填前 handle 仍是 0） */
|
||||
@Volatile
|
||||
private var connecting = false
|
||||
|
||||
@Volatile
|
||||
private var remoteW = 0
|
||||
@Volatile
|
||||
private var remoteH = 0
|
||||
|
||||
/** 最近一次触摸事件（诊断用：logcat 被限流，状态栏是唯一可靠通道） */
|
||||
/** 最近一次输入事件（触摸/IME）诊断：logcat 被限流，状态栏是唯一可靠通道 */
|
||||
@Volatile
|
||||
private var lastTouchDbg = "-"
|
||||
|
||||
@ -52,18 +77,100 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {}
|
||||
override fun surfaceChanged(h: SurfaceHolder, f: Int, w: Int, ht: Int) {}
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
// Surface 没了解码器输出会失效：断开，重进 App 重连
|
||||
// Surface 没了解码器输出会失效：断开，回到连接界面重连
|
||||
disconnect()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
val root = LinearLayout(this).apply {
|
||||
orientation = LinearLayout.VERTICAL
|
||||
setBackgroundColor(0xFF000000.toInt())
|
||||
}
|
||||
connectPanel = buildConnectPanel()
|
||||
remotePanel = buildRemotePanel()
|
||||
val lp = LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f)
|
||||
root.addView(connectPanel, lp)
|
||||
root.addView(remotePanel, LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f))
|
||||
setContentView(root)
|
||||
|
||||
showConnectState("未连接")
|
||||
|
||||
// intent 入口优先级高于表单默认值：build-android.ps1 与调试脚本依赖它
|
||||
val hostExtra = intent?.getStringExtra("host")
|
||||
if (!hostExtra.isNullOrEmpty()) {
|
||||
hostInput.setText(hostExtra)
|
||||
portInput.setText(intent.getIntExtra("port", 47800).toString())
|
||||
intent.getStringExtra("code")?.let { codeInput.setText(it) }
|
||||
startConnect()
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 连接态 UI ----
|
||||
|
||||
private fun buildConnectPanel(): LinearLayout {
|
||||
val prefs = getSharedPreferences(PREFS, Context.MODE_PRIVATE)
|
||||
// 只回填 host/port（上次**成功连接**过的值）；验证码每次手输
|
||||
hostInput = editText(
|
||||
"被控端 IP",
|
||||
InputType.TYPE_CLASS_TEXT,
|
||||
prefs.getString("host", "").orEmpty(),
|
||||
)
|
||||
portInput = editText(
|
||||
"端口",
|
||||
InputType.TYPE_CLASS_NUMBER,
|
||||
prefs.getInt("port", 47800).toString(),
|
||||
)
|
||||
// 验证码刻意不从 prefs 读（不落盘）
|
||||
codeInput = editText("验证码", InputType.TYPE_CLASS_TEXT, "")
|
||||
connectButton = Button(this).apply {
|
||||
text = "连接"
|
||||
setOnClickListener { startConnect() }
|
||||
}
|
||||
connectStatus = TextView(this).apply {
|
||||
textSize = 13f
|
||||
setTextColor(0xFFBBBBBB.toInt())
|
||||
setPadding(0, dp(8), 0, 0)
|
||||
}
|
||||
val title = TextView(this).apply {
|
||||
text = "连接被控端"
|
||||
textSize = 18f
|
||||
setTextColor(0xFFFFFFFF.toInt())
|
||||
setPadding(0, dp(16), 0, dp(8))
|
||||
}
|
||||
return LinearLayout(this).apply {
|
||||
orientation = LinearLayout.VERTICAL
|
||||
setPadding(dp(20), dp(24), dp(20), dp(24))
|
||||
addView(title)
|
||||
addView(hostInput)
|
||||
addView(portInput)
|
||||
addView(codeInput)
|
||||
addView(connectButton)
|
||||
addView(connectStatus)
|
||||
}
|
||||
}
|
||||
|
||||
private fun editText(hint: String, type: Int, initial: String) = EditText(this).apply {
|
||||
inputType = type
|
||||
this.hint = hint
|
||||
setText(initial)
|
||||
textSize = 16f
|
||||
setTextColor(0xFFFFFFFF.toInt())
|
||||
setHintTextColor(0xFF777777.toInt())
|
||||
// 表单文字不能被画面抢占焦点
|
||||
isSingleLine = true
|
||||
}
|
||||
|
||||
// ---- 远控态 UI ----
|
||||
|
||||
private fun buildRemotePanel(): LinearLayout {
|
||||
statusView = TextView(this).apply {
|
||||
textSize = 12f
|
||||
setPadding(20, 12, 20, 12)
|
||||
setPadding(dp(12), dp(6), dp(12), dp(6))
|
||||
setBackgroundColor(0xCC000000.toInt())
|
||||
setTextColor(0xFFFFFFFF.toInt())
|
||||
maxLines = 3
|
||||
}
|
||||
|
||||
// 画面居中、黑边填充。
|
||||
// 用 FrameLayout + MATCH_PARENT（不是 WRAP_CONTENT）：FrameLayout 会把
|
||||
// 子视图的 WRAP_CONTENT 当 MATCH_PARENT 处理，onMeasure 里拿到的
|
||||
@ -71,49 +178,69 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
// （2026-10-30 实测：视图被撑成 1080x2001，触摸按 2001 归一化 → 坐标全错）。
|
||||
// MATCH_PARENT + gravity=center 时父容器传 AT_MOST，onMeasure 的
|
||||
// setMeasuredDimension 才真正决定尺寸，再被 gravity 居中摆放。
|
||||
val stage = android.widget.FrameLayout(this).apply {
|
||||
val stage = FrameLayout(this).apply {
|
||||
setBackgroundColor(0xFF000000.toInt())
|
||||
addView(
|
||||
surfaceView,
|
||||
android.widget.FrameLayout.LayoutParams(
|
||||
android.widget.FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
android.widget.FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
android.view.Gravity.CENTER
|
||||
)
|
||||
FrameLayout.LayoutParams(
|
||||
FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
FrameLayout.LayoutParams.MATCH_PARENT,
|
||||
Gravity.CENTER,
|
||||
),
|
||||
)
|
||||
}
|
||||
val root = android.widget.LinearLayout(this).apply {
|
||||
orientation = android.widget.LinearLayout.VERTICAL
|
||||
addView(stage, android.widget.LinearLayout.LayoutParams(
|
||||
android.widget.LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f))
|
||||
val keyboardButton = Button(this).apply {
|
||||
text = "键盘"
|
||||
setOnClickListener { showKeyboard() }
|
||||
}
|
||||
val disconnectButton = Button(this).apply {
|
||||
text = "断开"
|
||||
setOnClickListener { disconnect() }
|
||||
}
|
||||
val bar = LinearLayout(this).apply {
|
||||
orientation = LinearLayout.HORIZONTAL
|
||||
addView(keyboardButton, LayoutParams(0, LayoutParams.WRAP_CONTENT, 1f))
|
||||
addView(disconnectButton, LayoutParams(0, LayoutParams.WRAP_CONTENT, 1f))
|
||||
}
|
||||
return LinearLayout(this).apply {
|
||||
orientation = LinearLayout.VERTICAL
|
||||
visibility = View.GONE
|
||||
addView(stage, LayoutParams(LayoutParams.MATCH_PARENT, 0, 1f))
|
||||
// 固定高度而非 WRAP_CONTENT：与 weight=1 的兄弟共存时，
|
||||
// WRAP_CONTENT 可能被压成 0（实测 bounds=[0,0][0,0]，状态栏不可见）
|
||||
addView(statusView, android.widget.LinearLayout.LayoutParams(
|
||||
android.widget.LinearLayout.LayoutParams.MATCH_PARENT,
|
||||
(48 * resources.displayMetrics.density).toInt()))
|
||||
}
|
||||
setContentView(root)
|
||||
refreshStatus()
|
||||
intent?.getStringExtra("host")?.let { host ->
|
||||
connect(host, intent.getIntExtra("port", 47800), intent.getStringExtra("code") ?: "123456")
|
||||
addView(statusView, LayoutParams(LayoutParams.MATCH_PARENT, dp(64)))
|
||||
addView(bar, LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.WRAP_CONTENT))
|
||||
}
|
||||
}
|
||||
|
||||
/** 状态栏统一刷新：连接态 + 帧数/RTT + 最近触摸诊断（logcat 被限流，这是唯一通道） */
|
||||
@Volatile
|
||||
private var lastFrames = 0L
|
||||
@Volatile
|
||||
private var lastRtt = 0L
|
||||
private fun showConnectState(msg: String) {
|
||||
connecting = false
|
||||
remotePanelVisible(false)
|
||||
connectStatus.text = msg
|
||||
}
|
||||
|
||||
private fun refreshStatus() {
|
||||
if (handle == 0L) {
|
||||
statusView.text = "未连接｜am start ... --es host <IP> --ei port 47800 --es code t"
|
||||
return
|
||||
}
|
||||
statusView.text = buildString {
|
||||
append("已连接 ${remoteW}x$remoteH 帧=$lastFrames RTT=${lastRtt}ms\n")
|
||||
append("触摸: $lastTouchDbg")
|
||||
}
|
||||
private fun showRemoteState() {
|
||||
remotePanelVisible(true)
|
||||
connectPanel.visibility = View.GONE
|
||||
}
|
||||
|
||||
private fun remotePanelVisible(show: Boolean) {
|
||||
connectPanel.visibility = if (show) View.GONE else View.VISIBLE
|
||||
remotePanel.visibility = if (show) View.VISIBLE else View.GONE
|
||||
}
|
||||
|
||||
/**
|
||||
* 显式唤起软键盘。不做自动弹：自动弹会挡画面，且 vivo ROM 行为不一致。
|
||||
* SurfaceView 已在触摸模式下可聚焦（见 RemoteSurfaceView.init），
|
||||
* 否则系统不会建立 InputConnection。
|
||||
*/
|
||||
private fun showKeyboard() {
|
||||
if (handle == 0L) return
|
||||
surfaceView.requestFocus()
|
||||
val imm = getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager
|
||||
imm.showSoftInput(surfaceView, InputMethodManager.SHOW_IMPLICIT)
|
||||
lastTouchDbg = "showSoftInput 已请求（焦点=${surfaceView.hasFocus()}）"
|
||||
refreshStatus()
|
||||
}
|
||||
|
||||
// ---- RemoteSurfaceView.Listener ----
|
||||
@ -133,19 +260,65 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
if (h != 0L) RustBridge.nativeSendWheel(h, dy)
|
||||
}
|
||||
|
||||
/** 返回键 = 断开（不退出 App，便于重连） */
|
||||
/** 协议 v3：IME 提交的整段文本（UTF-16 code unit，含落单代理项原样透传） */
|
||||
override fun sendText(units: CharArray) {
|
||||
val h = handle
|
||||
if (h != 0L) RustBridge.nativeSendText(h, units)
|
||||
}
|
||||
|
||||
override fun sendKey(key: WinKey, down: Boolean) {
|
||||
val h = handle
|
||||
if (h != 0L) {
|
||||
RustBridge.nativeSendKey(h, key.scancode, if (key.extended) 1 else 0, if (down) 1 else 0)
|
||||
}
|
||||
}
|
||||
|
||||
override fun isReadyForInput(): Boolean = handle != 0L
|
||||
|
||||
/**
|
||||
* 返回键：软键盘开着时先收键盘（否则一次返回同时关键盘和会话）；
|
||||
* 远控态已连接 = 断开回连接界面；连接态 = 交还系统（退出 App）。
|
||||
*/
|
||||
override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
|
||||
if (keyCode == KeyEvent.KEYCODE_BACK && handle != 0L) {
|
||||
disconnect()
|
||||
return true
|
||||
if (keyCode == KeyEvent.KEYCODE_BACK) {
|
||||
val imm = getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager
|
||||
// isAcceptingText = 当前有 IME 连接且正在接收输入（API 17+）
|
||||
if (imm.isAcceptingText) {
|
||||
imm.hideSoftInputFromWindow(surfaceView.windowToken, 0)
|
||||
return true
|
||||
}
|
||||
if (handle != 0L) {
|
||||
disconnect()
|
||||
return true
|
||||
}
|
||||
}
|
||||
return super.onKeyDown(keyCode, event)
|
||||
}
|
||||
|
||||
// ---- 连接 / 解码 ----
|
||||
|
||||
private fun connect(host: String, port: Int, code: String) {
|
||||
if (handle != 0L) return
|
||||
private fun startConnect() {
|
||||
if (connecting || handle != 0L) return
|
||||
val host = hostInput.text.toString().trim()
|
||||
val code = codeInput.text.toString()
|
||||
val port = portInput.text.toString().trim().toIntOrNull()
|
||||
when {
|
||||
host.isEmpty() -> {
|
||||
connectStatus.text = "请填写被控端 IP"
|
||||
return
|
||||
}
|
||||
port == null || port !in 1..65535 -> {
|
||||
connectStatus.text = "端口必须是 1..65535"
|
||||
return
|
||||
}
|
||||
code.isEmpty() -> {
|
||||
connectStatus.text = "请填写验证码"
|
||||
return
|
||||
}
|
||||
}
|
||||
connecting = true
|
||||
connectButton.isEnabled = false
|
||||
showRemoteState()
|
||||
statusView.text = "连接 $host:$port …"
|
||||
thread(name = "connect") {
|
||||
try {
|
||||
@ -156,7 +329,15 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
handle = h
|
||||
remoteW = w
|
||||
remoteH = ht
|
||||
connecting = false
|
||||
runOnUiThread {
|
||||
connectButton.isEnabled = true
|
||||
// 只有认证成功的会话才值得记住；验证码不落盘（用户裁决）
|
||||
getSharedPreferences(PREFS, Context.MODE_PRIVATE)
|
||||
.edit()
|
||||
.putString("host", host)
|
||||
.putInt("port", port)
|
||||
.apply()
|
||||
if (w > 0 && ht > 0) {
|
||||
surfaceView.videoAspectRatio = w.toFloat() / ht
|
||||
}
|
||||
@ -165,7 +346,12 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "connect failed", e)
|
||||
runOnUiThread { statusView.text = "连接失败: ${e.message}" }
|
||||
val msg = e.message ?: "连接失败"
|
||||
runOnUiThread {
|
||||
connectButton.isEnabled = true
|
||||
handle = 0L
|
||||
showConnectState("连接失败：$msg")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -222,7 +408,7 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
lastFrames = 0
|
||||
lastRtt = 0
|
||||
RustBridge.destroy(h)
|
||||
refreshStatus()
|
||||
showConnectState("已断开")
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
@ -230,7 +416,34 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener {
|
||||
disconnect()
|
||||
}
|
||||
|
||||
/** 状态栏统一刷新：连接态 + 帧数/RTT + 最近输入诊断（logcat 被限流，这是唯一通道） */
|
||||
@Volatile
|
||||
private var lastFrames = 0L
|
||||
@Volatile
|
||||
private var lastRtt = 0L
|
||||
|
||||
private fun refreshStatus() {
|
||||
if (handle == 0L) {
|
||||
statusView.text = if (connecting) statusView.text else "未连接"
|
||||
return
|
||||
}
|
||||
statusView.text = buildString {
|
||||
append("已连接 v$PROTO_LABEL ${remoteW}x$remoteH 帧=$lastFrames RTT=${lastRtt}ms\n")
|
||||
append("输入: $lastTouchDbg")
|
||||
}
|
||||
}
|
||||
|
||||
private fun dp(v: Int) = (v * resources.displayMetrics.density).toInt()
|
||||
|
||||
companion object {
|
||||
private const val TAG = "MainActivity"
|
||||
private const val PREFS = "connect"
|
||||
|
||||
/**
|
||||
* 与协议版本保持一致：远控画面状态栏显示当前握手版本（方案 R5 要求）。
|
||||
* 值随 `common::PROTO_VERSION` 人工同步——Android 侧不做线格式解析，
|
||||
* 版本由 Rust 握手校验，这里只作显示与排障用。
|
||||
*/
|
||||
private const val PROTO_LABEL = "3"
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,108 @@
|
||||
package com.verify.remote
|
||||
|
||||
import android.view.KeyEvent
|
||||
import android.view.View
|
||||
import android.view.inputmethod.BaseInputConnection
|
||||
import android.view.inputmethod.EditorInfo
|
||||
|
||||
/**
|
||||
* P1 IME 通道（loop-2b-plan §5，用户已裁决）：**本地没有编辑缓冲，远端才是权威**。
|
||||
*
|
||||
* 三条约定（与方案一致）：
|
||||
* 1. `commitText` = 用户确认的整段文本 → 直接走协议 v3 `Text` 通道（布局无关、支持中文）；
|
||||
* 2. `sendKeyEvent` = 硬键盘/控制键 → 既有扫描码通道（Enter/Backspace/方向键…）；
|
||||
* 3. `deleteSurroundingText`：本地不知道远端内容，`before>0 && after==0`
|
||||
* 是 IME 退格最常见形态 → 发对应次数的 Backspace；其余形态记录后放弃（U3）。
|
||||
*
|
||||
* composing（拼音未确认串）不显示到远端（U4）：`setComposingText` 吞掉但返回 true，
|
||||
* 让 IME 维持本地组合状态，候选确认后仍会走 `commitText`。
|
||||
*
|
||||
* 日志纪律：debug 通道只报**长度与事件类型**，不报用户输入内容（安全红线）。
|
||||
*/
|
||||
class RemoteInputConnection(
|
||||
target: View,
|
||||
private val sendText: (CharArray) -> Unit,
|
||||
private val sendKey: (WinKey, Boolean) -> Unit,
|
||||
private val debug: (String) -> Unit,
|
||||
) : BaseInputConnection(target, false) {
|
||||
|
||||
override fun commitText(text: CharSequence, newCursorPosition: Int): Boolean {
|
||||
val units = text.toString().toCharArray()
|
||||
if (units.isEmpty()) return true
|
||||
sendText(units)
|
||||
debug("commit ${units.size}u")
|
||||
return true
|
||||
}
|
||||
|
||||
override fun sendKeyEvent(event: KeyEvent): Boolean {
|
||||
if (event.action == KeyEvent.ACTION_MULTIPLE) {
|
||||
val chars: String? = event.characters
|
||||
if (!chars.isNullOrEmpty()) {
|
||||
sendText(chars.toCharArray())
|
||||
debug("multiple ${chars.length}u")
|
||||
}
|
||||
return true
|
||||
}
|
||||
val key = WinScancode.of(event.keyCode)
|
||||
if (key == null) {
|
||||
debug("IME 未映射键 code=${event.keyCode}")
|
||||
return false
|
||||
}
|
||||
sendKey(key, event.action == KeyEvent.ACTION_DOWN)
|
||||
return true
|
||||
}
|
||||
|
||||
override fun deleteSurroundingText(beforeLength: Int, afterLength: Int): Boolean =
|
||||
deleteBackspaces(beforeLength, afterLength)
|
||||
|
||||
override fun deleteSurroundingTextInCodePoints(beforeLength: Int, afterLength: Int): Boolean =
|
||||
deleteBackspaces(beforeLength, afterLength)
|
||||
|
||||
private fun deleteBackspaces(before: Int, after: Int): Boolean {
|
||||
if (before == 0 && after == 0) return true
|
||||
if (after != 0) {
|
||||
// U3：forward-delete 与「先删 after 再删 before」的形态本地无远端真值可依
|
||||
debug("delete($before,$after)：after>0 形态未支持")
|
||||
return false
|
||||
}
|
||||
val bk = WinScancode.of(KeyEvent.KEYCODE_DEL) ?: return false
|
||||
repeat(before.coerceAtMost(MAX_BACKSPACE)) {
|
||||
sendKey(bk, true)
|
||||
sendKey(bk, false)
|
||||
}
|
||||
debug("backspace x$before")
|
||||
return true
|
||||
}
|
||||
|
||||
/** 吞掉 composing 但返回 true：IME 保持本地组合流程，候选确认仍会 commitText。 */
|
||||
override fun setComposingText(text: CharSequence, newCursorPosition: Int): Boolean = true
|
||||
|
||||
override fun setComposingRegion(start: Int, end: Int): Boolean = true
|
||||
|
||||
override fun finishComposingText(): Boolean = true
|
||||
|
||||
// 本地无可信缓冲区（远端才是权威），一律回空串——比 null 温和，多数 IME 兼容。
|
||||
override fun getTextBeforeCursor(n: Int, flags: Int): CharSequence = ""
|
||||
override fun getTextAfterCursor(n: Int, flags: Int): CharSequence = ""
|
||||
override fun getSelectedText(flags: Int): CharSequence = ""
|
||||
override fun getCursorCapsMode(reqModes: Int): Int = 0
|
||||
|
||||
override fun performEditorAction(actionCode: Int): Boolean {
|
||||
if (actionCode == EditorInfo.IME_ACTION_DONE) {
|
||||
// 软键盘「完成/回车」在远控语义里就是 Enter
|
||||
WinScancode.of(KeyEvent.KEYCODE_ENTER)?.let {
|
||||
sendKey(it, true)
|
||||
sendKey(it, false)
|
||||
}
|
||||
debug("IME_ACTION_DONE→Enter")
|
||||
return true
|
||||
}
|
||||
debug("editorAction $actionCode 忽略")
|
||||
return true
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** 单次退格请求上限：防 IME 异常大值刷爆消息队列（与 agent 侧 WHEEL_MAX_CLICKS 同思路） */
|
||||
private const val MAX_BACKSPACE = 64
|
||||
}
|
||||
}
|
||||
@ -1,13 +1,17 @@
|
||||
package com.verify.remote
|
||||
|
||||
import android.content.Context
|
||||
import android.text.InputType
|
||||
import android.view.MotionEvent
|
||||
import android.view.SurfaceView
|
||||
import android.view.inputmethod.EditorInfo
|
||||
import android.view.inputmethod.InputConnection
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.hypot
|
||||
|
||||
/**
|
||||
* 保持视频宽高比的 SurfaceView（letterbox），并把触摸手势映射为远端鼠标事件。
|
||||
* 保持视频宽高比的 SurfaceView（letterbox），把触摸手势映射为远端鼠标事件，
|
||||
* 并作为软键盘的宿主（协议 v3 文本通道 + 扫描码控制键）。
|
||||
*
|
||||
* 为什么视图尺寸必须等于画面尺寸：触摸坐标按视图宽高归一化到远端 0..=65535。
|
||||
* 若视图被拉伸（阶段 1 用 MATCH_PARENT），画面与视图不再一一对应，手指位置与
|
||||
@ -21,6 +25,11 @@ import kotlin.math.hypot
|
||||
* - 双指上下滑动 = 滚轮
|
||||
*
|
||||
* 刻意不做「手指即光标」的绝对定位：手指会完全挡住目标，桌面精细操作不可用。
|
||||
*
|
||||
* **IME 接线（阶段 2b）**：本视图是文本编辑器（`onCheckIsTextEditor`），
|
||||
* 软键盘的提交经 `RemoteInputConnection` 上行。`isFocusableInTouchMode` 必须为
|
||||
* true，否则触摸态拿不到焦点、系统不建立 InputConnection（阶段 2a 只有
|
||||
* `isFocusable`，这就是当时软键盘弹不出来的原因之一）。
|
||||
*/
|
||||
class RemoteSurfaceView(context: Context) : SurfaceView(context) {
|
||||
|
||||
@ -29,6 +38,12 @@ class RemoteSurfaceView(context: Context) : SurfaceView(context) {
|
||||
fun sendMouseMove(x: Int, y: Int)
|
||||
fun sendMouseButton(button: Int, down: Boolean)
|
||||
fun sendWheel(dy: Int)
|
||||
/** 协议 v3：一段 UTF-16 code unit（IME 提交结果） */
|
||||
fun sendText(units: CharArray)
|
||||
/** 控制键（Enter/Backspace/方向键等）走扫描码通道 */
|
||||
fun sendKey(key: WinKey, down: Boolean)
|
||||
/** 连接态：只有已连接才声明自己是文本编辑器，避免未连接时误弹键盘 */
|
||||
fun isReadyForInput(): Boolean
|
||||
}
|
||||
|
||||
var listener: Listener? = null
|
||||
@ -60,6 +75,32 @@ class RemoteSurfaceView(context: Context) : SurfaceView(context) {
|
||||
|
||||
init {
|
||||
isFocusable = true
|
||||
// IME 需要触摸态焦点；没有这行，点「键盘」按钮系统不会建立输入连接
|
||||
isFocusableInTouchMode = true
|
||||
}
|
||||
|
||||
// ---- IME 通道 ----
|
||||
|
||||
override fun onCheckIsTextEditor(): Boolean = listener?.isReadyForInput() == true
|
||||
|
||||
override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection? {
|
||||
val l = listener ?: return null
|
||||
if (!l.isReadyForInput()) return null
|
||||
// TYPE_CLASS_TEXT：让输入法给出正常键盘（含中文候选）。
|
||||
// 常量从 InputType 引用而非 EditorInfo：Kotlin 不保证经实现类名访问接口常量。
|
||||
// 刻意不加 TYPE_TEXT_FLAG_NO_SUGGESTIONS——中文输入依赖候选词。
|
||||
outAttrs.inputType = InputType.TYPE_CLASS_TEXT
|
||||
// 抑制全屏/extract 模式抢走画面（API 29 上两个标志都还在，具体效果真机验证，见 U4/U5）
|
||||
outAttrs.imeOptions =
|
||||
EditorInfo.IME_FLAG_NO_EXTRACT_UI or EditorInfo.IME_FLAG_NO_FULLSCREEN
|
||||
// 右下动作键文案。字段名是 actionLabel（javap android-34 核实，不是 imeActionLabel）
|
||||
outAttrs.actionLabel = "发送"
|
||||
return RemoteInputConnection(
|
||||
target = this,
|
||||
sendText = { l.sendText(it) },
|
||||
sendKey = { k, down -> l.sendKey(k, down) },
|
||||
debug = { msg -> onDebug?.invoke(msg) },
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -26,6 +26,12 @@ object RustBridge {
|
||||
external fun nativeSendMouseButton(handle: Long, button: Int, down: Int)
|
||||
external fun nativeSendWheel(handle: Long, dy: Int)
|
||||
external fun nativeSendKey(handle: Long, scancode: Int, extended: Int, down: Int)
|
||||
/**
|
||||
* 协议 v3 文本上行。参数刻意用 `CharArray`（UTF-16 code unit）而不是 `String`：
|
||||
* jni 侧 `get_string` 走 Modified-UTF-8 往返，IME 提交边界的落单代理项会在
|
||||
* 往返中丢失；`char[]` 经 `GetCharArrayRegion` 按 u16 原样拷贝。
|
||||
*/
|
||||
external fun nativeSendText(handle: Long, units: CharArray)
|
||||
external fun nativeRequestKeyframe(handle: Long)
|
||||
external fun nativeDestroy(handle: Long)
|
||||
|
||||
|
||||
@ -0,0 +1,126 @@
|
||||
package com.verify.remote
|
||||
|
||||
import android.view.KeyEvent
|
||||
|
||||
/**
|
||||
* Android KeyEvent.KEYCODE_* → Windows PS/2 Set-1 扫描码 + extended 标志。
|
||||
*
|
||||
* **扫描码值直接取自桌面 viewer 自己的映射（`viewer/src/keymap.rs`，项目内部事实）**，
|
||||
* 不是凭记忆写的；Android 侧一律用命名常量 `KeyEvent.KEYCODE_*` 引用，
|
||||
* 数值由编译器解析（已用 `javap -constants android.jar` 验证这些常量在 API 34 存在）。
|
||||
*
|
||||
* 分工（协议 v3 之后）：
|
||||
* - 可打印字符优先走 **Text 通道**（`KeyEvent.unicodeChar` / IME `commitText`），
|
||||
* 布局无关、中文可用；
|
||||
* - 本表服务控制键（Enter/Tab/Backspace/方向键/功能键）与物理键盘，
|
||||
* 走 `KeyDown/KeyUp` 扫描码通道。
|
||||
*
|
||||
* 不做 Shift/Ctrl 组合状态跟踪：软键盘的大写字母由 IME 直接 commit 字符，
|
||||
* 不需要本地修饰键语义。物理键盘组合键（Ctrl+C 等）登记为后续事项。
|
||||
*/
|
||||
data class WinKey(val scancode: Int, val extended: Boolean)
|
||||
|
||||
object WinScancode {
|
||||
|
||||
fun of(keyCode: Int): WinKey? = when (keyCode) {
|
||||
// 字母区（Set-1 make code，同 keymap.rs）
|
||||
KeyEvent.KEYCODE_A -> WinKey(0x1E, false)
|
||||
KeyEvent.KEYCODE_B -> WinKey(0x30, false)
|
||||
KeyEvent.KEYCODE_C -> WinKey(0x2E, false)
|
||||
KeyEvent.KEYCODE_D -> WinKey(0x20, false)
|
||||
KeyEvent.KEYCODE_E -> WinKey(0x12, false)
|
||||
KeyEvent.KEYCODE_F -> WinKey(0x21, false)
|
||||
KeyEvent.KEYCODE_G -> WinKey(0x22, false)
|
||||
KeyEvent.KEYCODE_H -> WinKey(0x23, false)
|
||||
KeyEvent.KEYCODE_I -> WinKey(0x17, false)
|
||||
KeyEvent.KEYCODE_J -> WinKey(0x24, false)
|
||||
KeyEvent.KEYCODE_K -> WinKey(0x25, false)
|
||||
KeyEvent.KEYCODE_L -> WinKey(0x26, false)
|
||||
KeyEvent.KEYCODE_M -> WinKey(0x32, false)
|
||||
KeyEvent.KEYCODE_N -> WinKey(0x31, false)
|
||||
KeyEvent.KEYCODE_O -> WinKey(0x18, false)
|
||||
KeyEvent.KEYCODE_P -> WinKey(0x19, false)
|
||||
KeyEvent.KEYCODE_Q -> WinKey(0x10, false)
|
||||
KeyEvent.KEYCODE_R -> WinKey(0x13, false)
|
||||
KeyEvent.KEYCODE_S -> WinKey(0x1F, false)
|
||||
KeyEvent.KEYCODE_T -> WinKey(0x14, false)
|
||||
KeyEvent.KEYCODE_U -> WinKey(0x16, false)
|
||||
KeyEvent.KEYCODE_V -> WinKey(0x2F, false)
|
||||
KeyEvent.KEYCODE_W -> WinKey(0x11, false)
|
||||
KeyEvent.KEYCODE_X -> WinKey(0x2D, false)
|
||||
KeyEvent.KEYCODE_Y -> WinKey(0x15, false)
|
||||
KeyEvent.KEYCODE_Z -> WinKey(0x2C, false)
|
||||
|
||||
// 数字主键区（1..9, 0；Android keycode 不连续，逐个写）
|
||||
KeyEvent.KEYCODE_1 -> WinKey(0x02, false)
|
||||
KeyEvent.KEYCODE_2 -> WinKey(0x03, false)
|
||||
KeyEvent.KEYCODE_3 -> WinKey(0x04, false)
|
||||
KeyEvent.KEYCODE_4 -> WinKey(0x05, false)
|
||||
KeyEvent.KEYCODE_5 -> WinKey(0x06, false)
|
||||
KeyEvent.KEYCODE_6 -> WinKey(0x07, false)
|
||||
KeyEvent.KEYCODE_7 -> WinKey(0x08, false)
|
||||
KeyEvent.KEYCODE_8 -> WinKey(0x09, false)
|
||||
KeyEvent.KEYCODE_9 -> WinKey(0x0A, false)
|
||||
KeyEvent.KEYCODE_0 -> WinKey(0x0B, false)
|
||||
|
||||
// 符号区（US 布局物理位置，同 keymap.rs）
|
||||
KeyEvent.KEYCODE_SPACE -> WinKey(0x39, false)
|
||||
KeyEvent.KEYCODE_ENTER -> WinKey(0x1C, false)
|
||||
KeyEvent.KEYCODE_DEL -> WinKey(0x0E, false) // Backspace
|
||||
KeyEvent.KEYCODE_FORWARD_DEL -> WinKey(0x53, true) // Delete（扩展）
|
||||
KeyEvent.KEYCODE_TAB -> WinKey(0x0F, false)
|
||||
KeyEvent.KEYCODE_ESCAPE -> WinKey(0x01, false)
|
||||
KeyEvent.KEYCODE_CAPS_LOCK -> WinKey(0x3A, false)
|
||||
KeyEvent.KEYCODE_GRAVE -> WinKey(0x29, false)
|
||||
KeyEvent.KEYCODE_MINUS -> WinKey(0x0C, false)
|
||||
KeyEvent.KEYCODE_EQUALS -> WinKey(0x0D, false)
|
||||
KeyEvent.KEYCODE_LEFT_BRACKET -> WinKey(0x1A, false)
|
||||
KeyEvent.KEYCODE_RIGHT_BRACKET -> WinKey(0x1B, false)
|
||||
KeyEvent.KEYCODE_BACKSLASH -> WinKey(0x2B, false)
|
||||
KeyEvent.KEYCODE_SEMICOLON -> WinKey(0x27, false)
|
||||
KeyEvent.KEYCODE_APOSTROPHE -> WinKey(0x28, false)
|
||||
KeyEvent.KEYCODE_COMMA -> WinKey(0x33, false)
|
||||
KeyEvent.KEYCODE_PERIOD -> WinKey(0x34, false)
|
||||
KeyEvent.KEYCODE_SLASH -> WinKey(0x35, false)
|
||||
|
||||
// 修饰键
|
||||
KeyEvent.KEYCODE_SHIFT_LEFT -> WinKey(0x2A, false)
|
||||
// 右 Shift **不是**扩展键：微软 Keyboard Input Overview 明确
|
||||
// "The right-hand Shift key is not considered an extended key either;
|
||||
// it has a separate scan code (0x36) instead."（本会话实读该页核实）
|
||||
KeyEvent.KEYCODE_SHIFT_RIGHT -> WinKey(0x36, false)
|
||||
KeyEvent.KEYCODE_CTRL_LEFT -> WinKey(0x1D, false)
|
||||
KeyEvent.KEYCODE_CTRL_RIGHT -> WinKey(0x1D, true)
|
||||
KeyEvent.KEYCODE_ALT_LEFT -> WinKey(0x38, false)
|
||||
KeyEvent.KEYCODE_ALT_RIGHT -> WinKey(0x38, true)
|
||||
KeyEvent.KEYCODE_META_LEFT -> WinKey(0x5B, true)
|
||||
KeyEvent.KEYCODE_META_RIGHT -> WinKey(0x5C, true)
|
||||
|
||||
// 导航区（全部扩展键）
|
||||
KeyEvent.KEYCODE_DPAD_UP -> WinKey(0x48, true)
|
||||
KeyEvent.KEYCODE_DPAD_DOWN -> WinKey(0x50, true)
|
||||
KeyEvent.KEYCODE_DPAD_LEFT -> WinKey(0x4B, true)
|
||||
KeyEvent.KEYCODE_DPAD_RIGHT -> WinKey(0x4D, true)
|
||||
KeyEvent.KEYCODE_MOVE_HOME -> WinKey(0x47, true)
|
||||
KeyEvent.KEYCODE_MOVE_END -> WinKey(0x4F, true)
|
||||
KeyEvent.KEYCODE_PAGE_UP -> WinKey(0x49, true)
|
||||
KeyEvent.KEYCODE_PAGE_DOWN -> WinKey(0x51, true)
|
||||
KeyEvent.KEYCODE_INSERT -> WinKey(0x52, true)
|
||||
|
||||
// 功能键（F11/F12 的 0x57/0x58 与 linux-agent keycode_of 注释同源）
|
||||
KeyEvent.KEYCODE_F1 -> WinKey(0x3B, false)
|
||||
KeyEvent.KEYCODE_F2 -> WinKey(0x3C, false)
|
||||
KeyEvent.KEYCODE_F3 -> WinKey(0x3D, false)
|
||||
KeyEvent.KEYCODE_F4 -> WinKey(0x3E, false)
|
||||
KeyEvent.KEYCODE_F5 -> WinKey(0x3F, false)
|
||||
KeyEvent.KEYCODE_F6 -> WinKey(0x40, false)
|
||||
KeyEvent.KEYCODE_F7 -> WinKey(0x41, false)
|
||||
KeyEvent.KEYCODE_F8 -> WinKey(0x42, false)
|
||||
KeyEvent.KEYCODE_F9 -> WinKey(0x43, false)
|
||||
KeyEvent.KEYCODE_F10 -> WinKey(0x44, false)
|
||||
KeyEvent.KEYCODE_F11 -> WinKey(0x57, false)
|
||||
KeyEvent.KEYCODE_F12 -> WinKey(0x58, false)
|
||||
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user