diff --git a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/MainActivity.kt b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/MainActivity.kt index af1c79a..cdb92ee 100644 --- a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/MainActivity.kt +++ b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/MainActivity.kt @@ -1,29 +1,50 @@ package com.verify.remote import android.app.Activity +import android.content.Context import android.os.Bundle +import android.text.InputType import android.util.Log +import android.view.Gravity import android.view.KeyEvent import android.view.SurfaceHolder +import android.view.View +import android.view.inputmethod.InputMethodManager +import android.widget.Button +import android.widget.EditText +import android.widget.FrameLayout +import android.widget.LinearLayout +import android.widget.LinearLayout.LayoutParams import android.widget.TextView import kotlin.concurrent.thread /** - * 安卓主控端（阶段 2a）：连接被控端 → MediaCodec 解码 → SurfaceView 显示 + 触摸操作。 + * 安卓主控端（阶段 2b）：正式连接界面 + 远控画面 + 触摸操作 + 软键盘文本输入。 + * + * 两态： + * - 连接态：host / port / code 表单 + 状态文本（阶段 1/2a 只能靠 `am start --es host` + * 传参，本界面对它补位；intent extras 仍保留且优先级更高，构建脚本继续可用）。 + * - 远控态：letterbox 画面 + 底部状态/操作条（帧数、RTT、诊断、键盘、断开）。 * * 线程模型（与 android-ffi 的锁纪律对应）： - * - UI 线程：触摸/按键 → nativeSend*（入队不阻塞）；连接/断开 + * - UI 线程：触摸/按键/IME 提交 → nativeSend*（入队不阻塞）；连接/断开 * - 解码线程：nativeNextFrame 阻塞取帧 → H264Decoder.queue * - 断开顺序：**先置 stopDecoding 并 join 解码线程，再 destroy** * （Rust 侧明确：解码线程存活期间 destroy 是 UB） * - * 布局：SurfaceView 按视频宽高比 letterbox 居中（RemoteSurfaceView）， - * 触摸坐标只在画面区域内产生，与远端归一化坐标一一对应。 + * 持久化：只存 host/port（连接成功后落盘）；**验证码不落盘**（2026-10-30 用户裁决）。 */ class MainActivity : Activity(), RemoteSurfaceView.Listener { private lateinit var surfaceView: RemoteSurfaceView private lateinit var statusView: TextView + private lateinit var connectPanel: LinearLayout + private lateinit var remotePanel: LinearLayout + private lateinit var hostInput: EditText + private lateinit var portInput: EditText + private lateinit var codeInput: EditText + private lateinit var connectButton: Button + private lateinit var connectStatus: TextView @Volatile private var handle = 0L @@ -32,12 +53,16 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { private var stopDecoding = false private var decodeThread: Thread? = null + /** 连接中标志：防重复点击起出第二个会话（句柄未回填前 handle 仍是 0） */ + @Volatile + private var connecting = false + @Volatile private var remoteW = 0 @Volatile private var remoteH = 0 - /** 最近一次触摸事件（诊断用：logcat 被限流，状态栏是唯一可靠通道） */ + /** 最近一次输入事件（触摸/IME）诊断：logcat 被限流，状态栏是唯一可靠通道 */ @Volatile private var lastTouchDbg = "-" @@ -52,18 +77,100 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { override fun surfaceCreated(holder: SurfaceHolder) {} override fun surfaceChanged(h: SurfaceHolder, f: Int, w: Int, ht: Int) {} override fun surfaceDestroyed(holder: SurfaceHolder) { - // Surface 没了解码器输出会失效：断开，重进 App 重连 + // Surface 没了解码器输出会失效：断开，回到连接界面重连 disconnect() } }) } + + val root = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setBackgroundColor(0xFF000000.toInt()) + } + connectPanel = buildConnectPanel() + remotePanel = buildRemotePanel() + val lp = LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f) + root.addView(connectPanel, lp) + root.addView(remotePanel, LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f)) + setContentView(root) + + showConnectState("未连接") + + // intent 入口优先级高于表单默认值：build-android.ps1 与调试脚本依赖它 + val hostExtra = intent?.getStringExtra("host") + if (!hostExtra.isNullOrEmpty()) { + hostInput.setText(hostExtra) + portInput.setText(intent.getIntExtra("port", 47800).toString()) + intent.getStringExtra("code")?.let { codeInput.setText(it) } + startConnect() + } + } + + // ---- 连接态 UI ---- + + private fun buildConnectPanel(): LinearLayout { + val prefs = getSharedPreferences(PREFS, Context.MODE_PRIVATE) + // 只回填 host/port（上次**成功连接**过的值）；验证码每次手输 + hostInput = editText( + "被控端 IP", + InputType.TYPE_CLASS_TEXT, + prefs.getString("host", "").orEmpty(), + ) + portInput = editText( + "端口", + InputType.TYPE_CLASS_NUMBER, + prefs.getInt("port", 47800).toString(), + ) + // 验证码刻意不从 prefs 读（不落盘） + codeInput = editText("验证码", InputType.TYPE_CLASS_TEXT, "") + connectButton = Button(this).apply { + text = "连接" + setOnClickListener { startConnect() } + } + connectStatus = TextView(this).apply { + textSize = 13f + setTextColor(0xFFBBBBBB.toInt()) + setPadding(0, dp(8), 0, 0) + } + val title = TextView(this).apply { + text = "连接被控端" + textSize = 18f + setTextColor(0xFFFFFFFF.toInt()) + setPadding(0, dp(16), 0, dp(8)) + } + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setPadding(dp(20), dp(24), dp(20), dp(24)) + addView(title) + addView(hostInput) + addView(portInput) + addView(codeInput) + addView(connectButton) + addView(connectStatus) + } + } + + private fun editText(hint: String, type: Int, initial: String) = EditText(this).apply { + inputType = type + this.hint = hint + setText(initial) + textSize = 16f + setTextColor(0xFFFFFFFF.toInt()) + setHintTextColor(0xFF777777.toInt()) + // 表单文字不能被画面抢占焦点 + isSingleLine = true + } + + // ---- 远控态 UI ---- + + private fun buildRemotePanel(): LinearLayout { statusView = TextView(this).apply { textSize = 12f - setPadding(20, 12, 20, 12) + setPadding(dp(12), dp(6), dp(12), dp(6)) setBackgroundColor(0xCC000000.toInt()) setTextColor(0xFFFFFFFF.toInt()) + maxLines = 3 } - // 画面居中、黑边填充。 // 用 FrameLayout + MATCH_PARENT（不是 WRAP_CONTENT）：FrameLayout 会把 // 子视图的 WRAP_CONTENT 当 MATCH_PARENT 处理，onMeasure 里拿到的 @@ -71,49 +178,69 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { // （2026-10-30 实测：视图被撑成 1080x2001，触摸按 2001 归一化 → 坐标全错）。 // MATCH_PARENT + gravity=center 时父容器传 AT_MOST，onMeasure 的 // setMeasuredDimension 才真正决定尺寸，再被 gravity 居中摆放。 - val stage = android.widget.FrameLayout(this).apply { + val stage = FrameLayout(this).apply { setBackgroundColor(0xFF000000.toInt()) addView( surfaceView, - android.widget.FrameLayout.LayoutParams( - android.widget.FrameLayout.LayoutParams.MATCH_PARENT, - android.widget.FrameLayout.LayoutParams.MATCH_PARENT, - android.view.Gravity.CENTER - ) + FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, + FrameLayout.LayoutParams.MATCH_PARENT, + Gravity.CENTER, + ), ) } - val root = android.widget.LinearLayout(this).apply { - orientation = android.widget.LinearLayout.VERTICAL - addView(stage, android.widget.LinearLayout.LayoutParams( - android.widget.LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f)) + val keyboardButton = Button(this).apply { + text = "键盘" + setOnClickListener { showKeyboard() } + } + val disconnectButton = Button(this).apply { + text = "断开" + setOnClickListener { disconnect() } + } + val bar = LinearLayout(this).apply { + orientation = LinearLayout.HORIZONTAL + addView(keyboardButton, LayoutParams(0, LayoutParams.WRAP_CONTENT, 1f)) + addView(disconnectButton, LayoutParams(0, LayoutParams.WRAP_CONTENT, 1f)) + } + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + visibility = View.GONE + addView(stage, LayoutParams(LayoutParams.MATCH_PARENT, 0, 1f)) // 固定高度而非 WRAP_CONTENT：与 weight=1 的兄弟共存时， // WRAP_CONTENT 可能被压成 0（实测 bounds=[0,0][0,0]，状态栏不可见） - addView(statusView, android.widget.LinearLayout.LayoutParams( - android.widget.LinearLayout.LayoutParams.MATCH_PARENT, - (48 * resources.displayMetrics.density).toInt())) - } - setContentView(root) - refreshStatus() - intent?.getStringExtra("host")?.let { host -> - connect(host, intent.getIntExtra("port", 47800), intent.getStringExtra("code") ?: "123456") + addView(statusView, LayoutParams(LayoutParams.MATCH_PARENT, dp(64))) + addView(bar, LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.WRAP_CONTENT)) } } - /** 状态栏统一刷新：连接态 + 帧数/RTT + 最近触摸诊断（logcat 被限流，这是唯一通道） */ - @Volatile - private var lastFrames = 0L - @Volatile - private var lastRtt = 0L + private fun showConnectState(msg: String) { + connecting = false + remotePanelVisible(false) + connectStatus.text = msg + } - private fun refreshStatus() { - if (handle == 0L) { - statusView.text = "未连接｜am start ... --es host --ei port 47800 --es code t" - return - } - statusView.text = buildString { - append("已连接 ${remoteW}x$remoteH 帧=$lastFrames RTT=${lastRtt}ms\n") - append("触摸: $lastTouchDbg") - } + private fun showRemoteState() { + remotePanelVisible(true) + connectPanel.visibility = View.GONE + } + + private fun remotePanelVisible(show: Boolean) { + connectPanel.visibility = if (show) View.GONE else View.VISIBLE + remotePanel.visibility = if (show) View.VISIBLE else View.GONE + } + + /** + * 显式唤起软键盘。不做自动弹：自动弹会挡画面，且 vivo ROM 行为不一致。 + * SurfaceView 已在触摸模式下可聚焦（见 RemoteSurfaceView.init）， + * 否则系统不会建立 InputConnection。 + */ + private fun showKeyboard() { + if (handle == 0L) return + surfaceView.requestFocus() + val imm = getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager + imm.showSoftInput(surfaceView, InputMethodManager.SHOW_IMPLICIT) + lastTouchDbg = "showSoftInput 已请求（焦点=${surfaceView.hasFocus()}）" + refreshStatus() } // ---- RemoteSurfaceView.Listener ---- @@ -133,19 +260,65 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { if (h != 0L) RustBridge.nativeSendWheel(h, dy) } - /** 返回键 = 断开（不退出 App，便于重连） */ + /** 协议 v3：IME 提交的整段文本（UTF-16 code unit，含落单代理项原样透传） */ + override fun sendText(units: CharArray) { + val h = handle + if (h != 0L) RustBridge.nativeSendText(h, units) + } + + override fun sendKey(key: WinKey, down: Boolean) { + val h = handle + if (h != 0L) { + RustBridge.nativeSendKey(h, key.scancode, if (key.extended) 1 else 0, if (down) 1 else 0) + } + } + + override fun isReadyForInput(): Boolean = handle != 0L + + /** + * 返回键：软键盘开着时先收键盘（否则一次返回同时关键盘和会话）； + * 远控态已连接 = 断开回连接界面；连接态 = 交还系统（退出 App）。 + */ override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { - if (keyCode == KeyEvent.KEYCODE_BACK && handle != 0L) { - disconnect() - return true + if (keyCode == KeyEvent.KEYCODE_BACK) { + val imm = getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager + // isAcceptingText = 当前有 IME 连接且正在接收输入（API 17+） + if (imm.isAcceptingText) { + imm.hideSoftInputFromWindow(surfaceView.windowToken, 0) + return true + } + if (handle != 0L) { + disconnect() + return true + } } return super.onKeyDown(keyCode, event) } // ---- 连接 / 解码 ---- - private fun connect(host: String, port: Int, code: String) { - if (handle != 0L) return + private fun startConnect() { + if (connecting || handle != 0L) return + val host = hostInput.text.toString().trim() + val code = codeInput.text.toString() + val port = portInput.text.toString().trim().toIntOrNull() + when { + host.isEmpty() -> { + connectStatus.text = "请填写被控端 IP" + return + } + port == null || port !in 1..65535 -> { + connectStatus.text = "端口必须是 1..65535" + return + } + code.isEmpty() -> { + connectStatus.text = "请填写验证码" + return + } + } + connecting = true + connectButton.isEnabled = false + showRemoteState() statusView.text = "连接 $host:$port …" thread(name = "connect") { try { @@ -156,7 +329,15 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { handle = h remoteW = w remoteH = ht + connecting = false runOnUiThread { + connectButton.isEnabled = true + // 只有认证成功的会话才值得记住；验证码不落盘（用户裁决） + getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .edit() + .putString("host", host) + .putInt("port", port) + .apply() if (w > 0 && ht > 0) { surfaceView.videoAspectRatio = w.toFloat() / ht } @@ -165,7 +346,12 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { } } catch (e: Exception) { Log.e(TAG, "connect failed", e) - runOnUiThread { statusView.text = "连接失败: ${e.message}" } + val msg = e.message ?: "连接失败" + runOnUiThread { + connectButton.isEnabled = true + handle = 0L + showConnectState("连接失败：$msg") + } } } } @@ -222,7 +408,7 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { lastFrames = 0 lastRtt = 0 RustBridge.destroy(h) - refreshStatus() + showConnectState("已断开") } override fun onDestroy() { @@ -230,7 +416,34 @@ class MainActivity : Activity(), RemoteSurfaceView.Listener { disconnect() } + /** 状态栏统一刷新：连接态 + 帧数/RTT + 最近输入诊断（logcat 被限流，这是唯一通道） */ + @Volatile + private var lastFrames = 0L + @Volatile + private var lastRtt = 0L + + private fun refreshStatus() { + if (handle == 0L) { + statusView.text = if (connecting) statusView.text else "未连接" + return + } + statusView.text = buildString { + append("已连接 v$PROTO_LABEL ${remoteW}x$remoteH 帧=$lastFrames RTT=${lastRtt}ms\n") + append("输入: $lastTouchDbg") + } + } + + private fun dp(v: Int) = (v * resources.displayMetrics.density).toInt() + companion object { private const val TAG = "MainActivity" + private const val PREFS = "connect" + + /** + * 与协议版本保持一致：远控画面状态栏显示当前握手版本（方案 R5 要求）。 + * 值随 `common::PROTO_VERSION` 人工同步——Android 侧不做线格式解析， + * 版本由 Rust 握手校验，这里只作显示与排障用。 + */ + private const val PROTO_LABEL = "3" } } diff --git a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteInputConnection.kt b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteInputConnection.kt new file mode 100644 index 0000000..8c0944f --- /dev/null +++ b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteInputConnection.kt @@ -0,0 +1,108 @@ +package com.verify.remote + +import android.view.KeyEvent +import android.view.View +import android.view.inputmethod.BaseInputConnection +import android.view.inputmethod.EditorInfo + +/** + * P1 IME 通道（loop-2b-plan §5，用户已裁决）：**本地没有编辑缓冲，远端才是权威**。 + * + * 三条约定（与方案一致）： + * 1. `commitText` = 用户确认的整段文本 → 直接走协议 v3 `Text` 通道（布局无关、支持中文）； + * 2. `sendKeyEvent` = 硬键盘/控制键 → 既有扫描码通道（Enter/Backspace/方向键…）； + * 3. `deleteSurroundingText`：本地不知道远端内容，`before>0 && after==0` + * 是 IME 退格最常见形态 → 发对应次数的 Backspace；其余形态记录后放弃（U3）。 + * + * composing（拼音未确认串）不显示到远端（U4）：`setComposingText` 吞掉但返回 true， + * 让 IME 维持本地组合状态，候选确认后仍会走 `commitText`。 + * + * 日志纪律：debug 通道只报**长度与事件类型**，不报用户输入内容（安全红线）。 + */ +class RemoteInputConnection( + target: View, + private val sendText: (CharArray) -> Unit, + private val sendKey: (WinKey, Boolean) -> Unit, + private val debug: (String) -> Unit, +) : BaseInputConnection(target, false) { + + override fun commitText(text: CharSequence, newCursorPosition: Int): Boolean { + val units = text.toString().toCharArray() + if (units.isEmpty()) return true + sendText(units) + debug("commit ${units.size}u") + return true + } + + override fun sendKeyEvent(event: KeyEvent): Boolean { + if (event.action == KeyEvent.ACTION_MULTIPLE) { + val chars: String? = event.characters + if (!chars.isNullOrEmpty()) { + sendText(chars.toCharArray()) + debug("multiple ${chars.length}u") + } + return true + } + val key = WinScancode.of(event.keyCode) + if (key == null) { + debug("IME 未映射键 code=${event.keyCode}") + return false + } + sendKey(key, event.action == KeyEvent.ACTION_DOWN) + return true + } + + override fun deleteSurroundingText(beforeLength: Int, afterLength: Int): Boolean = + deleteBackspaces(beforeLength, afterLength) + + override fun deleteSurroundingTextInCodePoints(beforeLength: Int, afterLength: Int): Boolean = + deleteBackspaces(beforeLength, afterLength) + + private fun deleteBackspaces(before: Int, after: Int): Boolean { + if (before == 0 && after == 0) return true + if (after != 0) { + // U3：forward-delete 与「先删 after 再删 before」的形态本地无远端真值可依 + debug("delete($before,$after)：after>0 形态未支持") + return false + } + val bk = WinScancode.of(KeyEvent.KEYCODE_DEL) ?: return false + repeat(before.coerceAtMost(MAX_BACKSPACE)) { + sendKey(bk, true) + sendKey(bk, false) + } + debug("backspace x$before") + return true + } + + /** 吞掉 composing 但返回 true：IME 保持本地组合流程，候选确认仍会 commitText。 */ + override fun setComposingText(text: CharSequence, newCursorPosition: Int): Boolean = true + + override fun setComposingRegion(start: Int, end: Int): Boolean = true + + override fun finishComposingText(): Boolean = true + + // 本地无可信缓冲区（远端才是权威），一律回空串——比 null 温和，多数 IME 兼容。 + override fun getTextBeforeCursor(n: Int, flags: Int): CharSequence = "" + override fun getTextAfterCursor(n: Int, flags: Int): CharSequence = "" + override fun getSelectedText(flags: Int): CharSequence = "" + override fun getCursorCapsMode(reqModes: Int): Int = 0 + + override fun performEditorAction(actionCode: Int): Boolean { + if (actionCode == EditorInfo.IME_ACTION_DONE) { + // 软键盘「完成/回车」在远控语义里就是 Enter + WinScancode.of(KeyEvent.KEYCODE_ENTER)?.let { + sendKey(it, true) + sendKey(it, false) + } + debug("IME_ACTION_DONE→Enter") + return true + } + debug("editorAction $actionCode 忽略") + return true + } + + companion object { + /** 单次退格请求上限：防 IME 异常大值刷爆消息队列（与 agent 侧 WHEEL_MAX_CLICKS 同思路） */ + private const val MAX_BACKSPACE = 64 + } +} diff --git a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteSurfaceView.kt b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteSurfaceView.kt index d8326e8..3a9a38d 100644 --- a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteSurfaceView.kt +++ b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RemoteSurfaceView.kt @@ -1,13 +1,17 @@ package com.verify.remote import android.content.Context +import android.text.InputType import android.view.MotionEvent import android.view.SurfaceView +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection import kotlin.math.abs import kotlin.math.hypot /** - * 保持视频宽高比的 SurfaceView（letterbox），并把触摸手势映射为远端鼠标事件。 + * 保持视频宽高比的 SurfaceView（letterbox），把触摸手势映射为远端鼠标事件， + * 并作为软键盘的宿主（协议 v3 文本通道 + 扫描码控制键）。 * * 为什么视图尺寸必须等于画面尺寸：触摸坐标按视图宽高归一化到远端 0..=65535。 * 若视图被拉伸（阶段 1 用 MATCH_PARENT），画面与视图不再一一对应，手指位置与 @@ -21,6 +25,11 @@ import kotlin.math.hypot * - 双指上下滑动 = 滚轮 * * 刻意不做「手指即光标」的绝对定位：手指会完全挡住目标，桌面精细操作不可用。 + * + * **IME 接线（阶段 2b）**：本视图是文本编辑器（`onCheckIsTextEditor`）， + * 软键盘的提交经 `RemoteInputConnection` 上行。`isFocusableInTouchMode` 必须为 + * true，否则触摸态拿不到焦点、系统不建立 InputConnection（阶段 2a 只有 + * `isFocusable`，这就是当时软键盘弹不出来的原因之一）。 */ class RemoteSurfaceView(context: Context) : SurfaceView(context) { @@ -29,6 +38,12 @@ class RemoteSurfaceView(context: Context) : SurfaceView(context) { fun sendMouseMove(x: Int, y: Int) fun sendMouseButton(button: Int, down: Boolean) fun sendWheel(dy: Int) + /** 协议 v3：一段 UTF-16 code unit（IME 提交结果） */ + fun sendText(units: CharArray) + /** 控制键（Enter/Backspace/方向键等）走扫描码通道 */ + fun sendKey(key: WinKey, down: Boolean) + /** 连接态：只有已连接才声明自己是文本编辑器，避免未连接时误弹键盘 */ + fun isReadyForInput(): Boolean } var listener: Listener? = null @@ -60,6 +75,32 @@ class RemoteSurfaceView(context: Context) : SurfaceView(context) { init { isFocusable = true + // IME 需要触摸态焦点；没有这行，点「键盘」按钮系统不会建立输入连接 + isFocusableInTouchMode = true + } + + // ---- IME 通道 ---- + + override fun onCheckIsTextEditor(): Boolean = listener?.isReadyForInput() == true + + override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection? { + val l = listener ?: return null + if (!l.isReadyForInput()) return null + // TYPE_CLASS_TEXT：让输入法给出正常键盘（含中文候选）。 + // 常量从 InputType 引用而非 EditorInfo：Kotlin 不保证经实现类名访问接口常量。 + // 刻意不加 TYPE_TEXT_FLAG_NO_SUGGESTIONS——中文输入依赖候选词。 + outAttrs.inputType = InputType.TYPE_CLASS_TEXT + // 抑制全屏/extract 模式抢走画面（API 29 上两个标志都还在，具体效果真机验证，见 U4/U5） + outAttrs.imeOptions = + EditorInfo.IME_FLAG_NO_EXTRACT_UI or EditorInfo.IME_FLAG_NO_FULLSCREEN + // 右下动作键文案。字段名是 actionLabel（javap android-34 核实，不是 imeActionLabel） + outAttrs.actionLabel = "发送" + return RemoteInputConnection( + target = this, + sendText = { l.sendText(it) }, + sendKey = { k, down -> l.sendKey(k, down) }, + debug = { msg -> onDebug?.invoke(msg) }, + ) } /** diff --git a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RustBridge.kt b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RustBridge.kt index 0cf2da5..3202fcf 100644 --- a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RustBridge.kt +++ b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/RustBridge.kt @@ -26,6 +26,12 @@ object RustBridge { external fun nativeSendMouseButton(handle: Long, button: Int, down: Int) external fun nativeSendWheel(handle: Long, dy: Int) external fun nativeSendKey(handle: Long, scancode: Int, extended: Int, down: Int) + /** + * 协议 v3 文本上行。参数刻意用 `CharArray`（UTF-16 code unit）而不是 `String`： + * jni 侧 `get_string` 走 Modified-UTF-8 往返，IME 提交边界的落单代理项会在 + * 往返中丢失；`char[]` 经 `GetCharArrayRegion` 按 u16 原样拷贝。 + */ + external fun nativeSendText(handle: Long, units: CharArray) external fun nativeRequestKeyframe(handle: Long) external fun nativeDestroy(handle: Long) diff --git a/apps/android-viewer/app/src/main/kotlin/com/verify/remote/WinScancode.kt b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/WinScancode.kt new file mode 100644 index 0000000..496fa0f --- /dev/null +++ b/apps/android-viewer/app/src/main/kotlin/com/verify/remote/WinScancode.kt @@ -0,0 +1,126 @@ +package com.verify.remote + +import android.view.KeyEvent + +/** + * Android KeyEvent.KEYCODE_* → Windows PS/2 Set-1 扫描码 + extended 标志。 + * + * **扫描码值直接取自桌面 viewer 自己的映射（`viewer/src/keymap.rs`，项目内部事实）**， + * 不是凭记忆写的；Android 侧一律用命名常量 `KeyEvent.KEYCODE_*` 引用， + * 数值由编译器解析（已用 `javap -constants android.jar` 验证这些常量在 API 34 存在）。 + * + * 分工（协议 v3 之后）： + * - 可打印字符优先走 **Text 通道**（`KeyEvent.unicodeChar` / IME `commitText`）， + * 布局无关、中文可用； + * - 本表服务控制键（Enter/Tab/Backspace/方向键/功能键）与物理键盘， + * 走 `KeyDown/KeyUp` 扫描码通道。 + * + * 不做 Shift/Ctrl 组合状态跟踪：软键盘的大写字母由 IME 直接 commit 字符， + * 不需要本地修饰键语义。物理键盘组合键（Ctrl+C 等）登记为后续事项。 + */ +data class WinKey(val scancode: Int, val extended: Boolean) + +object WinScancode { + + fun of(keyCode: Int): WinKey? = when (keyCode) { + // 字母区（Set-1 make code，同 keymap.rs） + KeyEvent.KEYCODE_A -> WinKey(0x1E, false) + KeyEvent.KEYCODE_B -> WinKey(0x30, false) + KeyEvent.KEYCODE_C -> WinKey(0x2E, false) + KeyEvent.KEYCODE_D -> WinKey(0x20, false) + KeyEvent.KEYCODE_E -> WinKey(0x12, false) + KeyEvent.KEYCODE_F -> WinKey(0x21, false) + KeyEvent.KEYCODE_G -> WinKey(0x22, false) + KeyEvent.KEYCODE_H -> WinKey(0x23, false) + KeyEvent.KEYCODE_I -> WinKey(0x17, false) + KeyEvent.KEYCODE_J -> WinKey(0x24, false) + KeyEvent.KEYCODE_K -> WinKey(0x25, false) + KeyEvent.KEYCODE_L -> WinKey(0x26, false) + KeyEvent.KEYCODE_M -> WinKey(0x32, false) + KeyEvent.KEYCODE_N -> WinKey(0x31, false) + KeyEvent.KEYCODE_O -> WinKey(0x18, false) + KeyEvent.KEYCODE_P -> WinKey(0x19, false) + KeyEvent.KEYCODE_Q -> WinKey(0x10, false) + KeyEvent.KEYCODE_R -> WinKey(0x13, false) + KeyEvent.KEYCODE_S -> WinKey(0x1F, false) + KeyEvent.KEYCODE_T -> WinKey(0x14, false) + KeyEvent.KEYCODE_U -> WinKey(0x16, false) + KeyEvent.KEYCODE_V -> WinKey(0x2F, false) + KeyEvent.KEYCODE_W -> WinKey(0x11, false) + KeyEvent.KEYCODE_X -> WinKey(0x2D, false) + KeyEvent.KEYCODE_Y -> WinKey(0x15, false) + KeyEvent.KEYCODE_Z -> WinKey(0x2C, false) + + // 数字主键区（1..9, 0；Android keycode 不连续，逐个写） + KeyEvent.KEYCODE_1 -> WinKey(0x02, false) + KeyEvent.KEYCODE_2 -> WinKey(0x03, false) + KeyEvent.KEYCODE_3 -> WinKey(0x04, false) + KeyEvent.KEYCODE_4 -> WinKey(0x05, false) + KeyEvent.KEYCODE_5 -> WinKey(0x06, false) + KeyEvent.KEYCODE_6 -> WinKey(0x07, false) + KeyEvent.KEYCODE_7 -> WinKey(0x08, false) + KeyEvent.KEYCODE_8 -> WinKey(0x09, false) + KeyEvent.KEYCODE_9 -> WinKey(0x0A, false) + KeyEvent.KEYCODE_0 -> WinKey(0x0B, false) + + // 符号区（US 布局物理位置，同 keymap.rs） + KeyEvent.KEYCODE_SPACE -> WinKey(0x39, false) + KeyEvent.KEYCODE_ENTER -> WinKey(0x1C, false) + KeyEvent.KEYCODE_DEL -> WinKey(0x0E, false) // Backspace + KeyEvent.KEYCODE_FORWARD_DEL -> WinKey(0x53, true) // Delete（扩展） + KeyEvent.KEYCODE_TAB -> WinKey(0x0F, false) + KeyEvent.KEYCODE_ESCAPE -> WinKey(0x01, false) + KeyEvent.KEYCODE_CAPS_LOCK -> WinKey(0x3A, false) + KeyEvent.KEYCODE_GRAVE -> WinKey(0x29, false) + KeyEvent.KEYCODE_MINUS -> WinKey(0x0C, false) + KeyEvent.KEYCODE_EQUALS -> WinKey(0x0D, false) + KeyEvent.KEYCODE_LEFT_BRACKET -> WinKey(0x1A, false) + KeyEvent.KEYCODE_RIGHT_BRACKET -> WinKey(0x1B, false) + KeyEvent.KEYCODE_BACKSLASH -> WinKey(0x2B, false) + KeyEvent.KEYCODE_SEMICOLON -> WinKey(0x27, false) + KeyEvent.KEYCODE_APOSTROPHE -> WinKey(0x28, false) + KeyEvent.KEYCODE_COMMA -> WinKey(0x33, false) + KeyEvent.KEYCODE_PERIOD -> WinKey(0x34, false) + KeyEvent.KEYCODE_SLASH -> WinKey(0x35, false) + + // 修饰键 + KeyEvent.KEYCODE_SHIFT_LEFT -> WinKey(0x2A, false) + // 右 Shift **不是**扩展键：微软 Keyboard Input Overview 明确 + // "The right-hand Shift key is not considered an extended key either; + // it has a separate scan code (0x36) instead."（本会话实读该页核实） + KeyEvent.KEYCODE_SHIFT_RIGHT -> WinKey(0x36, false) + KeyEvent.KEYCODE_CTRL_LEFT -> WinKey(0x1D, false) + KeyEvent.KEYCODE_CTRL_RIGHT -> WinKey(0x1D, true) + KeyEvent.KEYCODE_ALT_LEFT -> WinKey(0x38, false) + KeyEvent.KEYCODE_ALT_RIGHT -> WinKey(0x38, true) + KeyEvent.KEYCODE_META_LEFT -> WinKey(0x5B, true) + KeyEvent.KEYCODE_META_RIGHT -> WinKey(0x5C, true) + + // 导航区（全部扩展键） + KeyEvent.KEYCODE_DPAD_UP -> WinKey(0x48, true) + KeyEvent.KEYCODE_DPAD_DOWN -> WinKey(0x50, true) + KeyEvent.KEYCODE_DPAD_LEFT -> WinKey(0x4B, true) + KeyEvent.KEYCODE_DPAD_RIGHT -> WinKey(0x4D, true) + KeyEvent.KEYCODE_MOVE_HOME -> WinKey(0x47, true) + KeyEvent.KEYCODE_MOVE_END -> WinKey(0x4F, true) + KeyEvent.KEYCODE_PAGE_UP -> WinKey(0x49, true) + KeyEvent.KEYCODE_PAGE_DOWN -> WinKey(0x51, true) + KeyEvent.KEYCODE_INSERT -> WinKey(0x52, true) + + // 功能键（F11/F12 的 0x57/0x58 与 linux-agent keycode_of 注释同源） + KeyEvent.KEYCODE_F1 -> WinKey(0x3B, false) + KeyEvent.KEYCODE_F2 -> WinKey(0x3C, false) + KeyEvent.KEYCODE_F3 -> WinKey(0x3D, false) + KeyEvent.KEYCODE_F4 -> WinKey(0x3E, false) + KeyEvent.KEYCODE_F5 -> WinKey(0x3F, false) + KeyEvent.KEYCODE_F6 -> WinKey(0x40, false) + KeyEvent.KEYCODE_F7 -> WinKey(0x41, false) + KeyEvent.KEYCODE_F8 -> WinKey(0x42, false) + KeyEvent.KEYCODE_F9 -> WinKey(0x43, false) + KeyEvent.KEYCODE_F10 -> WinKey(0x44, false) + KeyEvent.KEYCODE_F11 -> WinKey(0x57, false) + KeyEvent.KEYCODE_F12 -> WinKey(0x58, false) + + else -> null + } +}