add all keyboard langs to backstage + fix test
This commit is contained in:
@@ -4,6 +4,8 @@
|
|||||||
# Force LF line endings for Go source files
|
# Force LF line endings for Go source files
|
||||||
*.go text eol=lf
|
*.go text eol=lf
|
||||||
|
|
||||||
|
Overlord-Server/src/generated/wire-contract.ts text eol=lf
|
||||||
|
|
||||||
# Force CRLF for Windows batch files
|
# Force CRLF for Windows batch files
|
||||||
*.bat text eol=crlf
|
*.bat text eol=crlf
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package capture
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestPrintableKeyText(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "latin", in: "é", want: "é"},
|
||||||
|
{name: "cyrillic", in: "ж", want: "ж"},
|
||||||
|
{name: "cjk", in: "你", want: "你"},
|
||||||
|
{name: "supplementary plane", in: "😀", want: "😀"},
|
||||||
|
{name: "named key", in: "Enter", want: ""},
|
||||||
|
{name: "control", in: "\n", want: ""},
|
||||||
|
{name: "empty", in: "", want: ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
tt := tt
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if got := printableKeyText(tt.in); got != tt.want {
|
||||||
|
t.Fatalf("printableKeyText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,7 +47,7 @@ func BackstageInputMouseUp(button int) error {
|
|||||||
return errors.New("backstage not supported on this platform")
|
return errors.New("backstage not supported on this platform")
|
||||||
}
|
}
|
||||||
|
|
||||||
func BackstageInputKeyDown(vk uint16) error {
|
func BackstageInputKeyDown(vk uint16, text string) error {
|
||||||
return errors.New("backstage not supported on this platform")
|
return errors.New("backstage not supported on this platform")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1425,13 +1425,15 @@ func uiaHandleMouseWheel(hwnd uintptr, screenPt point, delta int32) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
|
func uiaHandleKey(hwnd uintptr, vk uint16, text string, down bool) error {
|
||||||
uiaEnsureWorker()
|
uiaEnsureWorker()
|
||||||
|
|
||||||
inputSite := findInputSiteChild(hwnd)
|
inputSite := findInputSiteChild(hwnd)
|
||||||
if inputSite != 0 {
|
if inputSite != 0 {
|
||||||
if down {
|
if down {
|
||||||
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
|
if text != "" && !isNonPrintableVK(vk) {
|
||||||
|
postTextMessage(inputSite, text)
|
||||||
|
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
|
||||||
for _, r := range ch {
|
for _, r := range ch {
|
||||||
procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1))
|
procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1))
|
||||||
}
|
}
|
||||||
@@ -1445,6 +1447,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
|
|||||||
|
|
||||||
if down {
|
if down {
|
||||||
key := vk
|
key := vk
|
||||||
|
keyText := text
|
||||||
uiaRunAsync(func() {
|
uiaRunAsync(func() {
|
||||||
if uiaSingleton == nil {
|
if uiaSingleton == nil {
|
||||||
return
|
return
|
||||||
@@ -1452,7 +1455,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
|
|||||||
if !isModifierVK(key) && !isNonPrintableVK(key) {
|
if !isModifierVK(key) && !isNonPrintableVK(key) {
|
||||||
elem := uiaSingleton.GetFocusedElement()
|
elem := uiaSingleton.GetFocusedElement()
|
||||||
if elem != nil {
|
if elem != nil {
|
||||||
uiaTypeChar(elem, key)
|
uiaTypeChar(elem, key, keyText)
|
||||||
elem.Release()
|
elem.Release()
|
||||||
}
|
}
|
||||||
} else if key == 0x08 || key == 0x2E {
|
} else if key == 0x08 || key == 0x2E {
|
||||||
@@ -1543,7 +1546,7 @@ func uiaScrollElement(elem *iuiAutomationElement, delta int32) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
|
func uiaTypeChar(elem *iuiAutomationElement, vk uint16, text string) bool {
|
||||||
pat := elem.GetCurrentPattern(uiaValuePatternID)
|
pat := elem.GetCurrentPattern(uiaValuePatternID)
|
||||||
if pat == nil {
|
if pat == nil {
|
||||||
return false
|
return false
|
||||||
@@ -1551,13 +1554,16 @@ func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
|
|||||||
vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat))
|
vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat))
|
||||||
defer vp.Release()
|
defer vp.Release()
|
||||||
|
|
||||||
ch := virtualKeyToChars(vk)
|
if text == "" {
|
||||||
if len(ch) == 0 {
|
ch := virtualKeyToChars(vk)
|
||||||
return false
|
if len(ch) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
text = string(ch)
|
||||||
}
|
}
|
||||||
|
|
||||||
current := vp.GetCurrentValue()
|
current := vp.GetCurrentValue()
|
||||||
newVal := current + string(ch)
|
newVal := current + text
|
||||||
vp.SetValue(newVal)
|
vp.SetValue(newVal)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -223,6 +224,7 @@ type backstageTask struct {
|
|||||||
y int32
|
y int32
|
||||||
button int
|
button int
|
||||||
vk uint16
|
vk uint16
|
||||||
|
text string
|
||||||
delta int32
|
delta int32
|
||||||
dllBytes []byte
|
dllBytes []byte
|
||||||
captureDllBytes []byte
|
captureDllBytes []byte
|
||||||
@@ -510,9 +512,9 @@ func ensurebackstageThread() error {
|
|||||||
case backstageTaskMouseUp:
|
case backstageTaskMouseUp:
|
||||||
result.err = backstageMouseButtonOnThread(task.button, false)
|
result.err = backstageMouseButtonOnThread(task.button, false)
|
||||||
case backstageTaskKeyDown:
|
case backstageTaskKeyDown:
|
||||||
result.err = backstageKeyOnThread(task.vk, true)
|
result.err = backstageKeyOnThread(task.vk, task.text, true)
|
||||||
case backstageTaskKeyUp:
|
case backstageTaskKeyUp:
|
||||||
result.err = backstageKeyOnThread(task.vk, false)
|
result.err = backstageKeyOnThread(task.vk, "", false)
|
||||||
case backstageTaskMouseWheel:
|
case backstageTaskMouseWheel:
|
||||||
result.err = backstageMouseWheelOnThread(task.delta)
|
result.err = backstageMouseWheelOnThread(task.delta)
|
||||||
case backstageTaskAutoStartExplorer:
|
case backstageTaskAutoStartExplorer:
|
||||||
@@ -650,8 +652,8 @@ func BackstageInputMouseUp(button int) error {
|
|||||||
return result.err
|
return result.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func BackstageInputKeyDown(vk uint16) error {
|
func BackstageInputKeyDown(vk uint16, text string) error {
|
||||||
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk}, 3*time.Second)
|
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk, text: printableKeyText(text)}, 3*time.Second)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1229,7 +1231,7 @@ func backstageMouseButtonOnThread(button int, down bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func backstageKeyOnThread(vk uint16, down bool) error {
|
func backstageKeyOnThread(vk uint16, text string, down bool) error {
|
||||||
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
|
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
|
||||||
pt := currentbackstageCursor()
|
pt := currentbackstageCursor()
|
||||||
hwnd := windowFromPoint(pt)
|
hwnd := windowFromPoint(pt)
|
||||||
@@ -1260,7 +1262,7 @@ func backstageKeyOnThread(vk uint16, down bool) error {
|
|||||||
if isModifierVK(vk) {
|
if isModifierVK(vk) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return uiaHandleKey(hwnd, vk, down)
|
return uiaHandleKey(hwnd, vk, text, down)
|
||||||
}
|
}
|
||||||
|
|
||||||
if isModifierVK(vk) {
|
if isModifierVK(vk) {
|
||||||
@@ -1268,7 +1270,9 @@ func backstageKeyOnThread(vk uint16, down bool) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if down {
|
if down {
|
||||||
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
|
if text != "" && !isNonPrintableVK(vk) {
|
||||||
|
postTextMessage(hwnd, text)
|
||||||
|
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
|
||||||
for _, r := range ch {
|
for _, r := range ch {
|
||||||
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1))
|
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1))
|
||||||
}
|
}
|
||||||
@@ -1281,6 +1285,25 @@ func backstageKeyOnThread(vk uint16, down bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func printableKeyText(text string) string {
|
||||||
|
if !utf8.ValidString(text) || utf8.RuneCountInString(text) != 1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
r, _ := utf8.DecodeRuneInString(text)
|
||||||
|
if r < 0x20 || r == 0x7f {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
func postTextMessage(hwnd uintptr, text string) {
|
||||||
|
for _, unit := range syscall.StringToUTF16(text) {
|
||||||
|
if unit != 0 {
|
||||||
|
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(unit), uintptr(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func foregroundWindow() uintptr {
|
func foregroundWindow() uintptr {
|
||||||
r, _, _ := procGetForegroundWindow.Call()
|
r, _, _ := procGetForegroundWindow.Call()
|
||||||
return r
|
return r
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ type backstageInputEvent struct {
|
|||||||
button int
|
button int
|
||||||
delta int32
|
delta int32
|
||||||
vk uint16
|
vk uint16
|
||||||
|
text string
|
||||||
}
|
}
|
||||||
|
|
||||||
type voiceRuntime struct {
|
type voiceRuntime struct {
|
||||||
@@ -233,7 +234,7 @@ func ensurebackstageInputWorker() {
|
|||||||
log.Printf("backstage input worker: mouse_wheel failed: %v", err)
|
log.Printf("backstage input worker: mouse_wheel failed: %v", err)
|
||||||
}
|
}
|
||||||
case BackstageInputKeyDown:
|
case BackstageInputKeyDown:
|
||||||
if err := capture.BackstageInputKeyDown(ev.vk); err != nil {
|
if err := capture.BackstageInputKeyDown(ev.vk, ev.text); err != nil {
|
||||||
log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err)
|
log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err)
|
||||||
}
|
}
|
||||||
case BackstageInputKeyUp:
|
case BackstageInputKeyUp:
|
||||||
@@ -1736,10 +1737,14 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
|
|||||||
}
|
}
|
||||||
payload, _ := envelope["payload"].(map[string]interface{})
|
payload, _ := envelope["payload"].(map[string]interface{})
|
||||||
code := ""
|
code := ""
|
||||||
|
text := ""
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
if v, ok := payload["code"].(string); ok {
|
if v, ok := payload["code"].(string); ok {
|
||||||
code = v
|
code = v
|
||||||
}
|
}
|
||||||
|
if v, ok := payload["text"].(string); ok {
|
||||||
|
text = v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if vk := keyCodeToVKbackstage(code); vk != 0 {
|
if vk := keyCodeToVKbackstage(code); vk != 0 {
|
||||||
if env.VirtualCancel != nil {
|
if env.VirtualCancel != nil {
|
||||||
@@ -1750,7 +1755,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
|
|||||||
capture.VirtualInputKeyDown(vk)
|
capture.VirtualInputKeyDown(vk)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk})
|
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk, text: text})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
case "backstage_key_up":
|
case "backstage_key_up":
|
||||||
|
|||||||
@@ -1740,7 +1740,16 @@ import { createSharedUiSettingsSaver, loadSharedUiSettings } from "./shared-ui-s
|
|||||||
const kbdCapture = createKeyboardCapture({
|
const kbdCapture = createKeyboardCapture({
|
||||||
container: canvasContainer,
|
container: canvasContainer,
|
||||||
sendKeyDown: (e) => {
|
sendKeyDown: (e) => {
|
||||||
if (desiredStreaming) sendCmd("backstage_key_down", { key: e.key, code: e.code });
|
if (!desiredStreaming) return;
|
||||||
|
const altGraph = typeof e.getModifierState === "function" && e.getModifierState("AltGraph");
|
||||||
|
const printable = typeof e.key === "string"
|
||||||
|
&& Array.from(e.key).length === 1
|
||||||
|
&& (!e.ctrlKey && !e.metaKey && !e.altKey || altGraph);
|
||||||
|
sendCmd("backstage_key_down", {
|
||||||
|
key: e.key,
|
||||||
|
code: e.code,
|
||||||
|
text: printable ? e.key : "",
|
||||||
|
});
|
||||||
},
|
},
|
||||||
sendKeyUp: (e) => {
|
sendKeyUp: (e) => {
|
||||||
if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
|
if (desiredStreaming) sendCmd("backstage_key_up", { key: e.key, code: e.code });
|
||||||
|
|||||||
@@ -418,6 +418,34 @@ describe("remote desktop viewer control", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("backstage viewer control", () => {
|
describe("backstage viewer control", () => {
|
||||||
|
test("forwards browser-resolved Unicode text with backstage key presses", () => {
|
||||||
|
const clientId = `backstage-unicode-${Date.now().toString(36)}`;
|
||||||
|
const { agentWs } = createClient(clientId);
|
||||||
|
const viewer = createMockWs({ role: "backstage_viewer", clientId });
|
||||||
|
|
||||||
|
handlebackstageViewerOpen(viewer as any);
|
||||||
|
backstageStreamingState.set(clientId, {
|
||||||
|
isStreaming: true,
|
||||||
|
virtualMode: false,
|
||||||
|
display: 0,
|
||||||
|
quality: 90,
|
||||||
|
codec: "",
|
||||||
|
maxFps: 120,
|
||||||
|
lastFps: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
handlebackstageViewerMessage(viewer as any, JSON.stringify({
|
||||||
|
type: "backstage_key_down",
|
||||||
|
key: "ж",
|
||||||
|
code: "KeyS",
|
||||||
|
text: "ж",
|
||||||
|
}));
|
||||||
|
|
||||||
|
const command = agentCommands(agentWs)
|
||||||
|
.find((msg) => msg.commandType === "backstage_key_down");
|
||||||
|
expect(command?.payload).toEqual({ key: "ж", code: "KeyS", text: "ж" });
|
||||||
|
});
|
||||||
|
|
||||||
test("forwards backstage_stop even when server stream state is stale", () => {
|
test("forwards backstage_stop even when server stream state is stale", () => {
|
||||||
const clientId = `backstage-stale-stop-${Date.now().toString(36)}`;
|
const clientId = `backstage-stale-stop-${Date.now().toString(36)}`;
|
||||||
const { agentWs } = createClient(clientId);
|
const { agentWs } = createClient(clientId);
|
||||||
|
|||||||
@@ -1392,7 +1392,11 @@ export function handlebackstageViewerMessage(ws: ServerWebSocket<SocketData>, ra
|
|||||||
if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 });
|
if (state.isStreaming) sendbackstageCommand(target, "backstage_mouse_wheel", { delta: Number(payload.delta) || 0, x: Number(payload.x) || 0, y: Number(payload.y) || 0 });
|
||||||
break;
|
break;
|
||||||
case "backstage_key_down":
|
case "backstage_key_down":
|
||||||
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", { key: payload.key || "", code: payload.code || "" });
|
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_down", {
|
||||||
|
key: String(payload.key || ""),
|
||||||
|
code: String(payload.code || ""),
|
||||||
|
text: String(payload.text || ""),
|
||||||
|
});
|
||||||
break;
|
break;
|
||||||
case "backstage_key_up":
|
case "backstage_key_up":
|
||||||
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" });
|
if (state.isStreaming) sendbackstageCommand(target, "backstage_key_up", { key: payload.key || "", code: payload.code || "" });
|
||||||
|
|||||||
@@ -274,7 +274,8 @@ func IsServerToAgentMessageType(value string) bool {
|
|||||||
async function emit(path: string, content: string): Promise<void> {
|
async function emit(path: string, content: string): Promise<void> {
|
||||||
if (checkOnly) {
|
if (checkOnly) {
|
||||||
const current = await Bun.file(path).text().catch(() => "");
|
const current = await Bun.file(path).text().catch(() => "");
|
||||||
if (current !== content) {
|
const normalizeLineEndings = (value: string) => value.replace(/\r\n/g, "\n");
|
||||||
|
if (normalizeLineEndings(current) !== normalizeLineEndings(content)) {
|
||||||
console.error(`Generated artifact is stale: ${path}`);
|
console.error(`Generated artifact is stale: ${path}`);
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user