test: verify Playground scenarios through shared GUI runner
This commit is contained in:
Kinneyzhang
parent
7c6eb45f4f
commit
b8579e57cc
30
Makefile
30
Makefile
@ -1,5 +1,5 @@
|
|||||||
EMACS ?= emacs
|
EMACS ?= emacs
|
||||||
LOAD_PATH = -L . -L ../etaf -L ../etaf-ui -L ../ebox -L ../ecss -L ../tp -L ../etaf-sqlite
|
LOAD_PATH = -L . -L scripts -L ../etaf -L ../etaf-ui -L ../ebox -L ../ecss -L ../tp -L ../etaf-sqlite
|
||||||
EBOX_NATIVE_RELEASE_DIR = $(shell $(EMACS) -Q --batch $(LOAD_PATH) \
|
EBOX_NATIVE_RELEASE_DIR = $(shell $(EMACS) -Q --batch $(LOAD_PATH) \
|
||||||
--eval '(setq load-prefer-newer t)' -l ../ebox/ebox-native-reflow.el \
|
--eval '(setq load-prefer-newer t)' -l ../ebox/ebox-native-reflow.el \
|
||||||
--eval '(princ (expand-file-name (format "../ebox/native/target/%s/release/" (ebox-native-reflow--rust-target))))')
|
--eval '(princ (expand-file-name (format "../ebox/native/target/%s/release/" (ebox-native-reflow--rust-target))))')
|
||||||
@ -9,7 +9,8 @@ EXAMPLE_ELC := $(EXAMPLES:.el=.elc)
|
|||||||
TEST_FILES := $(wildcard tests/*-tests.el)
|
TEST_FILES := $(wildcard tests/*-tests.el)
|
||||||
|
|
||||||
.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-regressions \
|
.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-regressions \
|
||||||
perf-check check checkdoc load clean
|
perf-check gui-doctor gui-research gui-flex gui-grid gui-all \
|
||||||
|
check checkdoc load clean
|
||||||
|
|
||||||
all: check
|
all: check
|
||||||
|
|
||||||
@ -23,12 +24,14 @@ compile:
|
|||||||
$(MAKE) -C ../etaf compile EMACS="$(EMACS)"
|
$(MAKE) -C ../etaf compile EMACS="$(EMACS)"
|
||||||
$(MAKE) -C ../etaf-ui compile EMACS="$(EMACS)"
|
$(MAKE) -C ../etaf-ui compile EMACS="$(EMACS)"
|
||||||
$(MAKE) -C ../etaf-sqlite compile EMACS="$(EMACS)"
|
$(MAKE) -C ../etaf-sqlite compile EMACS="$(EMACS)"
|
||||||
rm -f *.elc tests/*.elc $(EXAMPLE_ELC)
|
rm -f *.elc scripts/*.elc tests/*.elc $(EXAMPLE_ELC)
|
||||||
$(EMACS) -Q --batch $(LOAD_PATH) --eval '(setq load-prefer-newer t)' \
|
$(EMACS) -Q --batch $(LOAD_PATH) -L ../etaf/scripts -L ../ebox-playground \
|
||||||
|
--eval "(setq load-prefer-newer t byte-compile-error-on-warn t byte-compile-warnings '(not obsolete))" \
|
||||||
--eval '(load-file "etaf-playground.el")' \
|
--eval '(load-file "etaf-playground.el")' \
|
||||||
--eval '(dolist (file (quote ($(foreach file,$(ENTRY_EXAMPLES),"$(file)")))) (load-file file))' \
|
--eval '(dolist (file (quote ($(foreach file,$(ENTRY_EXAMPLES),"$(file)")))) (load-file file))' \
|
||||||
--eval '(dolist (file (quote ($(foreach file,$(EXAMPLES),"$(file)")))) (byte-compile-file file))' \
|
--eval '(dolist (file (quote ($(foreach file,$(EXAMPLES),"$(file)")))) (byte-compile-file file))' \
|
||||||
--eval '(byte-compile-file "etaf-playground.el")' \
|
--eval '(byte-compile-file "etaf-playground.el")' \
|
||||||
|
--eval '(byte-compile-file "scripts/playground-gui-scenarios.el")' \
|
||||||
--eval '(dolist (file (quote ($(foreach file,$(TEST_FILES),"$(file)")))) (byte-compile-file file))'
|
--eval '(dolist (file (quote ($(foreach file,$(TEST_FILES),"$(file)")))) (byte-compile-file file))'
|
||||||
|
|
||||||
examples-read: compile
|
examples-read: compile
|
||||||
@ -76,14 +79,29 @@ perf-check:
|
|||||||
$(MAKE) perf EMACS="$(EMACS)"
|
$(MAKE) perf EMACS="$(EMACS)"
|
||||||
$(MAKE) perf-regressions EMACS="$(EMACS)"
|
$(MAKE) perf-regressions EMACS="$(EMACS)"
|
||||||
|
|
||||||
|
gui-doctor:
|
||||||
|
scripts/run-gui-verification.sh doctor
|
||||||
|
|
||||||
|
gui-research:
|
||||||
|
scripts/run-gui-verification.sh research-shelf
|
||||||
|
|
||||||
|
gui-flex:
|
||||||
|
scripts/run-gui-verification.sh flex-reference
|
||||||
|
|
||||||
|
gui-grid:
|
||||||
|
scripts/run-gui-verification.sh grid-reference
|
||||||
|
|
||||||
|
gui-all:
|
||||||
|
scripts/run-gui-verification.sh all
|
||||||
|
|
||||||
load: compile
|
load: compile
|
||||||
$(EMACS) -Q --batch $(LOAD_PATH) --eval '(require (quote etaf-playground))' \
|
$(EMACS) -Q --batch $(LOAD_PATH) --eval '(require (quote etaf-playground))' \
|
||||||
--eval '(princ "etaf-playground load OK\\n")'
|
--eval '(princ "etaf-playground load OK\\n")'
|
||||||
|
|
||||||
checkdoc:
|
checkdoc:
|
||||||
$(EMACS) -Q --batch --eval '(progn (require (quote checkdoc)) (dolist (file (append (list "etaf-playground.el" "scripts/benchmark-research-shelf.el") (directory-files-recursively "tests" "\\.el$$") (directory-files-recursively "examples" "\\.el$$"))) (checkdoc-file file)))'
|
$(EMACS) -Q --batch --eval '(progn (require (quote checkdoc)) (dolist (file (append (list "etaf-playground.el" "scripts/benchmark-research-shelf.el" "scripts/playground-gui-scenarios.el") (directory-files-recursively "tests" "\\.el$$") (directory-files-recursively "examples" "\\.el$$"))) (checkdoc-file file)))'
|
||||||
|
|
||||||
check: checkdoc test
|
check: checkdoc test
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -f *.elc tests/*.elc $(EXAMPLE_ELC)
|
rm -f *.elc scripts/*.elc tests/*.elc $(EXAMPLE_ELC)
|
||||||
|
|||||||
41
README.md
41
README.md
@ -60,6 +60,47 @@ The companion registers `:reload-on-refresh t`, so saving the `.el`, `.etaf`, or
|
|||||||
`.ecss` source and refreshing reloads the complete consumer before the next
|
`.ecss` source and refreshing reloads the complete consumer before the next
|
||||||
mount.
|
mount.
|
||||||
|
|
||||||
|
## Repeatable Emacs 31.1 GUI verification
|
||||||
|
|
||||||
|
The repository owns one stable real-GUI runner instead of relying on ad-hoc
|
||||||
|
`emacsclient`, focus, and recording commands:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make gui-doctor
|
||||||
|
make gui-research
|
||||||
|
make gui-flex
|
||||||
|
make gui-grid
|
||||||
|
# Or capture all three sequentially:
|
||||||
|
make gui-all
|
||||||
|
```
|
||||||
|
|
||||||
|
The reusable engine and process runner live in `../etaf/scripts/`. They know
|
||||||
|
only Scenario, Action, Context, checkpoint, recording, and evidence contracts.
|
||||||
|
`scripts/playground-gui-scenarios.el` is a thin adapter layer: Research Shelf
|
||||||
|
defines its application actions, while Flex and Grid are two inputs to the same
|
||||||
|
Ebox reference scenario factory. Adding another application does not create a
|
||||||
|
second daemon/recording/checkpoint implementation.
|
||||||
|
|
||||||
|
Each scenario creates a unique named daemon, disables native-comp JIT, loads
|
||||||
|
the sibling repositories explicitly, creates one GUI frame, activates Emacs
|
||||||
|
from the controlling shell, records through a PTY-backed macOS screen recorder,
|
||||||
|
runs mount/resize/scroll/interaction checkpoints, writes screenshots and a
|
||||||
|
manifest, then shuts down every process it created. Flex is always extracted
|
||||||
|
from the committed `ebox-playground` HEAD, so a user's dirty fixture is never
|
||||||
|
read, overwritten, staged, or restored.
|
||||||
|
|
||||||
|
A fresh capture intentionally reports `INCOMPLETE` until a human or agent has
|
||||||
|
inspected `report.md`, `contact-sheet.png`, and its selected endpoint images.
|
||||||
|
After that temporal review, finalize the exact run directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
scripts/run-gui-verification.sh review /private/tmp/etaf-playground-gui.XXXXXX
|
||||||
|
```
|
||||||
|
|
||||||
|
Only `VERDICT=PASS` is valid GUI evidence. Assertion failure, a black segment,
|
||||||
|
missing recording, wrong buffer, split window, stale frame, or missing temporal
|
||||||
|
review remains fail-closed.
|
||||||
|
|
||||||
Run `make check EMACS=/Applications/Emacs.app/Contents/MacOS/Emacs`.
|
Run `make check EMACS=/Applications/Emacs.app/Contents/MacOS/Emacs`.
|
||||||
Run `make perf` for the fixed 1413×62 latency gate. Every scenario performs
|
Run `make perf` for the fixed 1413×62 latency gate. Every scenario performs
|
||||||
five unmeasured warmups followed by 30 measured samples; both p95 and max must
|
five unmeasured warmups followed by 30 measured samples; both p95 and max must
|
||||||
|
|||||||
@ -58,6 +58,43 @@ examples/research-shelf.el # DATA / THEME / STATE / VIEW / ROOT 分区
|
|||||||
该 companion 注册了 `:reload-on-refresh t`；保存 `.el`、`.etaf` 或 `.ecss` 后刷新
|
该 companion 注册了 `:reload-on-refresh t`；保存 `.el`、`.etaf` 或 `.ecss` 后刷新
|
||||||
source，会在下一次 mount 前重新加载完整 consumer。
|
source，会在下一次 mount 前重新加载完整 consumer。
|
||||||
|
|
||||||
|
## 可重复执行的 Emacs 31.1 GUI 实测
|
||||||
|
|
||||||
|
仓库提供一个稳定的真实 GUI runner，不再临时拼接 `emacsclient`、前台激活和录屏
|
||||||
|
命令：
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make gui-doctor
|
||||||
|
make gui-research
|
||||||
|
make gui-flex
|
||||||
|
make gui-grid
|
||||||
|
# 或按顺序采集三个场景：
|
||||||
|
make gui-all
|
||||||
|
```
|
||||||
|
|
||||||
|
可复用的执行引擎与进程 runner 位于 `../etaf/scripts/`，只理解 Scenario、Action、
|
||||||
|
Context、checkpoint、录屏和 evidence 合同。`scripts/playground-gui-scenarios.el` 只是
|
||||||
|
薄适配层：Research Shelf 提供应用动作，Flex 与 Grid 只是同一个 Ebox reference
|
||||||
|
scenario factory 的两个输入。增加新应用时不会复制第二套 daemon/录屏/checkpoint
|
||||||
|
实现。
|
||||||
|
|
||||||
|
每个场景都会创建唯一命名 daemon，关闭 native-comp JIT，显式加载 sibling 仓库，
|
||||||
|
创建单一 GUI frame，由外层 shell 激活 Emacs，通过保留 PTY 的 macOS recorder 录屏，
|
||||||
|
依次执行 mount/resize/scroll/interaction checkpoint，生成 screenshot 与 manifest，最后
|
||||||
|
清理自己创建的全部进程。Flex 始终从已提交的 `ebox-playground` HEAD 解包，绝不读取、
|
||||||
|
覆盖、暂存或 restore 用户的脏 fixture。
|
||||||
|
|
||||||
|
新采集有意保持 `INCOMPLETE`，直到人或 agent 检查 `report.md`、
|
||||||
|
`contact-sheet.png` 和报告选出的首尾图。完成时序审查后，对同一个 run directory
|
||||||
|
执行：
|
||||||
|
|
||||||
|
```sh
|
||||||
|
scripts/run-gui-verification.sh review /private/tmp/etaf-playground-gui.XXXXXX
|
||||||
|
```
|
||||||
|
|
||||||
|
只有 `VERDICT=PASS` 才是有效 GUI 证据。assertion 失败、黑帧、录屏缺失、错误 buffer、
|
||||||
|
split window、陈旧 frame 或没有完成时序审查都会保持 fail-closed。
|
||||||
|
|
||||||
验证命令：`make check EMACS=/Applications/Emacs.app/Contents/MacOS/Emacs`。
|
验证命令：`make check EMACS=/Applications/Emacs.app/Contents/MacOS/Emacs`。
|
||||||
性能门禁：`make perf`（1413×62 viewport 的 warm 选行/主题延迟）。
|
性能门禁：`make perf`（1413×62 viewport 的 warm 选行/主题延迟）。
|
||||||
|
|
||||||
|
|||||||
446
scripts/playground-gui-scenarios.el
Normal file
446
scripts/playground-gui-scenarios.el
Normal file
@ -0,0 +1,446 @@
|
|||||||
|
;;; playground-gui-scenarios.el --- Playground GUI verifier adapters -*- lexical-binding: t; -*-
|
||||||
|
|
||||||
|
;;; Commentary:
|
||||||
|
|
||||||
|
;; Concrete Research Shelf and Ebox reference adapters for the generic
|
||||||
|
;; `emacs-gui-verifier' engine. Process, recording, checkpoint ordering, and
|
||||||
|
;; evidence lifecycle remain outside this file.
|
||||||
|
|
||||||
|
;;; Code:
|
||||||
|
|
||||||
|
(require 'emacs-gui-verifier)
|
||||||
|
(require 'ebox-playground)
|
||||||
|
(require 'etaf-playground)
|
||||||
|
(require 'benchmark-research-shelf)
|
||||||
|
(require 'ebox-native-reflow)
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--report (context)
|
||||||
|
"Return CONTEXT target's public Ebox report, or nil before mount."
|
||||||
|
(let ((buffer (etaf-gui-verifier-context-target-buffer context)))
|
||||||
|
(when (and (buffer-live-p buffer)
|
||||||
|
(ebox-surface-buffer-mounted-p buffer))
|
||||||
|
(ebox-buffer-update-report buffer))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--visible-window-content-p (context)
|
||||||
|
"Return non-nil when CONTEXT's selected window shows nonblank text."
|
||||||
|
(let* ((buffer (etaf-gui-verifier-context-target-buffer context))
|
||||||
|
(window (selected-window)))
|
||||||
|
(and (buffer-live-p buffer)
|
||||||
|
(window-live-p window)
|
||||||
|
(eq (window-buffer window) buffer)
|
||||||
|
(let ((start (window-start window))
|
||||||
|
(end (window-end window t)))
|
||||||
|
(and (integer-or-marker-p start)
|
||||||
|
(integer-or-marker-p end)
|
||||||
|
(< start end)
|
||||||
|
(with-current-buffer buffer
|
||||||
|
(string-match-p
|
||||||
|
"[^[:space:]]"
|
||||||
|
(buffer-substring-no-properties start end))))))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--buffer-text (context)
|
||||||
|
"Return CONTEXT target's complete plain buffer text."
|
||||||
|
(let ((buffer (etaf-gui-verifier-context-target-buffer context)))
|
||||||
|
(and (buffer-live-p buffer)
|
||||||
|
(with-current-buffer buffer
|
||||||
|
(buffer-substring-no-properties (point-min) (point-max))))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--mounted-settled-p (context)
|
||||||
|
"Return non-nil when CONTEXT has a mounted, visibly rendered surface."
|
||||||
|
(let ((buffer (etaf-gui-verifier-context-target-buffer context)))
|
||||||
|
(and (buffer-live-p buffer)
|
||||||
|
(ebox-surface-buffer-mounted-p buffer)
|
||||||
|
(eq (frame-parameter nil 'fullscreen)
|
||||||
|
(etaf-gui-verifier-context-get context 'target-fullscreen))
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p context))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--begin-viewport-change
|
||||||
|
(context fullscreen &optional width height)
|
||||||
|
"Record CONTEXT state before targeting FULLSCREEN, WIDTH, and HEIGHT."
|
||||||
|
(let ((report (etaf-playground-gui-scenarios--report context)))
|
||||||
|
(etaf-gui-verifier-context-put
|
||||||
|
context 'viewport-revision-before
|
||||||
|
(plist-get report :runtime-revision))
|
||||||
|
(etaf-gui-verifier-context-put context 'target-fullscreen fullscreen)
|
||||||
|
(etaf-gui-verifier-context-put context 'target-frame-width width)
|
||||||
|
(etaf-gui-verifier-context-put context 'target-frame-height height)))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--frame-target-settled-p (context)
|
||||||
|
"Return non-nil when the selected frame reached CONTEXT's target geometry."
|
||||||
|
(let ((fullscreen
|
||||||
|
(etaf-gui-verifier-context-get context 'target-fullscreen))
|
||||||
|
(width (etaf-gui-verifier-context-get context 'target-frame-width))
|
||||||
|
(height (etaf-gui-verifier-context-get context 'target-frame-height)))
|
||||||
|
(and (eq (frame-parameter nil 'fullscreen) fullscreen)
|
||||||
|
(or (not (numberp width)) (= (frame-text-width) width))
|
||||||
|
(or (not (numberp height)) (= (frame-text-height) height)))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--viewport-settled-p (context)
|
||||||
|
"Return non-nil when CONTEXT published the selected window's viewport."
|
||||||
|
(let* ((report (etaf-playground-gui-scenarios--report context))
|
||||||
|
(actual (ebox-viewport-window-width (selected-window)))
|
||||||
|
(published (or (plist-get report :target-viewport-width)
|
||||||
|
(plist-get report :viewport-width)))
|
||||||
|
(revision (plist-get report :runtime-revision))
|
||||||
|
(previous
|
||||||
|
(etaf-gui-verifier-context-get
|
||||||
|
context 'viewport-revision-before)))
|
||||||
|
(and (numberp actual)
|
||||||
|
(numberp published)
|
||||||
|
(= actual published)
|
||||||
|
(numberp revision)
|
||||||
|
(or (not (numberp previous)) (> revision previous))
|
||||||
|
(plist-get report :runtime-published)
|
||||||
|
(not (plist-get report :tp-scope-fallback))
|
||||||
|
(etaf-playground-gui-scenarios--frame-target-settled-p context)
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p context))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--invariants (context)
|
||||||
|
"Return shared Ebox/ETAF publication invariants for CONTEXT."
|
||||||
|
(let* ((buffer (etaf-gui-verifier-context-target-buffer context))
|
||||||
|
(mounted (and (buffer-live-p buffer)
|
||||||
|
(ebox-surface-buffer-mounted-p buffer)))
|
||||||
|
(text (and mounted
|
||||||
|
(with-current-buffer buffer
|
||||||
|
(buffer-substring-no-properties
|
||||||
|
(point-min) (point-max))))))
|
||||||
|
(when (etaf-gui-verifier-context-get context 'expects-mounted)
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert "surface-mounted" mounted)
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"rendered-output-nonempty" (and text (> (length text) 0)))
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"visible-output-nonempty"
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p context))
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"no-render-error"
|
||||||
|
(and text
|
||||||
|
(not (string-match-p
|
||||||
|
"could not render\\|runtime error\\|Wrong type argument"
|
||||||
|
text))))))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--adapter (context)
|
||||||
|
"Return Playground-specific JSON data for CONTEXT."
|
||||||
|
(let* ((report (etaf-playground-gui-scenarios--report context))
|
||||||
|
(buffer (etaf-gui-verifier-context-target-buffer context))
|
||||||
|
(runtime (and (buffer-live-p buffer)
|
||||||
|
(etaf-runtime-for-buffer buffer)))
|
||||||
|
(stage (plist-get report :stage)))
|
||||||
|
(list
|
||||||
|
(cons 'generation (if runtime (etaf-runtime-generation runtime) 0))
|
||||||
|
(cons 'viewport_width
|
||||||
|
(or (plist-get report :target-viewport-width)
|
||||||
|
(plist-get report :viewport-width)
|
||||||
|
0))
|
||||||
|
(cons 'frame_text_width (frame-text-width))
|
||||||
|
(cons 'frame_text_height (frame-text-height))
|
||||||
|
(cons 'fullscreen
|
||||||
|
(format "%s" (frame-parameter nil 'fullscreen)))
|
||||||
|
(cons 'target_frame_width
|
||||||
|
(or (etaf-gui-verifier-context-get
|
||||||
|
context 'target-frame-width)
|
||||||
|
0))
|
||||||
|
(cons 'target_frame_height
|
||||||
|
(or (etaf-gui-verifier-context-get
|
||||||
|
context 'target-frame-height)
|
||||||
|
0))
|
||||||
|
(cons 'stage (if stage (symbol-name stage) "none")))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--resize (context width height)
|
||||||
|
"Resize CONTEXT's selected frame to pixel WIDTH and HEIGHT."
|
||||||
|
(etaf-playground-gui-scenarios--begin-viewport-change
|
||||||
|
context nil width height)
|
||||||
|
(let ((frame-resize-pixelwise t))
|
||||||
|
(set-frame-size nil width height t)))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--windowed-action ()
|
||||||
|
"Return an action that leaves fullscreen and publishes its viewport."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id "leave-fullscreen"
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-playground-gui-scenarios--begin-viewport-change context nil)
|
||||||
|
(set-frame-parameter nil 'fullscreen nil))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--viewport-settled-p
|
||||||
|
:assertions #'etaf-playground-gui-scenarios--resize-assertions))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--resize-assertions (context)
|
||||||
|
"Return exact accepted-viewport assertions for CONTEXT."
|
||||||
|
(let* ((report (etaf-playground-gui-scenarios--report context))
|
||||||
|
(actual (ebox-viewport-window-width (selected-window)))
|
||||||
|
(published (or (plist-get report :target-viewport-width)
|
||||||
|
(plist-get report :viewport-width))))
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"viewport-published"
|
||||||
|
(and (numberp actual) (numberp published) (= actual published)
|
||||||
|
(plist-get report :runtime-published)))
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"resize-not-scope-fallback"
|
||||||
|
(not (plist-get report :tp-scope-fallback))))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--resize-action (id width height)
|
||||||
|
"Return resize action ID for pixel WIDTH and HEIGHT."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id id
|
||||||
|
:execute (lambda (context)
|
||||||
|
(etaf-playground-gui-scenarios--resize
|
||||||
|
context width height))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--viewport-settled-p
|
||||||
|
:assertions #'etaf-playground-gui-scenarios--resize-assertions
|
||||||
|
:screenshot t))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--maximize-action ()
|
||||||
|
"Return a stable full-workarea viewport publication action."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id "maximize-frame"
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-playground-gui-scenarios--begin-viewport-change
|
||||||
|
context 'maximized)
|
||||||
|
(set-frame-parameter nil 'fullscreen 'maximized))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--viewport-settled-p
|
||||||
|
:assertions #'etaf-playground-gui-scenarios--resize-assertions
|
||||||
|
:screenshot t))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--scroll-progressed-p (context)
|
||||||
|
"Return non-nil when CONTEXT's pending scroll changed visible state."
|
||||||
|
(or
|
||||||
|
(/= (etaf-gui-verifier-context-get
|
||||||
|
context 'scroll-window-start (window-start))
|
||||||
|
(window-start))
|
||||||
|
(not
|
||||||
|
(equal
|
||||||
|
(etaf-gui-verifier-context-get context 'scroll-report)
|
||||||
|
(etaf-playground-gui-scenarios--report context)))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--scroll-settled-p (context)
|
||||||
|
"Return non-nil when CONTEXT's scroll progressed to visible content."
|
||||||
|
(and (etaf-playground-gui-scenarios--scroll-progressed-p context)
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p context)))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--scroll-action (id command)
|
||||||
|
"Return Ebox page-scroll action ID using public COMMAND."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id id
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-gui-verifier-context-put context 'scroll-window-start
|
||||||
|
(window-start))
|
||||||
|
(etaf-gui-verifier-context-put
|
||||||
|
context 'scroll-report
|
||||||
|
(etaf-playground-gui-scenarios--report context))
|
||||||
|
(with-current-buffer (etaf-gui-verifier-context-target-buffer context)
|
||||||
|
(goto-char (window-start)))
|
||||||
|
(funcall command))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--scroll-settled-p
|
||||||
|
:assertions
|
||||||
|
(lambda (context)
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"scroll-progressed"
|
||||||
|
(etaf-playground-gui-scenarios--scroll-progressed-p context))))
|
||||||
|
:screenshot t))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--reset-scroll-action ()
|
||||||
|
"Return an action that restores the outer Emacs window to its top."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id "reset-scroll"
|
||||||
|
:execute
|
||||||
|
(lambda (_context)
|
||||||
|
(set-window-start (selected-window) (point-min))
|
||||||
|
(goto-char (point-min)))
|
||||||
|
:settled-p
|
||||||
|
(lambda (context)
|
||||||
|
(and (= (window-start) (point-min))
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p context)))
|
||||||
|
:assertions
|
||||||
|
(lambda (_context)
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"scroll-reset" (= (window-start) (point-min)))))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios--research-event-action
|
||||||
|
(id reference postcondition &optional screenshot)
|
||||||
|
"Return Research action ID for REFERENCE, POSTCONDITION, and SCREENSHOT."
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id id
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(let ((runtime
|
||||||
|
(etaf-runtime-for-buffer
|
||||||
|
(etaf-gui-verifier-context-target-buffer context))))
|
||||||
|
(etaf-gui-verifier-context-put
|
||||||
|
context (intern (concat id "-generation"))
|
||||||
|
(etaf-runtime-generation runtime))
|
||||||
|
(with-current-buffer
|
||||||
|
(etaf-gui-verifier-context-target-buffer context)
|
||||||
|
(etaf-focus runtime reference)
|
||||||
|
(execute-kbd-macro (kbd "RET")))))
|
||||||
|
:settled-p
|
||||||
|
(lambda (context)
|
||||||
|
(let ((runtime
|
||||||
|
(etaf-runtime-for-buffer
|
||||||
|
(etaf-gui-verifier-context-target-buffer context))))
|
||||||
|
(and runtime
|
||||||
|
(> (etaf-runtime-generation runtime)
|
||||||
|
(etaf-gui-verifier-context-get
|
||||||
|
context (intern (concat id "-generation")) -1))
|
||||||
|
(funcall postcondition context)
|
||||||
|
(etaf-playground-gui-scenarios--visible-window-content-p
|
||||||
|
context))))
|
||||||
|
:assertions
|
||||||
|
(lambda (context)
|
||||||
|
(let ((runtime
|
||||||
|
(etaf-runtime-for-buffer
|
||||||
|
(etaf-gui-verifier-context-target-buffer context))))
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"generation-advanced"
|
||||||
|
(> (etaf-runtime-generation runtime)
|
||||||
|
(etaf-gui-verifier-context-get
|
||||||
|
context (intern (concat id "-generation")) -1)))
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"activated-host-focused"
|
||||||
|
(equal reference (etaf-focused-host-ref runtime)))
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"product-postcondition"
|
||||||
|
(funcall postcondition context)))))
|
||||||
|
:screenshot screenshot))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios-research ()
|
||||||
|
"Return the Research Shelf GUI scenario adapter."
|
||||||
|
(etaf-gui-verifier-scenario-create
|
||||||
|
:name "research-shelf"
|
||||||
|
:claim "Research Shelf mounts, resizes, scrolls, and handles interactions"
|
||||||
|
:initialize
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-gui-verifier-context-select-buffer
|
||||||
|
context (get-buffer-create "*ETAF GUI Research Shelf*")))
|
||||||
|
:invariants #'etaf-playground-gui-scenarios--invariants
|
||||||
|
:adapter #'etaf-playground-gui-scenarios--adapter
|
||||||
|
:actions
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id "mount"
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-gui-verifier-context-put context 'target-fullscreen 'maximized)
|
||||||
|
(set-frame-parameter nil 'fullscreen 'maximized)
|
||||||
|
(etaf-playground-refresh-examples)
|
||||||
|
(etaf-gui-verifier-context-select-buffer
|
||||||
|
context
|
||||||
|
(etaf-playground-open-example
|
||||||
|
"research-shelf" "*ETAF GUI Research Shelf*"))
|
||||||
|
(etaf-gui-verifier-context-put context 'expects-mounted t))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--mounted-settled-p
|
||||||
|
:assertions
|
||||||
|
(lambda (context)
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"runtime-mounted"
|
||||||
|
(etaf-runtime-for-buffer
|
||||||
|
(etaf-gui-verifier-context-target-buffer context)))))
|
||||||
|
:screenshot t)
|
||||||
|
(etaf-playground-gui-scenarios--windowed-action)
|
||||||
|
(etaf-playground-gui-scenarios--resize-action
|
||||||
|
"resize-narrow" 900 500)
|
||||||
|
(etaf-playground-gui-scenarios--scroll-action
|
||||||
|
"scroll-down" #'ebox-scroll-page-down)
|
||||||
|
(etaf-playground-gui-scenarios--reset-scroll-action)
|
||||||
|
(etaf-playground-gui-scenarios--resize-action
|
||||||
|
"resize-wide" 1300 750)
|
||||||
|
(etaf-playground-gui-scenarios--research-event-action
|
||||||
|
"row-select" 'research-shelf-row-2
|
||||||
|
(lambda (context)
|
||||||
|
(string-match-p
|
||||||
|
"John Berger · Book"
|
||||||
|
(etaf-playground-gui-scenarios--buffer-text context))))
|
||||||
|
(etaf-playground-gui-scenarios--research-event-action
|
||||||
|
"theme-toggle" 'research-shelf-theme-toggle
|
||||||
|
(lambda (context)
|
||||||
|
(string-match-p
|
||||||
|
"Dark theme"
|
||||||
|
(etaf-playground-gui-scenarios--buffer-text context)))
|
||||||
|
t)
|
||||||
|
(etaf-playground-gui-scenarios--research-event-action
|
||||||
|
"pagination" 'research-shelf-page-next
|
||||||
|
(lambda (context)
|
||||||
|
(let ((text
|
||||||
|
(etaf-playground-gui-scenarios--buffer-text context)))
|
||||||
|
(and (string-match-p "Page 2 /" text)
|
||||||
|
(string-match-p "13–24 of" text))))
|
||||||
|
t)
|
||||||
|
(etaf-playground-gui-scenarios--maximize-action))
|
||||||
|
:completion
|
||||||
|
(lambda (context)
|
||||||
|
(= (etaf-gui-verifier-context-action-count context) 10))))
|
||||||
|
|
||||||
|
(defun etaf-playground-gui-scenarios-ebox (scenario fixture)
|
||||||
|
"Return one Ebox SCENARIO adapter for FIXTURE."
|
||||||
|
(etaf-gui-verifier-scenario-create
|
||||||
|
:name scenario
|
||||||
|
:claim "Ebox reference mounts, follows frame resize, and scrolls"
|
||||||
|
:initialize
|
||||||
|
(lambda (context)
|
||||||
|
(let ((source (find-file fixture)))
|
||||||
|
(ebox-dsl-mode)
|
||||||
|
(etaf-gui-verifier-context-select-buffer context source)))
|
||||||
|
:invariants #'etaf-playground-gui-scenarios--invariants
|
||||||
|
:adapter #'etaf-playground-gui-scenarios--adapter
|
||||||
|
:actions
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-action-create
|
||||||
|
:id "mount"
|
||||||
|
:execute
|
||||||
|
(lambda (context)
|
||||||
|
(etaf-gui-verifier-context-put context 'target-fullscreen 'maximized)
|
||||||
|
(set-frame-parameter nil 'fullscreen 'maximized)
|
||||||
|
(etaf-gui-verifier-context-select-buffer context (ebox-dsl-render))
|
||||||
|
(etaf-gui-verifier-context-put context 'expects-mounted t))
|
||||||
|
:settled-p #'etaf-playground-gui-scenarios--mounted-settled-p
|
||||||
|
:assertions
|
||||||
|
(lambda (context)
|
||||||
|
(let ((preview (etaf-gui-verifier-context-target-buffer context)))
|
||||||
|
(list
|
||||||
|
(etaf-gui-verifier-assert
|
||||||
|
"preview-mounted"
|
||||||
|
(and (buffer-live-p preview)
|
||||||
|
(ebox-surface-buffer-mounted-p preview))))))
|
||||||
|
:screenshot t)
|
||||||
|
(etaf-playground-gui-scenarios--windowed-action)
|
||||||
|
(etaf-playground-gui-scenarios--resize-action
|
||||||
|
"resize-narrow" 900 500)
|
||||||
|
(etaf-playground-gui-scenarios--scroll-action
|
||||||
|
"scroll-down" #'ebox-scroll-page-down)
|
||||||
|
(etaf-playground-gui-scenarios--reset-scroll-action)
|
||||||
|
(etaf-playground-gui-scenarios--resize-action
|
||||||
|
"resize-wide" 1300 750)
|
||||||
|
(etaf-playground-gui-scenarios--maximize-action))
|
||||||
|
:completion
|
||||||
|
(lambda (context)
|
||||||
|
(= (etaf-gui-verifier-context-action-count context) 7))))
|
||||||
|
|
||||||
|
;;;###autoload
|
||||||
|
(defun etaf-playground-gui-scenarios-run-from-environment ()
|
||||||
|
"Build one concrete adapter from environment and run the generic engine."
|
||||||
|
(unless (plist-get (ebox-native-reflow-runtime-report) :layout-ready-p)
|
||||||
|
(error "Prepared Ebox native module is unavailable"))
|
||||||
|
(let* ((scenario (or (getenv "ETAF_GUI_SCENARIO")
|
||||||
|
(error "ETAF_GUI_SCENARIO is not configured")))
|
||||||
|
(run-directory (or (getenv "ETAF_GUI_RUN_DIR")
|
||||||
|
(error "ETAF_GUI_RUN_DIR is not configured")))
|
||||||
|
(fixture (getenv "ETAF_GUI_FIXTURE"))
|
||||||
|
(adapter
|
||||||
|
(pcase scenario
|
||||||
|
("research-shelf"
|
||||||
|
(etaf-playground-gui-scenarios-research))
|
||||||
|
((or "flex-reference" "grid-reference")
|
||||||
|
(unless (and fixture (file-readable-p fixture))
|
||||||
|
(error "ETAF_GUI_FIXTURE is not readable"))
|
||||||
|
(etaf-playground-gui-scenarios-ebox scenario fixture))
|
||||||
|
(_ (error "Unknown Playground GUI scenario: %s" scenario)))))
|
||||||
|
(etaf-gui-verifier-run adapter run-directory)))
|
||||||
|
|
||||||
|
(provide 'playground-gui-scenarios)
|
||||||
|
;;; playground-gui-scenarios.el ends here
|
||||||
122
scripts/run-gui-verification.sh
Executable file
122
scripts/run-gui-verification.sh
Executable file
@ -0,0 +1,122 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
PLAYGROUND_SCRIPT_DIR=$(CDPATH= cd "$(dirname "$0")" && pwd)
|
||||||
|
PLAYGROUND_DIR=$(dirname "$PLAYGROUND_SCRIPT_DIR")
|
||||||
|
REPOS_DIR=$(dirname "$PLAYGROUND_DIR")
|
||||||
|
GENERIC_RUNNER="$REPOS_DIR/etaf/scripts/run-emacs-gui-verification.sh"
|
||||||
|
SCENARIO_ADAPTER="$PLAYGROUND_SCRIPT_DIR/playground-gui-scenarios.el"
|
||||||
|
SCENARIO_ENTRY=etaf-playground-gui-scenarios-run-from-environment
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage:" >&2
|
||||||
|
echo " $0 doctor" >&2
|
||||||
|
echo " $0 review RUN_DIR" >&2
|
||||||
|
echo " $0 all [--run-dir DIRECTORY]" >&2
|
||||||
|
echo " $0 research-shelf|flex-reference|grid-reference [--run-dir DIRECTORY]" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
doctor() {
|
||||||
|
[ -x "$GENERIC_RUNNER" ] || {
|
||||||
|
echo "generic GUI runner is unavailable: $GENERIC_RUNNER" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[ -f "$SCENARIO_ADAPTER" ] || {
|
||||||
|
echo "Playground scenario adapter is unavailable: $SCENARIO_ADAPTER" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
"$GENERIC_RUNNER" doctor
|
||||||
|
echo "PLAYGROUND-GUI-ADAPTER DOCTOR PASS"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_run_dir() {
|
||||||
|
SCENARIO_RUN_DIR=""
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
case $1 in
|
||||||
|
--run-dir)
|
||||||
|
[ "$#" -ge 2 ] || usage
|
||||||
|
SCENARIO_RUN_DIR=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
if [ -z "$SCENARIO_RUN_DIR" ]; then
|
||||||
|
SCENARIO_RUN_DIR=$(mktemp -d /private/tmp/etaf-playground-gui.XXXXXX)
|
||||||
|
else
|
||||||
|
mkdir -p "$SCENARIO_RUN_DIR"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_fixture() {
|
||||||
|
case $SCENARIO_NAME in
|
||||||
|
research-shelf)
|
||||||
|
SCENARIO_FIXTURE=""
|
||||||
|
;;
|
||||||
|
grid-reference)
|
||||||
|
SCENARIO_FIXTURE="$REPOS_DIR/ebox-playground/examples/grid-reference.ebox"
|
||||||
|
;;
|
||||||
|
flex-reference)
|
||||||
|
SCENARIO_FIXTURE_ROOT="$SCENARIO_RUN_DIR/fixture"
|
||||||
|
mkdir -p "$SCENARIO_FIXTURE_ROOT"
|
||||||
|
git -C "$REPOS_DIR/ebox-playground" archive \
|
||||||
|
--format=tar --output="$SCENARIO_RUN_DIR/flex-fixture.tar" \
|
||||||
|
HEAD examples/flex-reference.ebox
|
||||||
|
tar -xf "$SCENARIO_RUN_DIR/flex-fixture.tar" \
|
||||||
|
-C "$SCENARIO_FIXTURE_ROOT"
|
||||||
|
SCENARIO_FIXTURE="$SCENARIO_FIXTURE_ROOT/examples/flex-reference.ebox"
|
||||||
|
;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
run_scenario() {
|
||||||
|
doctor >/dev/null
|
||||||
|
prepare_fixture
|
||||||
|
export ETAF_GUI_SCENARIO="$SCENARIO_NAME"
|
||||||
|
export ETAF_GUI_FIXTURE="$SCENARIO_FIXTURE"
|
||||||
|
"$GENERIC_RUNNER" run "$SCENARIO_ADAPTER" "$SCENARIO_ENTRY" \
|
||||||
|
--run-dir "$SCENARIO_RUN_DIR" \
|
||||||
|
--load-path "$REPOS_DIR/tp" \
|
||||||
|
--load-path "$REPOS_DIR/ecss" \
|
||||||
|
--load-path "$REPOS_DIR/ebox" \
|
||||||
|
--load-path "$REPOS_DIR/etaf" \
|
||||||
|
--load-path "$REPOS_DIR/etaf-ui" \
|
||||||
|
--load-path "$REPOS_DIR/etaf-sqlite" \
|
||||||
|
--load-path "$REPOS_DIR/ebox-playground" \
|
||||||
|
--load-path "$REPOS_DIR/ebox-playground/scripts" \
|
||||||
|
--load-path "$PLAYGROUND_DIR" \
|
||||||
|
--load-path "$PLAYGROUND_SCRIPT_DIR" \
|
||||||
|
--load-path "$PLAYGROUND_DIR/examples"
|
||||||
|
}
|
||||||
|
|
||||||
|
[ "$#" -ge 1 ] || usage
|
||||||
|
COMMAND=$1
|
||||||
|
shift
|
||||||
|
|
||||||
|
case $COMMAND in
|
||||||
|
doctor)
|
||||||
|
[ "$#" -eq 0 ] || usage
|
||||||
|
doctor
|
||||||
|
;;
|
||||||
|
review)
|
||||||
|
[ "$#" -eq 1 ] || usage
|
||||||
|
"$GENERIC_RUNNER" review "$1"
|
||||||
|
;;
|
||||||
|
all)
|
||||||
|
parse_run_dir "$@"
|
||||||
|
ALL_RUN_ROOT=$SCENARIO_RUN_DIR
|
||||||
|
for SCENARIO_ITEM in research-shelf flex-reference grid-reference; do
|
||||||
|
"$0" "$SCENARIO_ITEM" --run-dir "$ALL_RUN_ROOT/$SCENARIO_ITEM"
|
||||||
|
done
|
||||||
|
echo "PLAYGROUND-GUI ALL CAPTURED root=$ALL_RUN_ROOT"
|
||||||
|
;;
|
||||||
|
research-shelf|flex-reference|grid-reference)
|
||||||
|
SCENARIO_NAME=$COMMAND
|
||||||
|
parse_run_dir "$@"
|
||||||
|
run_scenario
|
||||||
|
;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
Loading…
Reference in New Issue
Block a user