From 630567da8fff008de105ef8cd3149adfdf44be08 Mon Sep 17 00:00:00 2001 From: Kinneyzhang Date: Sun, 6 Sep 2026 13:53:45 +0800 Subject: [PATCH] fix: validate owned-window capture and recorder geometry --- scripts/README.md | 35 ++- scripts/capture-emacs-window.sh | 69 +++++ scripts/record-emacs-window.swift | 322 ++++++++++++++++++++++ scripts/record-screen.exp | 31 --- scripts/run-emacs-gui-verification.sh | 69 ++--- tests/test_gui_window_capture.py | 382 ++++++++++++++++++++++++++ 6 files changed, 837 insertions(+), 71 deletions(-) create mode 100755 scripts/capture-emacs-window.sh create mode 100644 scripts/record-emacs-window.swift delete mode 100755 scripts/record-screen.exp create mode 100644 tests/test_gui_window_capture.py diff --git a/scripts/README.md b/scripts/README.md index 4a4b2fa..b5856a4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,19 +1,35 @@ # Generic Emacs GUI verification -This directory owns the reusable verification mechanism, not application -scenarios: +For an already running Emacs, use its existing server through `emacsclient`. +Load the checkout and call the example entry point in an explicit buffer, show +that buffer in the existing graphical frame, and capture only that owned +window. Preserve the user's font and chrome. Do not start another daemon or +frame for this workflow. The `emacsclient-render-capture` skill supplies the +foreground and before/after target guards. + +This directory also owns reusable verification mechanisms; application +scenarios live in their respective example repositories: - `emacs-gui-verifier.el` defines `Scenario`, ordered `Action`, run-local `Context`, checkpoint sequencing, assertions, completion, and fail-closed evidence finalization. -- `run-emacs-gui-verification.sh` owns one named daemon, explicit load paths, +- The legacy isolated runner `run-emacs-gui-verification.sh` owns one named daemon, explicit load paths, external application activation, recorder lifecycle, driver loading, report generation, and exact cleanup. -- `record-screen.exp` keeps macOS `screencapture -v` attached to a PTY and stops - it through an explicit signal. It contains no Emacs or application logic. +- `record-emacs-window.swift` uses macOS 15 ScreenCaptureKit to record only the + frame owned by the supplied Emacs PID. It follows replacement window IDs and + rejects changed pixel mappings. Its canvas is fixed at recording start: + when the scenario includes resize, prepare the largest tested frame before + starting the recorder. Smaller windows retain native pixels with padding; + a window larger than the original canvas invalidates the recording. +- `capture-emacs-window.sh` supplies window-only checkpoint screenshots and + rejects missing, ambiguous, or changing frame identities. The runner compiles + the video helper using the system Swift compiler; no package install is needed. Concrete repositories provide adapter files that construct a Scenario and an -entry function. The generic command is: +entry function. They can run through the existing server. The following command +instead starts the legacy isolated environment; use it only when that separate +environment is explicitly intended: ```sh scripts/run-emacs-gui-verification.sh run ADAPTER.el ENTRY \ @@ -21,11 +37,14 @@ scripts/run-emacs-gui-verification.sh run ADAPTER.el ENTRY \ --run-dir /private/tmp/my-gui-run ``` -Fresh captures intentionally remain `INCOMPLETE` until their selected images +In that isolated runner, fresh captures remain `INCOMPLETE` until their selected images and contact sheet are reviewed. Finalize the same evidence directory with: ```sh scripts/run-emacs-gui-verification.sh review /private/tmp/my-gui-run ``` -Only `VERDICT=PASS` is completion evidence. +Only `VERDICT=PASS` completes that runner's evidence bundle. An existing-server +run reports its actual interaction assertions and inspected screenshots +separately. A failed recorder is never evidence of continuous capture, and +neither screenshots nor recording establish an operation latency bound. diff --git a/scripts/capture-emacs-window.sh b/scripts/capture-emacs-window.sh new file mode 100755 index 0000000..50cd827 --- /dev/null +++ b/scripts/capture-emacs-window.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +# Select an owned frame each time: fullscreen transitions can replace its +# WindowServer ID. Never fall back to capturing the desktop. +set -eu + +usage() { + echo "usage: $0 --window-id PID | -x OUTPUT" >&2 + exit 2 +} + +window_id() { + case $1 in + ''|*[!0-9]*) echo "invalid Emacs capture PID: $1" >&2; return 1 ;; + esac + [ "$1" -gt 1 ] && kill -0 "$1" 2>/dev/null || { + echo "Emacs capture process is not alive: $1" >&2 + return 1 + } + osascript -l JavaScript - "$1" <<'JXA' +ObjC.import("CoreGraphics"); +function run(argv) { + var pid = Number(argv[0]); + var windows = ObjC.deepUnwrap(ObjC.castRefToObject( + $.CGWindowListCopyWindowInfo($.kCGWindowListOptionAll, + $.kCGNullWindowID))); + var candidates = windows.filter(function (window) { + var bounds = window.kCGWindowBounds; + return window.kCGWindowOwnerPID === pid && + window.kCGWindowOwnerName === "Emacs" && + window.kCGWindowLayer === 0 && window.kCGWindowAlpha > 0 && + window.kCGWindowName && bounds && + bounds.Width > 0 && bounds.Height > 0; + }); + candidates.sort(function (a, b) { + return b.kCGWindowBounds.Width * b.kCGWindowBounds.Height - + a.kCGWindowBounds.Width * a.kCGWindowBounds.Height; + }); + if (!candidates.length) { + throw new Error("No named Emacs frame owned by PID " + pid); + } + if (candidates.length > 1 && + candidates[0].kCGWindowBounds.Width * candidates[0].kCGWindowBounds.Height === + candidates[1].kCGWindowBounds.Width * candidates[1].kCGWindowBounds.Height) { + throw new Error("Ambiguous largest Emacs frame for PID " + pid); + } + return String(candidates[0].kCGWindowNumber); +} +JXA +} + +[ "$#" -eq 2 ] || usage +case $1 in + --window-id) window_id "$2" ;; + -x) + GUI_CAPTURE_PID=${ETAF_GUI_CAPTURE_PID:?ETAF_GUI_CAPTURE_PID is required} + GUI_CAPTURE_ID=$(window_id "$GUI_CAPTURE_PID") + GUI_CAPTURE_BIN=${ETAF_GUI_SCREENCAPTURE:-/usr/sbin/screencapture} + "$GUI_CAPTURE_BIN" -l "$GUI_CAPTURE_ID" -o -x "$2" + # Reject a capture made while macOS replaced the intended frame. + GUI_CAPTURE_AFTER=$(window_id "$GUI_CAPTURE_PID") + if [ "$GUI_CAPTURE_ID" != "$GUI_CAPTURE_AFTER" ]; then + rm -f -- "$2" + echo "Emacs frame changed during screenshot; capture again" >&2 + exit 1 + fi + ;; + *) usage ;; +esac diff --git a/scripts/record-emacs-window.swift b/scripts/record-emacs-window.swift new file mode 100644 index 0000000..5fd44d7 --- /dev/null +++ b/scripts/record-emacs-window.swift @@ -0,0 +1,322 @@ +// Record only the owned Emacs frame. Build with swiftc -parse-as-library. +import Foundation +import CoreGraphics +import ScreenCaptureKit +import AVFoundation + +struct CaptureFailure: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +// Delegate callbacks and signal handlers do not share an execution queue. +final class CaptureEvents: NSObject, SCStreamDelegate, SCRecordingOutputDelegate, + SCStreamOutput, @unchecked Sendable { + private let lock = NSLock() + private var started = false + private var finished = false + private var stopping = false + private var frames = 0 + private var failure: String? + private var nativeSize: (width: Int, height: Int)? + private var pixelMapping: Double? + private var firstPTS: Double? + private var lastPTS: Double? + private let requireNative: Bool + private let expectedPixelScale: Double + + init(pixelScale: Double, requireNative: Bool = true) { + self.expectedPixelScale = pixelScale + self.requireNative = requireNative + } + + func state() -> (started: Bool, finished: Bool, stopping: Bool, + frames: Int, failure: String?, nativeSize: (width: Int, height: Int)?, + pixelMapping: Double?, firstPTS: Double?, lastPTS: Double?) { + lock.withLock { + (started, finished, stopping, frames, failure, nativeSize, + pixelMapping, firstPTS, lastPTS) + } + } + func stop() { lock.withLock { stopping = true } } + func fail(_ message: String) { lock.withLock { failure = message } } + func recordingOutputDidStartRecording(_ output: SCRecordingOutput) { + lock.withLock { started = true } + } + func recordingOutputDidFinishRecording(_ output: SCRecordingOutput) { + lock.withLock { finished = true } + } + func recordingOutput(_ output: SCRecordingOutput, didFailWithError error: Error) { + fail("Recording failed: \(error)") + } + func stream(_ stream: SCStream, didStopWithError error: Error) { + fail("Capture stream stopped: \(error)") + } + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of type: SCStreamOutputType) { + guard type == .screen else { return } + inspectScreenSample(sampleBuffer) + } + func inspectScreenSample(_ sampleBuffer: CMSampleBuffer) { + guard let attachments = CMSampleBufferGetSampleAttachmentsArray( + sampleBuffer, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]], + let info = attachments.first, + let status = info[.status] as? Int, + status == SCFrameStatus.complete.rawValue else { return } + guard let scale = info[.contentScale] as? Double, scale.isFinite, scale > 0, + let backing = info[.scaleFactor] as? Double, backing.isFinite, backing > 0, + let rectDictionary = info[.contentRect] as? [String: Any], + let rect = CGRect(dictionaryRepresentation: rectDictionary as CFDictionary), + rect.origin.x.isFinite, rect.origin.y.isFinite, + rect.width.isFinite, rect.height.isFinite, + rect.width > 0, rect.height > 0 else { + fail("Captured frame has no usable native geometry") + return + } + // SCK can render at a higher resolution than the window's display. + // Its source-to-surface scale and source backing scale together define + // output pixels per logical point; compare that with the real display. + let mapping = scale * backing / expectedPixelScale + if requireNative && abs(mapping - 1) >= 0.0001 { + let image = CMSampleBufferGetImageBuffer(sampleBuffer) + let dimensions = image.map { "\(CVPixelBufferGetWidth($0))x\(CVPixelBufferGetHeight($0))" } ?? "missing" + fail("Captured frame was scaled; native pixels are required. Buffer=\(dimensions), attachments=\(info)") + return + } + let sourceWidth = rect.width / scale * expectedPixelScale + let sourceHeight = rect.height / scale * expectedPixelScale + if requireNative { + guard let image = CMSampleBufferGetImageBuffer(sampleBuffer) else { + fail("Captured frame has no pixel buffer") + return + } + let width = Double(CVPixelBufferGetWidth(image)) + let height = Double(CVPixelBufferGetHeight(image)) + guard sourceWidth <= width, sourceHeight <= height, + rect.minX >= 0, rect.minY >= 0, + rect.maxX * backing <= width, rect.maxY * backing <= height else { + fail("Captured frame exceeds original native-pixel canvas or is clipped") + return + } + } + let timestamp = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) + guard timestamp.isFinite else { fail("Captured frame has no valid timestamp"); return } + lock.withLock { + nativeSize = (Int(ceil(sourceWidth)), Int(ceil(sourceHeight))) + pixelMapping = mapping + if abs(mapping - 1) < 0.0001 { + frames += 1 + if firstPTS == nil { firstPTS = timestamp } + lastPTS = timestamp + } + } + } +} + +@main +struct RecordEmacsWindow { + static func windowID(pid: pid_t) throws -> CGWindowID { + guard kill(pid, 0) == 0, + let inventory = CGWindowListCopyWindowInfo(.optionAll, kCGNullWindowID) + as? [[String: Any]] else { + throw CaptureFailure("Owned Emacs process is unavailable: \(pid)") + } + let candidates = inventory.compactMap { item -> (CGWindowID, Double)? in + guard item[kCGWindowOwnerPID as String] as? Int == Int(pid), + item[kCGWindowOwnerName as String] as? String == "Emacs", + item[kCGWindowLayer as String] as? Int == 0, + let alpha = item[kCGWindowAlpha as String] as? Double, alpha > 0, + let title = item[kCGWindowName as String] as? String, !title.isEmpty, + let bounds = item[kCGWindowBounds as String] as? [String: Double], + let width = bounds["Width"], let height = bounds["Height"], + width > 0, height > 0, + let number = item[kCGWindowNumber as String] as? UInt32 else { return nil } + return (number, width * height) + }.sorted { $0.1 > $1.1 } + guard let first = candidates.first else { + throw CaptureFailure("No named Emacs frame owned by PID \(pid)") + } + guard candidates.count == 1 || first.1 != candidates[1].1 else { + throw CaptureFailure("Ambiguous largest Emacs frame for PID \(pid)") + } + return first.0 + } + + static func filter(pid: pid_t, id: CGWindowID) async throws -> SCContentFilter { + let content = try await SCShareableContent.excludingDesktopWindows( + true, onScreenWindowsOnly: false) + guard let window = content.windows.first(where: { + $0.windowID == id && $0.owningApplication?.processID == pid + && $0.windowLayer == 0 + }) else { throw CaptureFailure("Owned frame is not shareable: \(id)") } + return SCContentFilter(desktopIndependentWindow: window) + } + + static func emit(_ values: [String: Any]) throws { + let data = try JSONSerialization.data(withJSONObject: values, options: [.sortedKeys]) + FileHandle.standardOutput.write(data + Data([10])) + } + + static func configureNativeCanvas(filter: SCContentFilter, + configuration: SCStreamConfiguration) async throws { + // Fullscreen WindowServer bounds can exclude native window decoration. + // Discover the full source extent from one unrecorded sample; never + // guess offsets or grow the canvas iteratively after recording starts. + let probe = CaptureEvents(pixelScale: Double(filter.pointPixelScale), requireNative: false) + let stream = SCStream(filter: filter, configuration: configuration, delegate: probe) + try stream.addStreamOutput(probe, type: .screen, + sampleHandlerQueue: DispatchQueue(label: "capture-probe")) + try await stream.startCapture() + let deadline = Date().addingTimeInterval(10) + do { + while probe.state().nativeSize == nil { + if let failure = probe.state().failure { throw CaptureFailure(failure) } + guard Date() < deadline else { throw CaptureFailure("No native geometry sample") } + try await Task.sleep(nanoseconds: 20_000_000) + } + let size = probe.state().nativeSize! + configuration.width = size.width + size.width % 2 + configuration.height = size.height + size.height % 2 + try await stream.updateConfiguration(configuration) + while probe.state().pixelMapping.map({ abs($0 - 1) >= 0.0001 }) ?? true { + if let failure = probe.state().failure { throw CaptureFailure(failure) } + guard Date() < deadline else { + throw CaptureFailure("Measured canvas did not preserve native pixels") + } + try await Task.sleep(nanoseconds: 20_000_000) + } + try await stream.stopCapture() + } catch { + try? await stream.stopCapture() + throw error + } + } + + static func record(pid: pid_t, output: URL, ready: URL) async throws { + guard !FileManager.default.fileExists(atPath: output.path), + !FileManager.default.fileExists(atPath: ready.path) else { + throw CaptureFailure("Recording output or readiness file already exists") + } + var id = try windowID(pid: pid) + var currentFilter = try await filter(pid: pid, id: id) + let scale = Double(currentFilter.pointPixelScale) + let width = Int(ceil(currentFilter.contentRect.width * scale)) + let height = Int(ceil(currentFilter.contentRect.height * scale)) + guard width > 0, height > 0 else { throw CaptureFailure("Empty frame") } + let configuration = SCStreamConfiguration() + // Fixed canvas; smaller windows are padded, never enlarged. The frame + // callback rejects any change to the display's native pixel mapping. + configuration.width = width + width % 2 + configuration.height = height + height % 2 + configuration.scalesToFit = false + configuration.captureResolution = .automatic + configuration.preservesAspectRatio = true + configuration.ignoreShadowsSingleWindow = true + configuration.showsCursor = false + let background = CGColor(gray: 0, alpha: 1) + defer { withExtendedLifetime(background) {} } + configuration.backgroundColor = background + configuration.minimumFrameInterval = CMTime(value: 1, timescale: 60) + configuration.capturesAudio = false + try await configureNativeCanvas(filter: currentFilter, configuration: configuration) + let events = CaptureEvents(pixelScale: scale) + try emit(["event": "configuration", "window_id": id, "pid": pid, + "width": configuration.width, "height": configuration.height, + "pixel_scale": scale, "content_rect": NSStringFromRect(currentFilter.contentRect)]) + let stream = SCStream(filter: currentFilter, configuration: configuration, + delegate: events) + try stream.addStreamOutput(events, type: .screen, + sampleHandlerQueue: DispatchQueue(label: "capture-frames")) + let recordingConfiguration = SCRecordingOutputConfiguration() + recordingConfiguration.outputURL = output + recordingConfiguration.outputFileType = .mov + let recording = SCRecordingOutput(configuration: recordingConfiguration, delegate: events) + try stream.addRecordingOutput(recording) + let signals = [SIGTERM, SIGINT].map { number -> DispatchSourceSignal in + signal(number, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: number, queue: .global()) + source.setEventHandler { events.stop() } + source.resume() + return source + } + defer { signals.forEach { $0.cancel() } } + try await stream.startCapture() + let startedAt = Date() + var announced = false + do { + while true { + let nextID = try windowID(pid: pid) + // SCContentFilter is a snapshot: refresh even for the same ID, + // whose source can resize independently of window replacement. + currentFilter = try await filter(pid: pid, id: nextID) + let nextScale = Double(currentFilter.pointPixelScale) + guard nextScale.isFinite, nextScale > 0, + currentFilter.contentRect.width > 0, + currentFilter.contentRect.height > 0, + currentFilter.contentRect.width * nextScale <= Double(configuration.width), + currentFilter.contentRect.height * nextScale <= Double(configuration.height) else { + throw CaptureFailure("Owned frame exceeds original native-pixel canvas or has invalid geometry") + } + if nextID != id { + try await stream.updateContentFilter(currentFilter) + id = nextID + try emit(["event": "window-replaced", "window_id": id, "pid": pid]) + } + // Check the current source before readiness and requested stop; + // also re-read callback failures after the asynchronous refresh. + let state = events.state() + if let failure = state.failure { throw CaptureFailure(failure) } + if state.stopping { break } + if state.finished { throw CaptureFailure("Recording ended before requested stop") } + let elapsed = Date().timeIntervalSince(startedAt) + if elapsed > 600 { throw CaptureFailure("Recording exceeded 600 seconds") } + if !announced && elapsed > 10 { throw CaptureFailure("No verified recording frames") } + if !announced && state.started && state.frames > 0 { + let metadata: [String: Any] = ["pid": pid, "window_id": id, + "width": configuration.width, "height": configuration.height, + "pixel_scale": scale, "capture": "desktop-independent-window", + "ready_wall_time": Date().timeIntervalSince1970, + "source_first_pts": state.firstPTS!] + try JSONSerialization.data(withJSONObject: metadata).write(to: ready, options: .atomic) + try emit(metadata) + announced = true + } + try await Task.sleep(nanoseconds: 100_000_000) + } + try await stream.stopCapture() + let stopDeadline = Date().addingTimeInterval(4) + while !events.state().finished { + if let failure = events.state().failure { throw CaptureFailure(failure) } + guard Date() < stopDeadline else { throw CaptureFailure("Recording did not finalize") } + try await Task.sleep(nanoseconds: 20_000_000) + } + if let failure = events.state().failure { throw CaptureFailure(failure) } + guard announced, events.state().frames > 0 else { + throw CaptureFailure("Recording stopped without verified frames") + } + let final = events.state() + try emit(["event": "finished", "frames": final.frames, + "source_first_pts": final.firstPTS!, "source_last_pts": final.lastPTS!, + "source_span_seconds": final.lastPTS! - final.firstPTS!, + "recorded_duration_seconds": CMTimeGetSeconds(recording.recordedDuration), + "wall_duration_seconds": Date().timeIntervalSince(startedAt)]) + } catch { + try? await stream.stopCapture() + throw error + } + } + + static func main() async { + do { + let arguments = CommandLine.arguments + guard arguments.count == 4, let pid = Int32(arguments[1]), pid > 1 else { + throw CaptureFailure("usage: record-emacs-window PID OUTPUT.mov READY_FILE") + } + try await record(pid: pid, output: URL(fileURLWithPath: arguments[2]), + ready: URL(fileURLWithPath: arguments[3])) + } catch { + FileHandle.standardError.write(Data("\(error)\n".utf8)) + exit(1) + } + } +} diff --git a/scripts/record-screen.exp b/scripts/record-screen.exp deleted file mode 100755 index e95eddf..0000000 --- a/scripts/record-screen.exp +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/expect -f - -# Keep macOS screencapture attached to a real PTY. Redirecting its stdin to -# /dev/null makes current macOS builds stop recording immediately. - -set maximum_seconds 600 - -if {$argc != 1} { - puts stderr "usage: record-screen.exp OUTPUT" - exit 2 -} - -set output [lindex $argv 0] -spawn -noecho /usr/sbin/screencapture -v -D 1 -x $output -set deadline [expr {[clock seconds] + $maximum_seconds}] - -trap { - send -- "q" - expect eof - exit 0 -} {SIGTERM SIGINT} - -while {1} { - after 1000 - if {[clock seconds] >= $deadline} { - send -- "q" - expect eof - puts stderr "screen recording exceeded $maximum_seconds seconds" - exit 1 - } -} diff --git a/scripts/run-emacs-gui-verification.sh b/scripts/run-emacs-gui-verification.sh index 80e5194..9eb605d 100755 --- a/scripts/run-emacs-gui-verification.sh +++ b/scripts/run-emacs-gui-verification.sh @@ -4,11 +4,11 @@ set -eu GUI_CORE_SCRIPT_DIR=$(CDPATH= cd "$(dirname "$0")" && pwd) GUI_ENGINE="$GUI_CORE_SCRIPT_DIR/emacs-gui-verifier.el" -GUI_RECORDER="$GUI_CORE_SCRIPT_DIR/record-screen.exp" +GUI_RECORDER_SOURCE="$GUI_CORE_SCRIPT_DIR/record-emacs-window.swift" +GUI_CAPTURE="$GUI_CORE_SCRIPT_DIR/capture-emacs-window.sh" GUI_EMACS_APP=${EMACS_APP:-/Applications/Emacs.app} GUI_EMACS_BIN="$GUI_EMACS_APP/Contents/MacOS/Emacs" GUI_EMACSCLIENT="$GUI_EMACS_APP/Contents/MacOS/bin/emacsclient" -GUI_EMACS_APP_NAME=${EMACS_APP_NAME:-Emacs} GUI_CODEX_ROOT=${CODEX_HOME:-${HOME}/.codex} GUI_DYNAMIC_SKILL=${EMACS_DYNAMIC_UI_SKILL_DIR:-$GUI_CODEX_ROOT/skills/emacs-dynamic-ui-verification} GUI_CHECKPOINT_EL="$GUI_DYNAMIC_SKILL/scripts/capture-checkpoint.el" @@ -18,7 +18,7 @@ usage() { echo "usage:" >&2 echo " $0 doctor" >&2 echo " $0 review RUN_DIR" >&2 - echo " $0 run ADAPTER_EL ENTRY_FUNCTION [--run-dir DIR] [--load-path DIR]..." >&2 + echo " $0 run ADAPTER_EL ENTRY_FUNCTION [--initialize-function FUNCTION] [--run-dir DIR] [--load-path DIR]..." >&2 exit 2 } @@ -33,11 +33,12 @@ doctor() { require_file "$GUI_EMACS_BIN" require_file "$GUI_EMACSCLIENT" require_file "$GUI_ENGINE" - require_file "$GUI_RECORDER" + require_file "$GUI_RECORDER_SOURCE" + require_file "$GUI_CAPTURE" require_file "$GUI_CHECKPOINT_EL" require_file "$GUI_EVIDENCE_PY" command -v osascript >/dev/null 2>&1 - command -v expect >/dev/null 2>&1 + xcrun --find swiftc >/dev/null 2>&1 command -v ffmpeg >/dev/null 2>&1 command -v python3 >/dev/null 2>&1 echo "EMACS-GUI-VERIFIER DOCTOR PASS" @@ -55,7 +56,6 @@ review_run() { GUI_DAEMON="" GUI_DAEMON_PID="" GUI_RECORDER_PID="" -GUI_RECORDER_CHILD_PID="" GUI_DAEMON_STARTED=false valid_pid() { @@ -101,22 +101,15 @@ stop_recorder() { GUI_RECORDER_STOP_FAILED=1 elif ! wait_for_exit "$GUI_RECORDER_PID" 50; then GUI_RECORDER_STOP_FAILED=1 - if ! force_owned_exit "$GUI_RECORDER_PID" "recorder wrapper"; then + if ! force_owned_exit "$GUI_RECORDER_PID" "window recorder"; then GUI_RECORDER_STOP_FAILED=1 fi fi fi - if process_alive "$GUI_RECORDER_CHILD_PID"; then - GUI_RECORDER_STOP_FAILED=1 - if ! force_owned_exit "$GUI_RECORDER_CHILD_PID" "screen recorder"; then - GUI_RECORDER_STOP_FAILED=1 - fi - fi if ! wait "$GUI_RECORDER_PID" 2>/dev/null; then GUI_RECORDER_STOP_FAILED=1 fi GUI_RECORDER_PID="" - GUI_RECORDER_CHILD_PID="" [ "$GUI_RECORDER_STOP_FAILED" -eq 0 ] } @@ -165,17 +158,16 @@ trap on_exit EXIT trap 'exit 130' HUP INT TERM activate_emacs() { - osascript -e "tell application \"$GUI_EMACS_APP_NAME\" to activate" + osascript -e "tell application \"System Events\" to set frontmost of first application process whose unix id is $GUI_DAEMON_PID to true" GUI_ACTIVATE_ATTEMPTS=0 while [ "$GUI_ACTIVATE_ATTEMPTS" -lt 30 ]; do GUI_FRONTMOST=$(osascript -e \ - 'tell application "System Events" to get name of first application process whose frontmost is true') - case $GUI_FRONTMOST in - *Emacs*) - sleep 0.2 - return 0 - ;; - esac + 'tell application "System Events" to get unix id of first application process whose frontmost is true') + if [ "$GUI_FRONTMOST" = "$GUI_DAEMON_PID" ]; then + # macOS fullscreen/Space animations continue after focus changes. + sleep 1 + return 0 + fi sleep 0.1 GUI_ACTIVATE_ATTEMPTS=$((GUI_ACTIVATE_ATTEMPTS + 1)) done @@ -184,26 +176,23 @@ activate_emacs() { } start_recorder() { - /usr/bin/expect "$GUI_RECORDER" "$GUI_RUN_DIR/recording.mov" \ + "$GUI_RECORDER" "$GUI_DAEMON_PID" "$GUI_RUN_DIR/recording.mov" \ + "$GUI_RUN_DIR/recorder.ready" \ >"$GUI_RUN_DIR/recorder.log" 2>&1 & GUI_RECORDER_PID=$! GUI_RECORDER_ATTEMPTS=0 - while [ "$GUI_RECORDER_ATTEMPTS" -lt 30 ]; do + while [ "$GUI_RECORDER_ATTEMPTS" -lt 150 ]; do if ! kill -0 "$GUI_RECORDER_PID" 2>/dev/null; then echo "screen recorder exited before the scenario started" >&2 exit 1 fi - GUI_RECORDER_CHILD_PID=$( - pgrep -P "$GUI_RECORDER_PID" -x screencapture 2>/dev/null | - sed -n '1p' - ) - if process_alive "$GUI_RECORDER_CHILD_PID"; then + if [ -s "$GUI_RUN_DIR/recorder.ready" ]; then return 0 fi sleep 0.1 GUI_RECORDER_ATTEMPTS=$((GUI_RECORDER_ATTEMPTS + 1)) done - echo "screen recorder did not create its PTY child" >&2 + echo "window recorder did not produce a verified frame" >&2 exit 1 } @@ -214,8 +203,14 @@ parse_run_arguments() { shift 2 GUI_RUN_DIR="" GUI_LOAD_PATHS="" + GUI_INITIALIZE_FUNCTION="" while [ "$#" -gt 0 ]; do case $1 in + --initialize-function) + [ "$#" -ge 2 ] || usage + GUI_INITIALIZE_FUNCTION=$2 + shift 2 + ;; --run-dir) [ "$#" -ge 2 ] || usage GUI_RUN_DIR=$2 @@ -252,9 +247,14 @@ parse_run_arguments() { run_adapter() { doctor >/dev/null + GUI_RECORDER="$GUI_RUN_DIR/record-emacs-window" + xcrun swiftc -parse-as-library -warnings-as-errors -O \ + "$GUI_RECORDER_SOURCE" -o "$GUI_RECORDER" export ETAF_GUI_RUN_DIR="$GUI_RUN_DIR" export ETAF_GUI_LOAD_PATHS="$GUI_LOAD_PATHS" export ETAF_GUI_ENTRY="$GUI_ENTRY" + export ETAF_GUI_INITIALIZE_FUNCTION="$GUI_INITIALIZE_FUNCTION" + export SCREENCAPTURE="$GUI_CAPTURE" GUI_DAEMON="emacs-gui-verify-$$" "$GUI_EMACS_BIN" -Q --daemon="$GUI_DAEMON" \ --eval '(setq native-comp-jit-compilation nil load-prefer-newer t)' @@ -276,8 +276,13 @@ run_adapter() { (when (fboundp 'tool-bar-mode) (tool-bar-mode -1)) (when (fboundp 'menu-bar-mode) (menu-bar-mode -1)) (when (fboundp 'scroll-bar-mode) (scroll-bar-mode -1)) - (set-frame-parameter nil 'fullscreen 'maximized) - (raise-frame) + (setenv \"ETAF_GUI_CAPTURE_PID\" (number-to-string (emacs-pid))) + (set-frame-parameter nil 'fullscreen 'fullboth) + (when-let* ((initialize (getenv \"ETAF_GUI_INITIALIZE_FUNCTION\")) + ((> (length initialize) 0))) + (funcall (intern initialize))) + (select-frame-set-input-focus (selected-frame)) + (message nil) (redisplay t) t)" >"$GUI_RUN_DIR/bootstrap.out" activate_emacs diff --git a/tests/test_gui_window_capture.py b/tests/test_gui_window_capture.py new file mode 100644 index 0000000..e815eb3 --- /dev/null +++ b/tests/test_gui_window_capture.py @@ -0,0 +1,382 @@ +"""Owned-window capture routing preflight; never captures a real screen.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +CAPTURE = ROOT / "scripts/capture-emacs-window.sh" +RUNNER = ROOT / "scripts/run-emacs-gui-verification.sh" +RECORDER_SOURCE = ROOT / "scripts/record-emacs-window.swift" + + +class RecorderCanvasTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.build = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.build.cleanup) + source = RECORDER_SOURCE.read_text() + # Run the actual polling guard with fixture-only providers. Compiling + # the rest of the recorder also keeps its real validation available; + # the recorder's main and all capture APIs remain uninvoked. + guard = source.split(" let nextID = try windowID(pid: pid)\n", 1)[1] + guard_end = ' try emit(["event": "window-replaced", "window_id": id, "pid": pid])\n }' + guard = ("let nextID = try windowID(pid: pid)\n" + guard.split( + guard_end, 1)[0] + guard_end) + driver = r''' +struct FixtureFilter { + let contentRect: CGRect + let pointPixelScale: Double +} +final class FixtureStream { + var updates = 0 + func updateContentFilter(_ filter: FixtureFilter) async throws { updates += 1 } +} +extension RecordEmacsWindow { + static func checkFixture(_ input: [String: Any]) async -> [String: Any] { + let pid: pid_t = 123 + var id = CGWindowID(input["current_id"] as! Int) + var currentFilter = FixtureFilter(contentRect: CGRect(x: 0, y: 0, width: 500, height: 400), + pointPixelScale: 2) + let configuration = SCStreamConfiguration() + configuration.width = 1000 + configuration.height = 800 + let stream = FixtureStream() + var reads = 0 + func windowID(pid: pid_t) throws -> CGWindowID { CGWindowID(input["next_id"] as! Int) } + func filter(pid: pid_t, id: CGWindowID) async throws -> FixtureFilter { + reads += 1 + return FixtureFilter(contentRect: CGRect(x: 0, y: 0, + width: input["width"] as! Double, height: input["height"] as! Double), + pointPixelScale: input["scale"] as! Double) + } + func emit(_ values: [String: Any]) throws {} + var failure: String? + do { +''' + guard + r''' + } catch { failure = String(describing: error) } + return ["error": failure as Any? ?? NSNull(), "filter_reads": reads, + "updates": stream.updates, "window_id": id] + } +} +func checkFrameFixture(_ input: [String: Any]) throws -> [String: Any] { + var image: CVPixelBuffer? + guard CVPixelBufferCreate(kCFAllocatorDefault, 1000, 800, kCVPixelFormatType_32BGRA, + nil, &image) == kCVReturnSuccess, let image else { + throw CaptureFailure("Cannot create fixture pixels") + } + var format: CMVideoFormatDescription? + guard CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, + imageBuffer: image, formatDescriptionOut: &format) == noErr, let format else { + throw CaptureFailure("Cannot create fixture video format") + } + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: 60), + presentationTimeStamp: CMTime(value: 1, timescale: 60), decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + guard CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, + imageBuffer: image, formatDescription: format, sampleTiming: &timing, + sampleBufferOut: &sample) == noErr, let sample else { + throw CaptureFailure("Cannot create fixture sample") + } + let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: true)! + as! [NSMutableDictionary] + attachments[0][SCStreamFrameInfo.status.rawValue] = SCFrameStatus.complete.rawValue + attachments[0][SCStreamFrameInfo.contentScale.rawValue] = input["content_scale"] as! Double + attachments[0][SCStreamFrameInfo.scaleFactor.rawValue] = input["backing_scale"] as! Double + attachments[0][SCStreamFrameInfo.contentRect.rawValue] = CGRect( + x: input["x"] as! Double, y: input["y"] as! Double, + width: input["width"] as! Double, height: input["height"] as! Double).dictionaryRepresentation + let events = CaptureEvents(pixelScale: 2) + events.inspectScreenSample(sample) + let result = events.state() + return ["error": result.failure as Any? ?? NSNull(), "frames": result.frames] +} +@main +struct CanvasFixtureMain { + static func main() async throws { + let input = try JSONSerialization.jsonObject(with: FileHandle.standardInput.readDataToEndOfFile()) + as! [String: Any] + let result = input["frame"] as? Bool == true + ? try checkFrameFixture(input) : await RecordEmacsWindow.checkFixture(input) + try RecordEmacsWindow.emit(result) + } +} +''' + path = Path(cls.build.name) / "canvas-fixture.swift" + path.write_text(source.replace("@main\nstruct RecordEmacsWindow", + "struct RecordEmacsWindow", 1) + driver) + cls.fixture = Path(cls.build.name) / "canvas-fixture" + subprocess.run( + ["xcrun", "swiftc", "-parse-as-library", "-warnings-as-errors", "-O", + str(path), "-o", str(cls.fixture)], check=True, + ) + + def check_canvas(self, next_id, width, height, scale=2): + result = subprocess.run( + [str(self.fixture)], input=json.dumps(dict(current_id=123, next_id=next_id, + width=width, height=height, scale=scale)), + capture_output=True, text=True, check=True, timeout=10, + ) + return json.loads(result.stdout) + + def test_same_id_fit_uses_fresh_geometry(self): + result = self.check_canvas(123, 400, 300) + self.assertIsNone(result["error"]) + self.assertEqual(result["filter_reads"], 1) + self.assertEqual(result["updates"], 0) + + def test_same_id_growth_rejects_either_dimension(self): + for width, height in ((501, 400), (500, 401)): + with self.subTest(width=width, height=height): + result = self.check_canvas(123, width, height) + self.assertIsNotNone(result["error"]) + self.assertIn("exceeds original native-pixel canvas", result["error"]) + self.assertEqual(result["updates"], 0) + + def test_replacement_fit_updates_filter(self): + result = self.check_canvas(456, 400, 300) + self.assertIsNone(result["error"]) + self.assertEqual(result["filter_reads"], 1) + self.assertEqual(result["updates"], 1) + self.assertEqual(result["window_id"], 456) + + def test_replacement_growth_rejects_before_filter_update(self): + for width, height in ((501, 400), (500, 401)): + with self.subTest(width=width, height=height): + result = self.check_canvas(456, width, height) + self.assertIn("exceeds original native-pixel canvas", result["error"]) + self.assertEqual(result["updates"], 0) + self.assertEqual(result["window_id"], 123) + + def test_exact_native_canvas_boundary(self): + for identifier in (123, 456): + for width, height, scale in ((500, 400, 2), (1000, 800, 1)): + with self.subTest(identifier=identifier, scale=scale): + result = self.check_canvas(identifier, width, height, scale) + self.assertIsNone(result["error"]) + self.assertEqual(result["filter_reads"], 1) + + def test_fractional_native_pixel_overflow_rejects(self): + for identifier in (123, 456): + for width, height in ((500.01, 400), (500, 400.01)): + with self.subTest(identifier=identifier, width=width, height=height): + self.assertIsNotNone(self.check_canvas(identifier, width, height)["error"]) + + def test_unusable_fresh_geometry_is_rejected(self): + for identifier in (123, 456): + for width, height, scale in ((0, 400, 2), (500, 0, 2), (500, 400, 0)): + with self.subTest(identifier=identifier, width=width, height=height, scale=scale): + result = self.check_canvas(identifier, width, height, scale) + self.assertIsNotNone(result["error"]) + self.assertEqual(result["updates"], 0) + + def check_frame(self, width, height, x=0, y=0, content_scale=1, backing_scale=2): + result = subprocess.run( + [str(self.fixture)], input=json.dumps(dict(frame=True, width=width, height=height, + x=x, y=y, content_scale=content_scale, backing_scale=backing_scale)), + capture_output=True, text=True, check=True, timeout=10, + ) + return json.loads(result.stdout) + + def test_native_frame_exact_boundary_and_padding(self): + for width, height, x, y in ((500, 400, 0, 0), (400, 300, 50, 50)): + with self.subTest(width=width, height=height): + result = self.check_frame(width, height, x, y) + self.assertIsNone(result["error"]) + self.assertEqual(result["frames"], 1) + + def test_native_frame_extent_cannot_exceed_buffer(self): + for width, height in ((501, 400), (500, 401), (500.01, 400)): + with self.subTest(width=width, height=height): + result = self.check_frame(width, height) + self.assertIsNotNone(result["error"]) + self.assertEqual(result["frames"], 0) + + def test_native_frame_content_cannot_extend_outside_buffer(self): + for x, y in ((-1, 0), (0, -1), (1, 0), (0, 1)): + with self.subTest(x=x, y=y): + result = self.check_frame(500, 400, x, y) + self.assertIsNotNone(result["error"]) + self.assertEqual(result["frames"], 0) + + def test_scaled_frame_stays_rejected(self): + result = self.check_frame(400, 320, content_scale=0.8) + self.assertIn("scaled", result["error"]) + self.assertEqual(result["frames"], 0) + + +class WindowCaptureTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.build = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.build.cleanup) + cls.recorder = Path(cls.build.name) / "recorder" + subprocess.run( + ["xcrun", "swiftc", "-parse-as-library", "-warnings-as-errors", "-O", + str(RECORDER_SOURCE), "-o", str(cls.recorder)], check=True, + ) + + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) + self.output = self.path / "a frame.png" + self.log = self.path / "arguments.json" + self.environment = dict(os.environ) + self.environment.update( + PATH=f"{self.path}:{os.environ['PATH']}", + ETAF_GUI_CAPTURE_PID=str(os.getpid()), + CAPTURE_TEST_LOG=str(self.log), + CAPTURE_TEST_ID="123", + ) + self.script("osascript", "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$CAPTURE_TEST_ID\"\n") + self.capture = self.script( + "screencapture", + "#!/usr/bin/env python3\n" + "import json, os, pathlib, sys\n" + "pathlib.Path(os.environ['CAPTURE_TEST_LOG']).write_text(json.dumps(sys.argv[1:]))\n" + "pathlib.Path(sys.argv[-1]).write_bytes(b'capture stub')\n" + ) + self.environment["ETAF_GUI_SCREENCAPTURE"] = str(self.capture) + + def script(self, name, contents): + path = self.path / name + path.write_text(contents) + path.chmod(0o755) + return path + + def run_capture(self, *arguments): + return subprocess.run( + [str(CAPTURE), *arguments], env=self.environment, + capture_output=True, text=True, check=False, + ) + + def test_screenshot_captures_exact_owned_window_without_shadow(self): + result = self.run_capture("-x", str(self.output)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(self.log.read_text()), + ["-l", "123", "-o", "-x", str(self.output)]) + + def test_screenshot_resolves_replacement_window_for_next_capture(self): + for window_id in ("123", "456"): + self.environment["CAPTURE_TEST_ID"] = window_id + result = self.run_capture("-x", str(self.output)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(self.log.read_text())[1], window_id) + + def test_missing_owner_fails_without_invoking_capture(self): + self.script("osascript", "#!/bin/sh\nexit 1\n") + result = self.run_capture("-x", str(self.output)) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.log.exists()) + + def test_invalid_pid_fails_without_invoking_capture(self): + self.environment["ETAF_GUI_CAPTURE_PID"] = "1; echo unsafe" + result = self.run_capture("-x", str(self.output)) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.log.exists()) + + def test_window_replacement_during_capture_rejects_image(self): + counter = self.path / "counter" + self.script("osascript", f'''#!/bin/sh +cat >/dev/null +if [ -f '{counter}' ]; then echo 456; else touch '{counter}'; echo 123; fi +''') + result = self.run_capture("-x", str(self.output)) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + + def test_video_rejects_missing_or_invalid_owner_before_capture(self): + for arguments in ([], ["0", str(self.output), "ready"], + ["123; echo unsafe", str(self.output), "ready"]): + result = subprocess.run( + [str(self.recorder), *arguments], + env=self.environment, capture_output=True, check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.log.exists()) + + def run_recorder_lifecycle(self, recorder): + source = RUNNER.read_text().rsplit('[ "$#" -ge 1 ] || usage', 1)[0] + self.environment.update(TEST_RECORDER=str(recorder), TEST_RUN_DIR=str(self.path)) + return subprocess.run( + ["sh", "-c", source + ''' +GUI_RUN_DIR=$TEST_RUN_DIR +GUI_RECORDER=$TEST_RECORDER +GUI_DAEMON_PID=$ETAF_GUI_CAPTURE_PID +start_recorder +stop_recorder +'''], env=self.environment, capture_output=True, text=True, timeout=10, check=False, + ) + + def test_runner_waits_for_verified_frame_and_stops_owned_recorder(self): + recorder = self.script( + "recorder-stub", "#!/usr/bin/env python3\n" + "import json, os, pathlib, signal, sys\n" + "signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))\n" + "pathlib.Path(os.environ['CAPTURE_TEST_LOG']).write_text(json.dumps(sys.argv[1:]))\n" + "pathlib.Path(sys.argv[3]).write_text('verified frame')\n" + "signal.pause()\n", + ) + result = self.run_recorder_lifecycle(recorder) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(self.log.read_text()), + [str(os.getpid()), str(self.path / "recording.mov"), + str(self.path / "recorder.ready")]) + + def test_runner_rejects_recording_exit_before_ready(self): + recorder = self.script("recorder-stub", "#!/bin/sh\nexit 1\n") + result = self.run_recorder_lifecycle(recorder) + self.assertNotEqual(result.returncode, 0) + self.assertIn("exited before", result.stderr) + + def test_selector_rejects_other_owners_helpers_and_ambiguous_frames(self): + source = CAPTURE.read_text().split("<<'JXA'\n", 1)[1].split("\nJXA", 1)[0] + source = source[source.index("function run(argv)"):] + + def window(identifier, **overrides): + result = dict(kCGWindowNumber=identifier, kCGWindowOwnerPID=123, + kCGWindowOwnerName="Emacs", kCGWindowLayer=0, + kCGWindowAlpha=1, kCGWindowName="*target*", + kCGWindowBounds=dict(Width=1000, Height=800)) + result.update(overrides) + return result + + def select(windows): + # Evaluate the actual JXA selector against a fixed WindowServer + # inventory. The test does not enumerate or capture live windows. + fixture = ( + "var ObjC = {deepUnwrap: function(x) {return x;}, " + "castRefToObject: function(x) {return x;}};\n" + "var $ = {CGWindowListCopyWindowInfo: function() {return " + + json.dumps(windows) + ";}};\n" + ) + script = source.replace("function run(argv) {", + "function run(argv) {\n" + fixture, 1) + return subprocess.run( + ["/usr/bin/osascript", "-l", "JavaScript", "-", "123"], + input=script, capture_output=True, text=True, check=False, + ) + + candidates = [ + window(1, kCGWindowOwnerPID=456), + window(2, kCGWindowOwnerName="Other"), + window(3, kCGWindowName=""), + window(4, kCGWindowAlpha=0), + window(5, kCGWindowLayer=1), + window(6, kCGWindowBounds=dict(Width=1000, Height=24)), + window(7), + ] + result = select(candidates) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "7") + self.assertNotEqual(select(candidates + [window(8)]).returncode, 0) + self.assertNotEqual(select(candidates[:5]).returncode, 0) + + +if __name__ == "__main__": + unittest.main()