From d237b9bc673de117e0fa5fca7b210cec6424d935 Mon Sep 17 00:00:00 2001 From: Kinneyzhang Date: Mon, 31 Aug 2026 16:57:31 +0800 Subject: [PATCH] test: serialize repeatable performance gates --- Makefile | 14 +++++-- scripts/with-performance-lock.sh | 63 ++++++++++++++++++++++++++++++++ tests/etaf-playground-tests.el | 36 +++++++++++++++--- 3 files changed, 104 insertions(+), 9 deletions(-) create mode 100755 scripts/with-performance-lock.sh diff --git a/Makefile b/Makefile index 701d90b..9f454dc 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ EXAMPLES := $(ENTRY_EXAMPLES) EXAMPLE_ELC := $(EXAMPLES:.el=.elc) TEST_FILES := $(wildcard tests/*-tests.el) -.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-regressions \ +.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-repeatability perf-regressions \ perf-check gui-doctor gui-research gui-flex gui-grid gui-all \ check checkdoc load clean @@ -55,13 +55,21 @@ perf-prepare: compile # Run only the already-built product. This separate target lets an absolute # max-latency sample start after compilation CPU activity has settled. perf-evaluator: + scripts/with-performance-lock.sh env \ EBOX_NATIVE_REFLOW_MODULE_PATH="$(EBOX_NATIVE_RELEASE_DIR)" \ - $(EMACS) -Q --batch $(LOAD_PATH) \ + "$(EMACS)" -Q --batch $(LOAD_PATH) \ --eval '(setq load-prefer-newer t native-comp-jit-compilation nil)' \ --eval '(setq ebox-native-reflow-module-path (getenv "EBOX_NATIVE_REFLOW_MODULE_PATH"))' \ -l scripts/benchmark-research-shelf.el \ -f etaf-performance-evaluator-batch +# A final hard-max claim must survive three complete independent sample sets. +# The lock above prevents parallel reviewers from benchmarking one another. +perf-repeatability: perf-prepare + $(MAKE) perf-evaluator EMACS="$(EMACS)" + $(MAKE) perf-evaluator EMACS="$(EMACS)" + $(MAKE) perf-evaluator EMACS="$(EMACS)" + # Run package regressions in dependency order. Keeping them in one recipe # prevents parallel make from cleaning/recompiling a sibling while another # package is loading its compiled dependencies. @@ -77,7 +85,7 @@ perf-regressions: # Measure absolute latency before the CPU-heavy regression graph. Regressions # still gate the same target, but cannot thermally contaminate product samples. perf-check: - $(MAKE) perf EMACS="$(EMACS)" + $(MAKE) perf-repeatability EMACS="$(EMACS)" $(MAKE) perf-regressions EMACS="$(EMACS)" gui-doctor: diff --git a/scripts/with-performance-lock.sh b/scripts/with-performance-lock.sh new file mode 100755 index 0000000..1a385cd --- /dev/null +++ b/scripts/with-performance-lock.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +set -eu + +if [ "$#" -eq 0 ]; then + echo "usage: $0 COMMAND [ARG ...]" >&2 + exit 2 +fi + +PERF_LOCK_DIR=${ETAF_PERF_LOCK_DIR:-${TMPDIR:-/tmp}/etaf-performance-evaluator.lock} +PERF_LOCK_WAIT_SECONDS=${ETAF_PERF_LOCK_WAIT_SECONDS:-180} + +case $PERF_LOCK_WAIT_SECONDS in + ''|*[!0-9]*) + echo "ETAF_PERF_LOCK_WAIT_SECONDS must be a non-negative integer" >&2 + exit 2 + ;; +esac + +PERF_LOCK_ACQUIRED=false +PERF_LOCK_WAITED=0 + +release_lock() { + if [ "$PERF_LOCK_ACQUIRED" = true ]; then + rm -f "$PERF_LOCK_DIR/pid" + rmdir "$PERF_LOCK_DIR" 2>/dev/null || true + fi +} + +trap release_lock EXIT HUP INT TERM + +while ! mkdir "$PERF_LOCK_DIR" 2>/dev/null; do + PERF_LOCK_OWNER= + if [ -r "$PERF_LOCK_DIR/pid" ]; then + IFS= read -r PERF_LOCK_OWNER < "$PERF_LOCK_DIR/pid" || PERF_LOCK_OWNER= + fi + case $PERF_LOCK_OWNER in + ''|*[!0-9]*) PERF_LOCK_OWNER= ;; + esac + + if [ -n "$PERF_LOCK_OWNER" ] && + ! kill -0 "$PERF_LOCK_OWNER" 2>/dev/null; then + rm -f "$PERF_LOCK_DIR/pid" + rmdir "$PERF_LOCK_DIR" 2>/dev/null || true + continue + fi + + if [ "$PERF_LOCK_WAITED" -ge "$PERF_LOCK_WAIT_SECONDS" ]; then + echo "performance evaluator lock timed out: $PERF_LOCK_DIR" >&2 + exit 1 + fi + if [ "$PERF_LOCK_WAITED" -eq 0 ]; then + echo "performance evaluator waiting for exclusive host slot" >&2 + fi + sleep 1 + PERF_LOCK_WAITED=$((PERF_LOCK_WAITED + 1)) +done + +PERF_LOCK_ACQUIRED=true +printf '%s\n' "$$" > "$PERF_LOCK_DIR/pid" +echo "performance evaluator acquired exclusive host slot" + +"$@" diff --git a/tests/etaf-playground-tests.el b/tests/etaf-playground-tests.el index 1dbd6ad..7b988dd 100644 --- a/tests/etaf-playground-tests.el +++ b/tests/etaf-playground-tests.el @@ -171,21 +171,45 @@ frame." (should (string-match-p (regexp-quote dependency) makefile))))) (ert-deftest etaf-playground-performance-gate-runs-before-regressions () - "CPU-heavy regressions must not contaminate absolute latency samples." + "Repeatable exclusive latency gates run before CPU-heavy regressions." (let* ((makefile (with-temp-buffer (insert-file-contents "Makefile") (buffer-string))) (target (string-match "^perf-check:" makefile)) - (evaluator (and target - (string-match "$(MAKE) perf " makefile target))) + (repeatability + (and target + (string-match "$(MAKE) perf-repeatability " makefile target))) (regressions (and target (string-match "perf-regressions" makefile target)))) (should target) - (should evaluator) + (should repeatability) (should regressions) - (should (< evaluator regressions)) + (should (< repeatability regressions)) (should (string-match-p "^perf-prepare: compile" makefile)) - (should (string-match-p "^perf-evaluator:" makefile)))) + (should (string-match-p "^perf-evaluator:" makefile)) + (should (string-match-p "^perf-repeatability: perf-prepare" makefile)) + (should (string-match-p "scripts/with-performance-lock.sh" makefile)) + (let* ((start (string-match "^perf-repeatability:" makefile)) + (end (string-match "^# Run package regressions" makefile start)) + (body (substring makefile start end)) + (offset 0) + (count 0)) + (while (string-match "$(MAKE) perf-evaluator" body offset) + (setq count (1+ count) + offset (match-end 0))) + (should (= count 3))))) + +(ert-deftest etaf-playground-performance-lock-is-exclusive-and-stale-safe () + "The evaluator wrapper serializes runs and clears stale owner locks." + (let ((script (with-temp-buffer + (insert-file-contents "scripts/with-performance-lock.sh") + (buffer-string)))) + (dolist (contract '("mkdir \"$PERF_LOCK_DIR\"" + "kill -0 \"$PERF_LOCK_OWNER\"" + "rmdir \"$PERF_LOCK_DIR\"" + "performance evaluator waiting for exclusive host slot")) + (should (string-match-p (regexp-quote contract) script))) + (should-not (string-match-p "rm -rf" script)))) (ert-deftest etaf-playground-static-reader-is-inert-and-strict () "Read pair structure as inert data and reject executable AST nodes."