test: serialize repeatable performance gates
This commit is contained in:
Kinneyzhang
parent
3ed2d39f4c
commit
d237b9bc67
14
Makefile
14
Makefile
@ -8,7 +8,7 @@ EXAMPLES := $(ENTRY_EXAMPLES)
|
||||
EXAMPLE_ELC := $(EXAMPLES:.el=.elc)
|
||||
TEST_FILES := $(wildcard tests/*-tests.el)
|
||||
|
||||
.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-regressions \
|
||||
.PHONY: all compile examples-read test perf perf-prepare perf-evaluator perf-repeatability perf-regressions \
|
||||
perf-check gui-doctor gui-research gui-flex gui-grid gui-all \
|
||||
check checkdoc load clean
|
||||
|
||||
@ -55,13 +55,21 @@ perf-prepare: compile
|
||||
# Run only the already-built product. This separate target lets an absolute
|
||||
# max-latency sample start after compilation CPU activity has settled.
|
||||
perf-evaluator:
|
||||
scripts/with-performance-lock.sh env \
|
||||
EBOX_NATIVE_REFLOW_MODULE_PATH="$(EBOX_NATIVE_RELEASE_DIR)" \
|
||||
$(EMACS) -Q --batch $(LOAD_PATH) \
|
||||
"$(EMACS)" -Q --batch $(LOAD_PATH) \
|
||||
--eval '(setq load-prefer-newer t native-comp-jit-compilation nil)' \
|
||||
--eval '(setq ebox-native-reflow-module-path (getenv "EBOX_NATIVE_REFLOW_MODULE_PATH"))' \
|
||||
-l scripts/benchmark-research-shelf.el \
|
||||
-f etaf-performance-evaluator-batch
|
||||
|
||||
# A final hard-max claim must survive three complete independent sample sets.
|
||||
# The lock above prevents parallel reviewers from benchmarking one another.
|
||||
perf-repeatability: perf-prepare
|
||||
$(MAKE) perf-evaluator EMACS="$(EMACS)"
|
||||
$(MAKE) perf-evaluator EMACS="$(EMACS)"
|
||||
$(MAKE) perf-evaluator EMACS="$(EMACS)"
|
||||
|
||||
# Run package regressions in dependency order. Keeping them in one recipe
|
||||
# prevents parallel make from cleaning/recompiling a sibling while another
|
||||
# package is loading its compiled dependencies.
|
||||
@ -77,7 +85,7 @@ perf-regressions:
|
||||
# Measure absolute latency before the CPU-heavy regression graph. Regressions
|
||||
# still gate the same target, but cannot thermally contaminate product samples.
|
||||
perf-check:
|
||||
$(MAKE) perf EMACS="$(EMACS)"
|
||||
$(MAKE) perf-repeatability EMACS="$(EMACS)"
|
||||
$(MAKE) perf-regressions EMACS="$(EMACS)"
|
||||
|
||||
gui-doctor:
|
||||
|
||||
63
scripts/with-performance-lock.sh
Executable file
63
scripts/with-performance-lock.sh
Executable file
@ -0,0 +1,63 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
echo "usage: $0 COMMAND [ARG ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
PERF_LOCK_DIR=${ETAF_PERF_LOCK_DIR:-${TMPDIR:-/tmp}/etaf-performance-evaluator.lock}
|
||||
PERF_LOCK_WAIT_SECONDS=${ETAF_PERF_LOCK_WAIT_SECONDS:-180}
|
||||
|
||||
case $PERF_LOCK_WAIT_SECONDS in
|
||||
''|*[!0-9]*)
|
||||
echo "ETAF_PERF_LOCK_WAIT_SECONDS must be a non-negative integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
PERF_LOCK_ACQUIRED=false
|
||||
PERF_LOCK_WAITED=0
|
||||
|
||||
release_lock() {
|
||||
if [ "$PERF_LOCK_ACQUIRED" = true ]; then
|
||||
rm -f "$PERF_LOCK_DIR/pid"
|
||||
rmdir "$PERF_LOCK_DIR" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
trap release_lock EXIT HUP INT TERM
|
||||
|
||||
while ! mkdir "$PERF_LOCK_DIR" 2>/dev/null; do
|
||||
PERF_LOCK_OWNER=
|
||||
if [ -r "$PERF_LOCK_DIR/pid" ]; then
|
||||
IFS= read -r PERF_LOCK_OWNER < "$PERF_LOCK_DIR/pid" || PERF_LOCK_OWNER=
|
||||
fi
|
||||
case $PERF_LOCK_OWNER in
|
||||
''|*[!0-9]*) PERF_LOCK_OWNER= ;;
|
||||
esac
|
||||
|
||||
if [ -n "$PERF_LOCK_OWNER" ] &&
|
||||
! kill -0 "$PERF_LOCK_OWNER" 2>/dev/null; then
|
||||
rm -f "$PERF_LOCK_DIR/pid"
|
||||
rmdir "$PERF_LOCK_DIR" 2>/dev/null || true
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ "$PERF_LOCK_WAITED" -ge "$PERF_LOCK_WAIT_SECONDS" ]; then
|
||||
echo "performance evaluator lock timed out: $PERF_LOCK_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERF_LOCK_WAITED" -eq 0 ]; then
|
||||
echo "performance evaluator waiting for exclusive host slot" >&2
|
||||
fi
|
||||
sleep 1
|
||||
PERF_LOCK_WAITED=$((PERF_LOCK_WAITED + 1))
|
||||
done
|
||||
|
||||
PERF_LOCK_ACQUIRED=true
|
||||
printf '%s\n' "$$" > "$PERF_LOCK_DIR/pid"
|
||||
echo "performance evaluator acquired exclusive host slot"
|
||||
|
||||
"$@"
|
||||
@ -171,21 +171,45 @@ frame."
|
||||
(should (string-match-p (regexp-quote dependency) makefile)))))
|
||||
|
||||
(ert-deftest etaf-playground-performance-gate-runs-before-regressions ()
|
||||
"CPU-heavy regressions must not contaminate absolute latency samples."
|
||||
"Repeatable exclusive latency gates run before CPU-heavy regressions."
|
||||
(let* ((makefile (with-temp-buffer
|
||||
(insert-file-contents "Makefile")
|
||||
(buffer-string)))
|
||||
(target (string-match "^perf-check:" makefile))
|
||||
(evaluator (and target
|
||||
(string-match "$(MAKE) perf " makefile target)))
|
||||
(repeatability
|
||||
(and target
|
||||
(string-match "$(MAKE) perf-repeatability " makefile target)))
|
||||
(regressions (and target
|
||||
(string-match "perf-regressions" makefile target))))
|
||||
(should target)
|
||||
(should evaluator)
|
||||
(should repeatability)
|
||||
(should regressions)
|
||||
(should (< evaluator regressions))
|
||||
(should (< repeatability regressions))
|
||||
(should (string-match-p "^perf-prepare: compile" makefile))
|
||||
(should (string-match-p "^perf-evaluator:" makefile))))
|
||||
(should (string-match-p "^perf-evaluator:" makefile))
|
||||
(should (string-match-p "^perf-repeatability: perf-prepare" makefile))
|
||||
(should (string-match-p "scripts/with-performance-lock.sh" makefile))
|
||||
(let* ((start (string-match "^perf-repeatability:" makefile))
|
||||
(end (string-match "^# Run package regressions" makefile start))
|
||||
(body (substring makefile start end))
|
||||
(offset 0)
|
||||
(count 0))
|
||||
(while (string-match "$(MAKE) perf-evaluator" body offset)
|
||||
(setq count (1+ count)
|
||||
offset (match-end 0)))
|
||||
(should (= count 3)))))
|
||||
|
||||
(ert-deftest etaf-playground-performance-lock-is-exclusive-and-stale-safe ()
|
||||
"The evaluator wrapper serializes runs and clears stale owner locks."
|
||||
(let ((script (with-temp-buffer
|
||||
(insert-file-contents "scripts/with-performance-lock.sh")
|
||||
(buffer-string))))
|
||||
(dolist (contract '("mkdir \"$PERF_LOCK_DIR\""
|
||||
"kill -0 \"$PERF_LOCK_OWNER\""
|
||||
"rmdir \"$PERF_LOCK_DIR\""
|
||||
"performance evaluator waiting for exclusive host slot"))
|
||||
(should (string-match-p (regexp-quote contract) script)))
|
||||
(should-not (string-match-p "rm -rf" script))))
|
||||
|
||||
(ert-deftest etaf-playground-static-reader-is-inert-and-strict ()
|
||||
"Read pair structure as inert data and reject executable AST nodes."
|
||||
|
||||
Loading…
Reference in New Issue
Block a user