From 0e14b339c08bc18eda09b374ae66a7098beb29e4 Mon Sep 17 00:00:00 2001 From: Kinneyzhang Date: Tue, 1 Sep 2026 17:18:15 +0800 Subject: [PATCH] test: enforce trace overhead p95 budget --- scripts/benchmark-research-shelf.el | 27 +++++++++++++++++++------ tests/benchmark-research-shelf-tests.el | 13 ++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/scripts/benchmark-research-shelf.el b/scripts/benchmark-research-shelf.el index c28ef57..968a9b6 100644 --- a/scripts/benchmark-research-shelf.el +++ b/scripts/benchmark-research-shelf.el @@ -353,6 +353,23 @@ RESET-REF defaults to the first Research Shelf row." failures)) (nreverse failures)))) +(defun etaf-performance-evaluator--trace-overhead-failures (overhead) + "Return hard trace budget failures from OVERHEAD statistics." + (let* ((signed (plist-get overhead :signed)) + (p50 (plist-get signed :p50)) + (p95 (plist-get signed :p95)) + failures) + (when (> (abs p50) etaf-performance-evaluator-overhead-budget-ms) + (push (format "trace signed |p50| %.3fms > %.3fms" + (abs p50) + etaf-performance-evaluator-overhead-budget-ms) + failures)) + (when (> p95 etaf-performance-evaluator-overhead-budget-ms) + (push (format "trace signed p95 %.3fms > %.3fms" + p95 etaf-performance-evaluator-overhead-budget-ms) + failures)) + (nreverse failures))) + (defun etaf-performance-evaluator--trace-scenario (runtime label function &optional verify) "Run FUNCTION and return the one new operation for LABEL. @@ -900,12 +917,10 @@ ETAF generation and preserve RUNTIME's mounted surface in BUFFER." (setq overhead (etaf-performance-evaluator--measure-overhead runtime buffer) failures (etaf-performance-evaluator--latency-failures results)) - (when (> (abs (plist-get (plist-get overhead :signed) :p50)) - etaf-performance-evaluator-overhead-budget-ms) - (push (format "trace signed |p50| %.3fms > %.3fms" - (abs (plist-get (plist-get overhead :signed) :p50)) - etaf-performance-evaluator-overhead-budget-ms) - failures)) + (setq failures + (nconc failures + (etaf-performance-evaluator--trace-overhead-failures + overhead))) (if failures (let ((failure-count (length failures))) (princ "etaf-cross-package-perf FAIL\n") diff --git a/tests/benchmark-research-shelf-tests.el b/tests/benchmark-research-shelf-tests.el index 32f276e..ceacdf4 100644 --- a/tests/benchmark-research-shelf-tests.el +++ b/tests/benchmark-research-shelf-tests.el @@ -12,6 +12,8 @@ "../scripts/benchmark-research-shelf") (declare-function etaf-performance-evaluator--latency-failures "../scripts/benchmark-research-shelf") +(declare-function etaf-performance-evaluator--trace-overhead-failures + "../scripts/benchmark-research-shelf") (declare-function etaf-performance-evaluator--verify-runtime-accelerator "../scripts/benchmark-research-shelf") (declare-function etaf-performance-evaluator--verify-environment @@ -89,6 +91,17 @@ (dolist (failure failures) (should (string-match-p "> 50.000ms" failure))))) +(ert-deftest etaf-performance-evaluator-gates-trace-overhead-p95 () + "Apply the 2ms trace budget to signed p95 as well as median bias." + (let ((failures + (etaf-performance-evaluator--trace-overhead-failures + '(:signed (:p50 0.4 :p95 2.5))))) + (should (= 1 (length failures))) + (should (string-match-p "p95 2.500ms > 2.000ms" (car failures)))) + (should-not + (etaf-performance-evaluator--trace-overhead-failures + '(:signed (:p50 0.4 :p95 1.9))))) + (ert-deftest etaf-performance-evaluator-trace-scenario-retains-one-operation () "The trace driver uses the public Runtime recorder and preserves results." (let ((buffer-name (generate-new-buffer-name " *etaf-driver-trace-test*"))