Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env bash
# Cold-tests which skill fires for a set of prompts, to catch trigger regressions on the
# perftest / perf-regression-triage / profiler-analysis boundary.
#
# ./trigger-tests.sh [results-dir]
#
# Exits 0 if every case routes as expected, 1 otherwise. Each run spawns a fresh headless
# session granted only the Skill tool: granting Read as well lets the model reach skill
# content without invoking Skill, which silently defeats the measurement.
#
# Advisory, not a gate. This measures model behaviour, so a case can fail for reasons
# unrelated to the skill descriptions (model update, nondeterminism). Re-run before
# concluding an edit broke routing, and do not wire it into CI.
#
# Add a case as "name|expected-skill|prompt", using <none> when nothing should fire.
set -uo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
RESULTS_DIR="${1:-$REPO_ROOT/artifacts/skill-trigger-tests/$(date +%Y%m%dT%H%M%S)}"
# Project skills only load for sessions rooted in the repo; without this every case
# silently reports <none> when the script is invoked from elsewhere.
cd "$REPO_ROOT" || { echo "cannot cd to $REPO_ROOT" >&2; exit 2; }
CASES=(
"pos-bug-live-framing|perf-regression-triage|https://bugzilla.mozilla.org/show_bug.cgi?id=2034476 my patch caused this regression, what do I do now"
"pos-alert-no-url|perf-regression-triage|Perfherder filed an alert saying my patch regressed speedometer3 by 12% on linux2404-64-shippable. Alert summary 49649. What now?"
"pos-perfcompare|perf-regression-triage|what does this PerfCompare result mean, my patch looks slower"
"neg-raptor-local|perftest|how do I run raptor speedometer3 locally"
"neg-alert-tests-on-try|perftest|how do I run the tests from alert summary 49649 on try"
"neg-mochitest-failure|<none>|this mochitest started failing on autoland"
)
command -v claude >/dev/null || { echo "claude not found in PATH" >&2; exit 2; }
command -v rg >/dev/null || { echo "rg not found in PATH; run ./mach bootstrap" >&2; exit 2; }
mkdir -p "$RESULTS_DIR"
echo "Running ${#CASES[@]} cases -> $RESULTS_DIR"
echo
for entry in "${CASES[@]}"; do
name="${entry%%|*}"
rest="${entry#*|}"
prompt="${rest#*|}"
claude -p "$prompt" \
--allowedTools Skill \
--output-format stream-json \
--verbose \
> "$RESULTS_DIR/$name.jsonl" 2>&1 &
done
wait
failures=0
for entry in "${CASES[@]}"; do
name="${entry%%|*}"
rest="${entry#*|}"
expected="${rest%%|*}"
fired=$(rg -o '"name":"Skill","input":\{"skill":"[^"]*"' "$RESULTS_DIR/$name.jsonl" 2>/dev/null \
| sed 's/.*"skill":"//; s/"$//' | sort -u | paste -sd, -)
[ -z "$fired" ] && fired="<none>"
if [ "$fired" = "$expected" ]; then
printf 'PASS %-24s %s\n' "$name" "$fired"
else
printf 'FAIL %-24s expected %s, got %s\n' "$name" "$expected" "$fired"
failures=$((failures + 1))
fi
done
echo
if [ "$failures" -eq 0 ]; then
echo "All ${#CASES[@]} cases routed as expected."
exit 0
fi
echo "$failures of ${#CASES[@]} cases mis-routed. Transcripts in $RESULTS_DIR"
exit 1