🧪 test: add JWT secret rotation BDD scenarios and step implementations (#12)
✨ merge: implement JWT secret rotation with BDD scenario isolation - Implement JWT secret rotation mechanism (closes #8) - Add per-scenario state isolation for BDD tests (closes #14) - Validate password reset workflow via BDD tests (closes #7) - Fix port conflicts in test validation - Add state tracer for debugging test execution - Document BDD isolation strategies in ADR 0025 - Fix PostgreSQL configuration environment variables Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai> Co-authored-by: Gabriel Radureau <arcodange@gmail.com> Co-committed-by: Gabriel Radureau <arcodange@gmail.com>
This commit was merged in pull request #12.
This commit is contained in:
263
scripts/validate-test-suite.sh
Executable file
263
scripts/validate-test-suite.sh
Executable file
@@ -0,0 +1,263 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test Suite Validation Script
|
||||
# Runs tests N times with separate unit and BDD test phases
|
||||
# Usage: ./scripts/validate-test-suite.sh [N] [OPTIONS]
|
||||
# N - Number of times to run tests (default: 20)
|
||||
# OPTIONS:
|
||||
# --parallel - Run feature tests in parallel
|
||||
# --count=C - Override -count flag for go test (default: same as N)
|
||||
# --quick - Run only core tests (skip @flaky)
|
||||
# --features=X - Test specific features only (comma-separated)
|
||||
|
||||
set -e
|
||||
|
||||
# Default values
|
||||
RUN_COUNT=${1:-20}
|
||||
GOTEST_COUNT=""
|
||||
PARALLEL=false
|
||||
QUICK=false
|
||||
FEATURES_FILTER=""
|
||||
|
||||
# Parse arguments
|
||||
shift
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--parallel)
|
||||
PARALLEL=true
|
||||
shift
|
||||
;;
|
||||
--count=*)
|
||||
GOTEST_COUNT="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
--quick)
|
||||
QUICK=true
|
||||
shift
|
||||
;;
|
||||
--features=*)
|
||||
FEATURES_FILTER="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Use GOTEST_COUNT if set, otherwise use RUN_COUNT
|
||||
if [ -z "$GOTEST_COUNT" ]; then
|
||||
GOTEST_COUNT=$RUN_COUNT
|
||||
fi
|
||||
|
||||
SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Temporary files
|
||||
UNIT_FAILURE_LOG=$(mktemp)
|
||||
BDD_FAILURE_LOG=$(mktemp)
|
||||
SUMMARY_REPORT=$(mktemp)
|
||||
|
||||
# Cleanup temporary files on exit
|
||||
cleanup() {
|
||||
rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "🧪 Test Suite Validation Script"
|
||||
echo "=============================="
|
||||
echo "Runs: $RUN_COUNT"
|
||||
echo "Unit Tests: ./cmd/... ./pkg/..."
|
||||
echo "BDD Tests: ./features/..."
|
||||
echo "Date: $(date)"
|
||||
echo
|
||||
|
||||
# Initialize counters
|
||||
UNIT_SUCCESS=0
|
||||
UNIT_FAILURE=0
|
||||
BDD_SUCCESS=0
|
||||
BDD_FAILURE=0
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
echo "Starting validation runs..."
|
||||
echo
|
||||
|
||||
# Main validation loop
|
||||
for (( run=1; run<=$RUN_COUNT; run++ )); do
|
||||
echo "Run $run/$RUN_COUNT..."
|
||||
|
||||
# ===== UNIT TESTS =====
|
||||
echo " 🧪 Unit tests..."
|
||||
go clean -testcache > /dev/null 2>&1
|
||||
|
||||
set +e # Temporarily disable exit on error
|
||||
UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1)
|
||||
UNIT_EXIT_CODE=$?
|
||||
set -e # Re-enable exit on error
|
||||
|
||||
if [ $UNIT_EXIT_CODE -eq 0 ]; then
|
||||
echo " ✅ Passed"
|
||||
((UNIT_SUCCESS++))
|
||||
else
|
||||
echo " ❌ Failed"
|
||||
((UNIT_FAILURE++))
|
||||
|
||||
# Extract detailed unit test failures
|
||||
echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG"
|
||||
echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG"
|
||||
echo "---" >> "$UNIT_FAILURE_LOG"
|
||||
fi
|
||||
|
||||
# ===== BDD TESTS =====
|
||||
echo " 🧪 BDD tests..."
|
||||
go clean -testcache > /dev/null 2>&1
|
||||
|
||||
# Set environment variables for consistent BDD test behavior
|
||||
export DLC_DATABASE_HOST=localhost
|
||||
export DLC_DATABASE_PORT=5432
|
||||
export DLC_DATABASE_USER=postgres
|
||||
export DLC_DATABASE_PASSWORD=postgres
|
||||
export DLC_DATABASE_NAME=dance_lessons_coach_test
|
||||
|
||||
export BDD_SCHEMA_ISOLATION=true
|
||||
|
||||
# Build feature test arguments
|
||||
FEATURE_PACKAGES=("config" "auth" "greet" "health" "jwt")
|
||||
|
||||
# Filter features if specified
|
||||
if [ -n "$FEATURES_FILTER" ]; then
|
||||
IFS=',' read -ra FILTERED_FEATURES <<< "$FEATURES_FILTER"
|
||||
ALL_FEATURES=("config" "auth" "greet" "health" "jwt")
|
||||
FEATURE_PACKAGES=()
|
||||
for feat in "${FILTERED_FEATURES[@]}"; do
|
||||
if [[ " ${ALL_FEATURES[@]} " =~ " ${feat} " ]]; then
|
||||
FEATURE_PACKAGES+=("$feat")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Build go test command for features
|
||||
FEATURE_TESTS=""
|
||||
for feat in "${FEATURE_PACKAGES[@]}"; do
|
||||
FEATURE_TESTS+="./features/$feat "
|
||||
done
|
||||
|
||||
# Set tags for quick mode
|
||||
if [ "$QUICK" = true ]; then
|
||||
export GODOG_TAGS="~@flaky && ~@todo && ~@skip"
|
||||
fi
|
||||
|
||||
set +e # Temporarily disable exit on error
|
||||
# Force sequential package testing and use fixed port to prevent race conditions
|
||||
FIXED_TEST_PORT=true BDD_SCHEMA_ISOLATION=true go test ${FEATURE_TESTS} -count=$GOTEST_COUNT -v -p 1 2>&1 | tee /tmp/bdd_raw_$$.txt | grep -v '^{"level"' > /tmp/bdd_output_$$.txt && BDD_OUTPUT=$(cat /tmp/bdd_output_$$.txt) && rm -f /tmp/bdd_output_$$.txt /tmp/bdd_raw_$$.txt || true
|
||||
BDD_EXIT_CODE=${PIPESTATUS[0]}
|
||||
set -e # Re-enable exit on error
|
||||
|
||||
if [ $BDD_EXIT_CODE -eq 0 ]; then
|
||||
echo " ✅ Passed"
|
||||
((BDD_SUCCESS++))
|
||||
else
|
||||
echo " ❌ Failed"
|
||||
((BDD_FAILURE++))
|
||||
|
||||
# Extract detailed BDD test failures with actual test names
|
||||
echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG"
|
||||
echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG"
|
||||
echo "---" >> "$BDD_FAILURE_LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
|
||||
echo "Validation Complete"
|
||||
echo "=================="
|
||||
echo "Total Runs: $RUN_COUNT"
|
||||
echo "Unit Tests:"
|
||||
echo -e " Success: ${GREEN}$UNIT_SUCCESS${NC}"
|
||||
echo -e " Failures: ${RED}$UNIT_FAILURE${NC}"
|
||||
echo -e "BDD Tests:"
|
||||
echo -e " Success: ${GREEN}$BDD_SUCCESS${NC}"
|
||||
echo -e " Failures: ${RED}$BDD_FAILURE${NC}"
|
||||
echo "Duration: $DURATION seconds"
|
||||
echo
|
||||
|
||||
# Check overall success
|
||||
TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE))
|
||||
|
||||
if [ $TOTAL_FAILURES -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ All tests passed successfully!${NC}"
|
||||
echo "Test suite is stable and ready for production"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}❌ Some tests failed during validation${NC}"
|
||||
echo
|
||||
|
||||
# Process unit test failures
|
||||
if [ -s "$UNIT_FAILURE_LOG" ]; then
|
||||
echo "Unit Test Failures:"
|
||||
echo "=================="
|
||||
|
||||
# Count unit test failures
|
||||
UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn)
|
||||
if [ -n "$UNIT_FAILURES" ]; then
|
||||
echo "$UNIT_FAILURES"
|
||||
else
|
||||
echo " None (check log for details)"
|
||||
fi
|
||||
|
||||
echo
|
||||
fi
|
||||
|
||||
# Process BDD test failures
|
||||
if [ -s "$BDD_FAILURE_LOG" ]; then
|
||||
echo "BDD Test Failures:"
|
||||
echo "==============="
|
||||
|
||||
# Count BDD test failures with granularity
|
||||
BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | \
|
||||
grep -v "dance-lessons-coach/features" | \
|
||||
grep -v "^[0-9].*FAIL" | \
|
||||
grep "/" | \
|
||||
sort | uniq -c | sort -rn)
|
||||
if [ -n "$BDD_FAILURES" ]; then
|
||||
echo "Summary:"
|
||||
while IFS= read -r line; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//')
|
||||
echo " $count x $test"
|
||||
done <<< "$BDD_FAILURES"
|
||||
else
|
||||
echo " None (check log for details)"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Detailed BDD Failure Log (first 20 lines):"
|
||||
echo "=========================================="
|
||||
# Show only the relevant failure lines with actual test names
|
||||
# Filter out non-specific failures and test suite lines
|
||||
grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | \
|
||||
grep -v "dance-lessons-coach/features" | \
|
||||
grep -v "^[0-9].*FAIL" | \
|
||||
grep "/" | \
|
||||
head -20
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Recommendations:"
|
||||
echo " 1. Investigate unit test failures first (faster to fix)"
|
||||
echo " 2. Check for race conditions in failing tests"
|
||||
echo " 3. Review test dependencies and isolation (schema/database isolation)"
|
||||
echo " 4. Run individual failing tests with: FIXED_TEST_PORT=true go test ./features -v -run TestBDD/Name"
|
||||
echo " 5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags"
|
||||
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user