#!/bin/bash # Test Suite Validation Script # Runs tests N times with separate unit and BDD test phases # Usage: ./scripts/validate-test-suite.sh [N] # N - Number of times to run tests (default: 20) set -e # Default values RUN_COUNT=${1:-20} SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Temporary files UNIT_FAILURE_LOG=$(mktemp) BDD_FAILURE_LOG=$(mktemp) SUMMARY_REPORT=$(mktemp) # Cleanup temporary files on exit cleanup() { rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT" } trap cleanup EXIT echo "๐Ÿงช Test Suite Validation Script" echo "==============================" echo "Runs: $RUN_COUNT" echo "Unit Tests: ./cmd/... ./pkg/..." echo "BDD Tests: ./features/..." echo "Date: $(date)" echo # Initialize counters UNIT_SUCCESS=0 UNIT_FAILURE=0 BDD_SUCCESS=0 BDD_FAILURE=0 START_TIME=$(date +%s) echo "Starting validation runs..." echo # Main validation loop for (( run=1; run<=$RUN_COUNT; run++ )); do echo "Run $run/$RUN_COUNT..." # ===== UNIT TESTS ===== echo " ๐Ÿงช Unit tests..." go clean -testcache > /dev/null 2>&1 set +e # Temporarily disable exit on error UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1) UNIT_EXIT_CODE=$? set -e # Re-enable exit on error if [ $UNIT_EXIT_CODE -eq 0 ]; then echo " โœ… Passed" ((UNIT_SUCCESS++)) else echo " โŒ Failed" ((UNIT_FAILURE++)) # Extract detailed unit test failures echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG" echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG" echo "---" >> "$UNIT_FAILURE_LOG" fi # ===== BDD TESTS ===== echo " ๐Ÿงช BDD tests..." go clean -testcache > /dev/null 2>&1 set +e # Temporarily disable exit on error BDD_OUTPUT=$(go test ./features/... -v 2>&1) BDD_EXIT_CODE=$? set -e # Re-enable exit on error if [ $BDD_EXIT_CODE -eq 0 ]; then echo " โœ… Passed" ((BDD_SUCCESS++)) else echo " โŒ Failed" ((BDD_FAILURE++)) # Extract detailed BDD test failures with actual test names echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG" echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG" echo "---" >> "$BDD_FAILURE_LOG" fi done echo END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) echo "Validation Complete" echo "==================" echo "Total Runs: $RUN_COUNT" echo "Unit Tests:" echo " Success: ${GREEN}$UNIT_SUCCESS${NC}" echo " Failures: ${RED}$UNIT_FAILURE${NC}" echo "BDD Tests:" echo " Success: ${GREEN}$BDD_SUCCESS${NC}" echo " Failures: ${RED}$BDD_FAILURE${NC}" echo "Duration: $DURATION seconds" echo # Check overall success TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE)) if [ $TOTAL_FAILURES -eq 0 ]; then echo "${GREEN}โœ… All tests passed successfully!${NC}" echo "Test suite is stable and ready for production" exit 0 else echo "${RED}โŒ Some tests failed during validation${NC}" echo # Process unit test failures if [ -s "$UNIT_FAILURE_LOG" ]; then echo "Unit Test Failures:" echo "==================" # Count unit test failures UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn) if [ -n "$UNIT_FAILURES" ]; then echo "$UNIT_FAILURES" else echo " None (check log for details)" fi echo fi # Process BDD test failures if [ -s "$BDD_FAILURE_LOG" ]; then echo "BDD Test Failures:" echo "================" # Count BDD test failures with granularity BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | sort | uniq -c | sort -rn) if [ -n "$BDD_FAILURES" ]; then echo "Summary:" while IFS= read -r line; do count=$(echo "$line" | awk '{print $1}') test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//') echo " $count ร— $test" done <<< "$BDD_FAILURES" else echo " None (check log for details)" fi echo echo "Detailed BDD Failure Log (first 20 lines):" echo "==========================================" # Show only the relevant failure lines with actual test names # Filter out non-specific failures (no '/' means not a specific test) grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | \ grep -v "dance-lessons-coach/features" | \ grep "/" | \ head -20 fi echo echo "Recommendations:" echo " 1. Mark flaky BDD tests with @flaky tag" echo " 2. Investigate unit test failures first (faster to fix)" echo " 3. Check for race conditions in failing tests" echo " 4. Run with FIXED_TEST_PORT=true for debugging" echo " 5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags" exit 1 fi