dance-lessons-coach/scripts/validate-test-suite.sh

#!/bin/bash

# Test Suite Validation Script
# Runs tests N times with separate unit and BDD test phases
# Usage: ./scripts/validate-test-suite.sh [N]
#   N          - Number of times to run tests (default: 20)

set -e

# Default values
RUN_COUNT=${1:-20}
SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Temporary files
UNIT_FAILURE_LOG=$(mktemp)
BDD_FAILURE_LOG=$(mktemp)
SUMMARY_REPORT=$(mktemp)

# Cleanup temporary files on exit
cleanup() {
    rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT"
}
trap cleanup EXIT

echo "🧪 Test Suite Validation Script"
echo "=============================="
echo "Runs: $RUN_COUNT"
echo "Unit Tests: ./cmd/... ./pkg/..."
echo "BDD Tests: ./features/..."
echo "Date: $(date)"
echo

# Initialize counters
UNIT_SUCCESS=0
UNIT_FAILURE=0
BDD_SUCCESS=0
BDD_FAILURE=0
START_TIME=$(date +%s)

echo "Starting validation runs..."
echo

# Main validation loop
for (( run=1; run<=$RUN_COUNT; run++ )); do
    echo "Run $run/$RUN_COUNT..."
    
    # ===== UNIT TESTS =====
    echo "  🧪 Unit tests..."
    go clean -testcache > /dev/null 2>&1
    
    set +e  # Temporarily disable exit on error
    UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1)
    UNIT_EXIT_CODE=$?
    set -e  # Re-enable exit on error
    
    if [ $UNIT_EXIT_CODE -eq 0 ]; then
        echo "    ✅ Passed"
        ((UNIT_SUCCESS++))
    else
        echo "    ❌ Failed"
        ((UNIT_FAILURE++))
        
        # Extract detailed unit test failures
        echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG"
        echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG"
        echo "---" >> "$UNIT_FAILURE_LOG"
    fi
    
    # ===== BDD TESTS =====
    echo "  🧪 BDD tests..."
    go clean -testcache > /dev/null 2>&1
    
    set +e  # Temporarily disable exit on error
    BDD_OUTPUT=$(go test ./features/... -v 2>&1)
    BDD_EXIT_CODE=$?
    set -e  # Re-enable exit on error
    
    if [ $BDD_EXIT_CODE -eq 0 ]; then
        echo "    ✅ Passed"
        ((BDD_SUCCESS++))
    else
        echo "    ❌ Failed"
        ((BDD_FAILURE++))
        
        # Extract detailed BDD test failures with actual test names
        echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG"
        echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG"
        echo "---" >> "$BDD_FAILURE_LOG"
    fi
done

echo
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))

echo "Validation Complete"
echo "=================="
echo "Total Runs: $RUN_COUNT"
echo "Unit Tests:"
echo "  Success: ${GREEN}$UNIT_SUCCESS${NC}"
echo "  Failures: ${RED}$UNIT_FAILURE${NC}"
echo "BDD Tests:"
echo "  Success: ${GREEN}$BDD_SUCCESS${NC}"
echo "  Failures: ${RED}$BDD_FAILURE${NC}"
echo "Duration: $DURATION seconds"
echo

# Check overall success
TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE))

if [ $TOTAL_FAILURES -eq 0 ]; then
    echo "${GREEN}✅ All tests passed successfully!${NC}"
    echo "Test suite is stable and ready for production"
    exit 0
else
    echo "${RED}❌ Some tests failed during validation${NC}"
    echo
    
    # Process unit test failures
    if [ -s "$UNIT_FAILURE_LOG" ]; then
        echo "Unit Test Failures:"
        echo "=================="
        
        # Count unit test failures
        UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn)
        if [ -n "$UNIT_FAILURES" ]; then
            echo "$UNIT_FAILURES"
        else
            echo "  None (check log for details)"
        fi
        
        echo
    fi
    
    # Process BDD test failures
    if [ -s "$BDD_FAILURE_LOG" ]; then
        echo "BDD Test Failures:"
        echo "================"
        
        # Count BDD test failures with granularity
        BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | \
            grep -v "dance-lessons-coach/features" | \
            grep -v "^[0-9].*FAIL" | \
            grep "/" | \
            sort | uniq -c | sort -rn)
        if [ -n "$BDD_FAILURES" ]; then
            echo "Summary:"
            while IFS= read -r line; do
                count=$(echo "$line" | awk '{print $1}')
                test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//')
                echo "  $count × $test"
            done <<< "$BDD_FAILURES"
        else
            echo "  None (check log for details)"
        fi
        
        echo
        echo "Detailed BDD Failure Log (first 20 lines):"
        echo "=========================================="
        # Show only the relevant failure lines with actual test names
        # Filter out non-specific failures and test suite lines
        grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | \
            grep -v "dance-lessons-coach/features" | \
            grep -v "^[0-9].*FAIL" | \
            grep "/" | \
            head -20
    fi
    
    echo
    echo "Recommendations:"
    echo "  1. Mark flaky BDD tests with @flaky tag"
    echo "  2. Investigate unit test failures first (faster to fix)"
    echo "  3. Check for race conditions in failing tests"
    echo "  4. Run with FIXED_TEST_PORT=true for debugging"
    echo "  5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags"
    
    exit 1
fi