dance-lessons-coach/scripts/validate-test-suite.sh

#!/bin/bash

# Test Suite Validation Script
# Runs tests N times with separate unit and BDD test phases
# Usage: ./scripts/validate-test-suite.sh [N] [OPTIONS]
#   N          - Number of times to run tests (default: 20)
#   OPTIONS:
#     --parallel      - Run feature tests in parallel
#     --count=C      - Override -count flag for go test (default: same as N)
#     --quick        - Run only core tests (skip @flaky)
#     --features=X   - Test specific features only (comma-separated)

set -e

# Default values
RUN_COUNT=${1:-20}
GOTEST_COUNT=""
PARALLEL=false
QUICK=false
FEATURES_FILTER=""

# Parse arguments
shift
while [[ $# -gt 0 ]]; do
    case "$1" in
        --parallel)
            PARALLEL=true
            shift
            ;;
        --count=*)
            GOTEST_COUNT="${1#*=}"
            shift
            ;;
        --quick)
            QUICK=true
            shift
            ;;
        --features=*)
            FEATURES_FILTER="${1#*=}"
            shift
            ;;
        *)
            echo "Unknown option: $1"
            exit 1
            ;;
    esac
done

# Use GOTEST_COUNT if set, otherwise use RUN_COUNT
if [ -z "$GOTEST_COUNT" ]; then
    GOTEST_COUNT=$RUN_COUNT
fi

SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Temporary files
UNIT_FAILURE_LOG=$(mktemp)
BDD_FAILURE_LOG=$(mktemp)
SUMMARY_REPORT=$(mktemp)

# Cleanup temporary files on exit
cleanup() {
    rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT"
}
trap cleanup EXIT

echo "🧪 Test Suite Validation Script"
echo "=============================="
echo "Runs: $RUN_COUNT"
echo "Unit Tests: ./cmd/... ./pkg/..."
echo "BDD Tests: ./features/..."
echo "Date: $(date)"
echo

# Initialize counters
UNIT_SUCCESS=0
UNIT_FAILURE=0
BDD_SUCCESS=0
BDD_FAILURE=0
START_TIME=$(date +%s)

echo "Starting validation runs..."
echo

# Main validation loop
for (( run=1; run<=$RUN_COUNT; run++ )); do
    echo "Run $run/$RUN_COUNT..."
    
    # ===== UNIT TESTS =====
    echo "  🧪 Unit tests..."
    go clean -testcache > /dev/null 2>&1
    
    set +e  # Temporarily disable exit on error
    UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1)
    UNIT_EXIT_CODE=$?
    set -e  # Re-enable exit on error
    
    if [ $UNIT_EXIT_CODE -eq 0 ]; then
        echo "    ✅ Passed"
        ((UNIT_SUCCESS++))
    else
        echo "    ❌ Failed"
        ((UNIT_FAILURE++))
        
        # Extract detailed unit test failures
        echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG"
        echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG"
        echo "---" >> "$UNIT_FAILURE_LOG"
    fi
    
    # ===== BDD TESTS =====
    echo "  🧪 BDD tests..."
    go clean -testcache > /dev/null 2>&1
    
    # Set environment variables for consistent BDD test behavior
    export DLC_DATABASE_HOST=localhost
    export DLC_DATABASE_PORT=5432
    export DLC_DATABASE_USER=postgres
    export DLC_DATABASE_PASSWORD=postgres
    export DLC_DATABASE_NAME=dance_lessons_coach_test

    export BDD_SCHEMA_ISOLATION=true
    
    # Build feature test arguments
    FEATURE_PACKAGES=("config" "auth" "greet" "health" "jwt")
    
    # Filter features if specified
    if [ -n "$FEATURES_FILTER" ]; then
        IFS=',' read -ra FILTERED_FEATURES <<< "$FEATURES_FILTER"
        ALL_FEATURES=("config" "auth" "greet" "health" "jwt")
        FEATURE_PACKAGES=()
        for feat in "${FILTERED_FEATURES[@]}"; do
            if [[ " ${ALL_FEATURES[@]} " =~ " ${feat} " ]]; then
                FEATURE_PACKAGES+=("$feat")
            fi
        done
    fi
    
    # Build go test command for features
    FEATURE_TESTS=""
    for feat in "${FEATURE_PACKAGES[@]}"; do
        FEATURE_TESTS+="./features/$feat "
    done
    
    # Set tags for quick mode
    if [ "$QUICK" = true ]; then
        export GODOG_TAGS="~@flaky && ~@todo && ~@skip"
    fi
    
    set +e  # Temporarily disable exit on error
    # Force sequential package testing and use fixed port to prevent race conditions
    FIXED_TEST_PORT=true BDD_SCHEMA_ISOLATION=true go test ${FEATURE_TESTS} -count=$GOTEST_COUNT -v -p 1 2>&1 | tee /tmp/bdd_raw_$$.txt | grep -v '^{"level"' > /tmp/bdd_output_$$.txt && BDD_OUTPUT=$(cat /tmp/bdd_output_$$.txt) && rm -f /tmp/bdd_output_$$.txt /tmp/bdd_raw_$$.txt || true
    BDD_EXIT_CODE=${PIPESTATUS[0]}
    set -e  # Re-enable exit on error
    
    if [ $BDD_EXIT_CODE -eq 0 ]; then
        echo "    ✅ Passed"
        ((BDD_SUCCESS++))
    else
        echo "    ❌ Failed"
        ((BDD_FAILURE++))
        
        # Extract detailed BDD test failures with actual test names
        echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG"
        echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG"
        echo "---" >> "$BDD_FAILURE_LOG"
    fi
done

echo
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))

echo "Validation Complete"
echo "=================="
echo "Total Runs: $RUN_COUNT"
echo "Unit Tests:"
echo -e "  Success: ${GREEN}$UNIT_SUCCESS${NC}"
echo -e "  Failures: ${RED}$UNIT_FAILURE${NC}"
echo -e "BDD Tests:"
echo -e "  Success: ${GREEN}$BDD_SUCCESS${NC}"
echo -e "  Failures: ${RED}$BDD_FAILURE${NC}"
echo "Duration: $DURATION seconds"
echo

# Check overall success
TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE))

if [ $TOTAL_FAILURES -eq 0 ]; then
    echo -e "${GREEN}✅ All tests passed successfully!${NC}"
    echo "Test suite is stable and ready for production"
    exit 0
else
    echo -e "${RED}❌ Some tests failed during validation${NC}"
    echo
    
    # Process unit test failures
    if [ -s "$UNIT_FAILURE_LOG" ]; then
        echo "Unit Test Failures:"
        echo "=================="
        
        # Count unit test failures
        UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn)
        if [ -n "$UNIT_FAILURES" ]; then
            echo "$UNIT_FAILURES"
        else
            echo "  None (check log for details)"
        fi
        
        echo
    fi
    
    # Process BDD test failures
    if [ -s "$BDD_FAILURE_LOG" ]; then
        echo "BDD Test Failures:"
        echo "==============="
        
        # Count BDD test failures with granularity
        BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | \
            grep -v "dance-lessons-coach/features" | \
            grep -v "^[0-9].*FAIL" | \
            grep "/" | \
            sort | uniq -c | sort -rn)
        if [ -n "$BDD_FAILURES" ]; then
            echo "Summary:"
            while IFS= read -r line; do
                count=$(echo "$line" | awk '{print $1}')
                test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//')
                echo "  $count x $test"
            done <<< "$BDD_FAILURES"
        else
            echo "  None (check log for details)"
        fi
        
        echo
        echo "Detailed BDD Failure Log (first 20 lines):"
        echo "=========================================="
        # Show only the relevant failure lines with actual test names
        # Filter out non-specific failures and test suite lines
        grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | \
            grep -v "dance-lessons-coach/features" | \
            grep -v "^[0-9].*FAIL" | \
            grep "/" | \
            head -20
    fi
    
    echo
    echo "Recommendations:"
    echo "  1. Investigate unit test failures first (faster to fix)"
    echo "  2. Check for race conditions in failing tests"
    echo "  3. Review test dependencies and isolation (schema/database isolation)"
    echo "  4. Run individual failing tests with: FIXED_TEST_PORT=true go test ./features -v -run TestBDD/Name"
    echo "  5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags"
    
    exit 1
fi