#!/bin/bash # Test Suite Validation Script # Runs tests N times with separate unit and BDD test phases # Usage: ./scripts/validate-test-suite.sh [N] [OPTIONS] # N - Number of times to run tests (default: 20) # OPTIONS: # --parallel - Run feature tests in parallel # --count=C - Override -count flag for go test (default: same as N) # --quick - Run only core tests (skip @flaky) # --features=X - Test specific features only (comma-separated) set -e # Default values RUN_COUNT=${1:-20} GOTEST_COUNT="" PARALLEL=false QUICK=false FEATURES_FILTER="" # Parse arguments shift while [[ $# -gt 0 ]]; do case "$1" in --parallel) PARALLEL=true shift ;; --count=*) GOTEST_COUNT="${1#*=}" shift ;; --quick) QUICK=true shift ;; --features=*) FEATURES_FILTER="${1#*=}" shift ;; *) echo "Unknown option: $1" exit 1 ;; esac done # Use GOTEST_COUNT if set, otherwise use RUN_COUNT if [ -z "$GOTEST_COUNT" ]; then GOTEST_COUNT=$RUN_COUNT fi SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Temporary files UNIT_FAILURE_LOG=$(mktemp) BDD_FAILURE_LOG=$(mktemp) SUMMARY_REPORT=$(mktemp) # Cleanup temporary files on exit cleanup() { rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT" } trap cleanup EXIT echo "๐Ÿงช Test Suite Validation Script" echo "==============================" echo "Runs: $RUN_COUNT" echo "Unit Tests: ./cmd/... ./pkg/..." echo "BDD Tests: ./features/..." echo "Date: $(date)" echo # Initialize counters UNIT_SUCCESS=0 UNIT_FAILURE=0 BDD_SUCCESS=0 BDD_FAILURE=0 START_TIME=$(date +%s) echo "Starting validation runs..." echo # Main validation loop for (( run=1; run<=$RUN_COUNT; run++ )); do echo "Run $run/$RUN_COUNT..." # ===== UNIT TESTS ===== echo " ๐Ÿงช Unit tests..." go clean -testcache > /dev/null 2>&1 set +e # Temporarily disable exit on error UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1) UNIT_EXIT_CODE=$? set -e # Re-enable exit on error if [ $UNIT_EXIT_CODE -eq 0 ]; then echo " โœ… Passed" ((UNIT_SUCCESS++)) else echo " โŒ Failed" ((UNIT_FAILURE++)) # Extract detailed unit test failures echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG" echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG" echo "---" >> "$UNIT_FAILURE_LOG" fi # ===== BDD TESTS ===== echo " ๐Ÿงช BDD tests..." go clean -testcache > /dev/null 2>&1 # Set environment variables for consistent BDD test behavior export DLC_DATABASE_HOST=localhost export DLC_DATABASE_PORT=5432 export DLC_DATABASE_USER=postgres export DLC_DATABASE_PASSWORD=postgres export DLC_DATABASE_NAME=dance_lessons_coach_test export BDD_SCHEMA_ISOLATION=true # Build feature test arguments FEATURE_PACKAGES=("config" "auth" "greet" "health" "jwt") # Filter features if specified if [ -n "$FEATURES_FILTER" ]; then IFS=',' read -ra FILTERED_FEATURES <<< "$FEATURES_FILTER" ALL_FEATURES=("config" "auth" "greet" "health" "jwt") FEATURE_PACKAGES=() for feat in "${FILTERED_FEATURES[@]}"; do if [[ " ${ALL_FEATURES[@]} " =~ " ${feat} " ]]; then FEATURE_PACKAGES+=("$feat") fi done fi # Build go test command for features FEATURE_TESTS="" for feat in "${FEATURE_PACKAGES[@]}"; do FEATURE_TESTS+="./features/$feat " done # Set tags for quick mode if [ "$QUICK" = true ]; then export GODOG_TAGS="~@flaky && ~@todo && ~@skip" fi set +e # Temporarily disable exit on error # Force sequential package testing and use fixed port to prevent race conditions FIXED_TEST_PORT=true BDD_SCHEMA_ISOLATION=true go test ${FEATURE_TESTS} -count=$GOTEST_COUNT -v -p 1 2>&1 | tee /tmp/bdd_raw_$$.txt | grep -v '^{"level"' > /tmp/bdd_output_$$.txt && BDD_OUTPUT=$(cat /tmp/bdd_output_$$.txt) && rm -f /tmp/bdd_output_$$.txt /tmp/bdd_raw_$$.txt || true BDD_EXIT_CODE=${PIPESTATUS[0]} set -e # Re-enable exit on error if [ $BDD_EXIT_CODE -eq 0 ]; then echo " โœ… Passed" ((BDD_SUCCESS++)) else echo " โŒ Failed" ((BDD_FAILURE++)) # Extract detailed BDD test failures with actual test names echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG" echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG" echo "---" >> "$BDD_FAILURE_LOG" fi done echo END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) echo "Validation Complete" echo "==================" echo "Total Runs: $RUN_COUNT" echo "Unit Tests:" echo -e " Success: ${GREEN}$UNIT_SUCCESS${NC}" echo -e " Failures: ${RED}$UNIT_FAILURE${NC}" echo -e "BDD Tests:" echo -e " Success: ${GREEN}$BDD_SUCCESS${NC}" echo -e " Failures: ${RED}$BDD_FAILURE${NC}" echo "Duration: $DURATION seconds" echo # Check overall success TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE)) if [ $TOTAL_FAILURES -eq 0 ]; then echo -e "${GREEN}โœ… All tests passed successfully!${NC}" echo "Test suite is stable and ready for production" exit 0 else echo -e "${RED}โŒ Some tests failed during validation${NC}" echo # Process unit test failures if [ -s "$UNIT_FAILURE_LOG" ]; then echo "Unit Test Failures:" echo "==================" # Count unit test failures UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn) if [ -n "$UNIT_FAILURES" ]; then echo "$UNIT_FAILURES" else echo " None (check log for details)" fi echo fi # Process BDD test failures if [ -s "$BDD_FAILURE_LOG" ]; then echo "BDD Test Failures:" echo "===============" # Count BDD test failures with granularity BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | \ grep -v "dance-lessons-coach/features" | \ grep -v "^[0-9].*FAIL" | \ grep "/" | \ sort | uniq -c | sort -rn) if [ -n "$BDD_FAILURES" ]; then echo "Summary:" while IFS= read -r line; do count=$(echo "$line" | awk '{print $1}') test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//') echo " $count x $test" done <<< "$BDD_FAILURES" else echo " None (check log for details)" fi echo echo "Detailed BDD Failure Log (first 20 lines):" echo "==========================================" # Show only the relevant failure lines with actual test names # Filter out non-specific failures and test suite lines grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | \ grep -v "dance-lessons-coach/features" | \ grep -v "^[0-9].*FAIL" | \ grep "/" | \ head -20 fi echo echo "Recommendations:" echo " 1. Investigate unit test failures first (faster to fix)" echo " 2. Check for race conditions in failing tests" echo " 3. Review test dependencies and isolation (schema/database isolation)" echo " 4. Run individual failing tests with: FIXED_TEST_PORT=true go test ./features -v -run TestBDD/Name" echo " 5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags" exit 1 fi