""" Dashboard Test Runner - Automated testing through the Atomizer dashboard. Supports test types: - API tests (REST endpoint verification) - Browser tests (UI interaction via Playwright) - CLI tests (command line execution) - Filesystem tests (file/directory verification) """ import asyncio import json import logging import subprocess from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional import aiohttp logger = logging.getLogger(__name__) @dataclass class TestStep: """A single step in a test scenario.""" action: str target: Optional[str] = None data: Optional[Dict] = None timeout_ms: int = 5000 @dataclass class TestScenario: """A complete test scenario.""" id: str name: str type: str # "api", "browser", "cli", "filesystem" steps: List[Dict] = field(default_factory=list) expected_outcome: Dict = field(default_factory=lambda: {"status": "pass"}) timeout_ms: int = 30000 @dataclass class TestResult: """Result of a single test.""" scenario_id: str scenario_name: str passed: bool duration_ms: float error: Optional[str] = None details: Optional[Dict] = None @dataclass class TestReport: """Complete test report.""" timestamp: str = field(default_factory=lambda: datetime.now().isoformat()) scenarios: List[TestResult] = field(default_factory=list) summary: Dict = field(default_factory=lambda: {"passed": 0, "failed": 0, "total": 0}) class DashboardTestRunner: """ Automated test runner for Atomizer dashboard. Executes test scenarios against: - Backend API endpoints - Frontend UI (via Playwright if available) - CLI commands - Filesystem assertions """ def __init__(self, config: Optional[Dict] = None): """ Initialize the test runner. Args: config: Configuration with dashboard URLs and timeouts """ self.config = config or {} self.base_url = self.config.get("dashboard_url", "http://localhost:8000") self.ws_url = self.config.get("websocket_url", "ws://localhost:8000") self.timeout_ms = self.config.get("test_timeout_ms", 30000) self.studies_dir = Path(self.config.get("studies_dir", "C:/Users/antoi/Atomizer/studies")) self._session: Optional[aiohttp.ClientSession] = None self._ws: Optional[aiohttp.ClientWebSocketResponse] = None self._playwright = None self._browser = None async def connect(self): """Initialize connections.""" if self._session is None: self._session = aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=self.timeout_ms / 1000) ) async def disconnect(self): """Clean up connections.""" if self._ws: await self._ws.close() self._ws = None if self._session: await self._session.close() self._session = None if self._browser: await self._browser.close() self._browser = None async def run_test_suite(self, scenarios: List[Dict]) -> Dict: """ Run a complete test suite. Args: scenarios: List of test scenario dicts Returns: Test report as dict """ await self.connect() report = TestReport() for scenario_dict in scenarios: scenario = self._parse_scenario(scenario_dict) start_time = datetime.now() try: result = await self._execute_scenario(scenario) result.duration_ms = (datetime.now() - start_time).total_seconds() * 1000 report.scenarios.append(result) if result.passed: report.summary["passed"] += 1 else: report.summary["failed"] += 1 except Exception as e: logger.error(f"Scenario {scenario.id} failed with error: {e}") report.scenarios.append( TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=(datetime.now() - start_time).total_seconds() * 1000, error=str(e), ) ) report.summary["failed"] += 1 report.summary["total"] += 1 return { "timestamp": report.timestamp, "scenarios": [self._result_to_dict(r) for r in report.scenarios], "summary": report.summary, } def _parse_scenario(self, scenario_dict: Dict) -> TestScenario: """Parse a scenario dict into TestScenario.""" return TestScenario( id=scenario_dict.get("id", "unknown"), name=scenario_dict.get("name", "Unnamed test"), type=scenario_dict.get("type", "api"), steps=scenario_dict.get("steps", []), expected_outcome=scenario_dict.get("expected_outcome", {"status": "pass"}), timeout_ms=scenario_dict.get("timeout_ms", self.timeout_ms), ) def _result_to_dict(self, result: TestResult) -> Dict: """Convert TestResult to dict.""" return { "scenario_id": result.scenario_id, "scenario_name": result.scenario_name, "passed": result.passed, "duration_ms": result.duration_ms, "error": result.error, "details": result.details, } async def _execute_scenario(self, scenario: TestScenario) -> TestResult: """Execute a single test scenario.""" logger.info(f"Executing test: {scenario.name} ({scenario.type})") if scenario.type == "api": return await self._execute_api_scenario(scenario) elif scenario.type == "browser": return await self._execute_browser_scenario(scenario) elif scenario.type == "cli": return await self._execute_cli_scenario(scenario) elif scenario.type == "filesystem": return await self._execute_filesystem_scenario(scenario) else: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Unknown test type: {scenario.type}", ) async def _execute_api_scenario(self, scenario: TestScenario) -> TestResult: """Execute an API test scenario.""" details = {} for step in scenario.steps: action = step.get("action", "get").lower() endpoint = step.get("endpoint", step.get("target", "/")) data = step.get("data") url = f"{self.base_url}{endpoint}" try: if action == "get": async with self._session.get(url) as resp: details["status_code"] = resp.status details["response"] = await resp.json() elif action == "post": async with self._session.post(url, json=data) as resp: details["status_code"] = resp.status details["response"] = await resp.json() elif action == "put": async with self._session.put(url, json=data) as resp: details["status_code"] = resp.status details["response"] = await resp.json() elif action == "delete": async with self._session.delete(url) as resp: details["status_code"] = resp.status details["response"] = await resp.json() except aiohttp.ClientError as e: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"API request failed: {e}", details={"url": url, "action": action}, ) except json.JSONDecodeError: details["response"] = "Non-JSON response" # Check expected outcome passed = self._check_outcome(details, scenario.expected_outcome) return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=passed, duration_ms=0, details=details, ) async def _execute_browser_scenario(self, scenario: TestScenario) -> TestResult: """Execute a browser test scenario using Playwright.""" try: from playwright.async_api import async_playwright except ImportError: logger.warning("Playwright not available, skipping browser test") return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=True, # Skip, don't fail duration_ms=0, error="Playwright not installed - test skipped", ) details = {} try: async with async_playwright() as p: browser = await p.chromium.launch(headless=True) page = await browser.new_page() for step in scenario.steps: action = step.get("action", "navigate") if action == "navigate": url = step.get("url", "/") # Use frontend URL (port 3003 for Vite dev server) full_url = f"http://localhost:3003{url}" if url.startswith("/") else url await page.goto(full_url, timeout=scenario.timeout_ms) details["navigated_to"] = full_url elif action == "wait_for": selector = step.get("selector") if selector: await page.wait_for_selector(selector, timeout=scenario.timeout_ms) details["found_selector"] = selector elif action == "click": selector = step.get("selector") if selector: await page.click(selector) details["clicked"] = selector elif action == "fill": selector = step.get("selector") value = step.get("value", "") if selector: await page.fill(selector, value) details["filled"] = {selector: value} elif action == "screenshot": path = step.get("path", f"test_{scenario.id}.png") await page.screenshot(path=path) details["screenshot"] = path await browser.close() passed = True except Exception as e: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Browser test failed: {e}", details=details, ) return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=passed, duration_ms=0, details=details, ) async def _execute_cli_scenario(self, scenario: TestScenario) -> TestResult: """Execute a CLI test scenario.""" details = {} for step in scenario.steps: command = step.get("command", step.get("target", "")) cwd = step.get("cwd", str(self.studies_dir)) if not command: continue try: # Use PowerShell on Windows result = subprocess.run( ["powershell", "-Command", command], capture_output=True, text=True, cwd=cwd, timeout=scenario.timeout_ms / 1000, ) details["command"] = command details["returncode"] = result.returncode details["stdout"] = result.stdout[:1000] if result.stdout else "" details["stderr"] = result.stderr[:1000] if result.stderr else "" if result.returncode != 0: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Command failed with code {result.returncode}", details=details, ) except subprocess.TimeoutExpired: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Command timed out after {scenario.timeout_ms}ms", details={"command": command}, ) except Exception as e: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"CLI execution failed: {e}", details={"command": command}, ) passed = self._check_outcome(details, scenario.expected_outcome) return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=passed, duration_ms=0, details=details, ) async def _execute_filesystem_scenario(self, scenario: TestScenario) -> TestResult: """Execute a filesystem test scenario.""" details = {} for step in scenario.steps: action = step.get("action", "check_exists") path_str = step.get("path", "") # Resolve relative paths if not Path(path_str).is_absolute(): path = self.studies_dir.parent / path_str else: path = Path(path_str) if action == "check_exists": exists = path.exists() details["path"] = str(path) details["exists"] = exists if scenario.expected_outcome.get("exists", True) != exists: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Path {'does not exist' if not exists else 'exists but should not'}: {path}", details=details, ) elif action == "check_file_contains": content_check = step.get("contains", "") if path.exists() and path.is_file(): content = path.read_text() contains = content_check in content details["contains"] = contains details["search_term"] = content_check if not contains: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"File does not contain: {content_check}", details=details, ) else: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"File not found: {path}", details=details, ) elif action == "check_json_valid": if path.exists() and path.is_file(): try: with open(path) as f: json.load(f) details["valid_json"] = True except json.JSONDecodeError as e: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"Invalid JSON: {e}", details={"path": str(path)}, ) else: return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=False, duration_ms=0, error=f"File not found: {path}", details=details, ) return TestResult( scenario_id=scenario.id, scenario_name=scenario.name, passed=True, duration_ms=0, details=details, ) def _check_outcome(self, details: Dict, expected: Dict) -> bool: """Check if test details match expected outcome.""" for key, expected_value in expected.items(): if key not in details: continue actual_value = details[key] # Handle nested dicts if isinstance(expected_value, dict) and isinstance(actual_value, dict): if not self._check_outcome(actual_value, expected_value): return False # Handle lists elif isinstance(expected_value, list) and isinstance(actual_value, list): if expected_value != actual_value: return False # Handle simple values elif actual_value != expected_value: return False return True async def verify_fix(self, fix: Dict) -> Dict: """ Verify that a specific fix was successful. Args: fix: Fix dict with issue_id and files_modified Returns: Verification result """ issue_id = fix.get("issue_id", "unknown") files_modified = fix.get("files_modified", []) # Run quick verification passed = True details = {} # Check that modified files exist for file_path in files_modified: path = Path(file_path) if not path.exists(): passed = False details["missing_file"] = str(path) break # Could add more sophisticated verification here return { "issue_id": issue_id, "passed": passed, "details": details, } async def run_health_check(self) -> Dict: """ Run a quick health check on dashboard components. Returns: Health status dict """ await self.connect() health = { "timestamp": datetime.now().isoformat(), "api": "unknown", "frontend": "unknown", "websocket": "unknown", } # Check API try: async with self._session.get(f"{self.base_url}/health") as resp: if resp.status == 200: health["api"] = "healthy" else: health["api"] = f"unhealthy (status {resp.status})" except Exception as e: health["api"] = f"error: {e}" # Check frontend (if available) try: async with self._session.get("http://localhost:3000") as resp: if resp.status == 200: health["frontend"] = "healthy" else: health["frontend"] = f"unhealthy (status {resp.status})" except Exception as e: health["frontend"] = f"error: {e}" return health