Atomizer/atomizer-dashboard/backend/api/routes/optimization.py

"""
Optimization API endpoints
Handles study status, history retrieval, and control operations
"""

from fastapi import APIRouter, HTTPException
from pathlib import Path
from typing import List, Dict, Optional
import json
import sys
import sqlite3

# Add project root to path
sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent))

router = APIRouter()

# Base studies directory
STUDIES_DIR = Path(__file__).parent.parent.parent.parent.parent / "studies"

@router.get("/studies")
async def list_studies():
    """List all available optimization studies"""
    try:
        studies = []
        if not STUDIES_DIR.exists():
            return {"studies": []}

        for study_dir in STUDIES_DIR.iterdir():
            if not study_dir.is_dir():
                continue

            # Look for optimization config (check multiple locations)
            config_file = study_dir / "optimization_config.json"
            if not config_file.exists():
                config_file = study_dir / "1_setup" / "optimization_config.json"
            if not config_file.exists():
                continue

            # Load config
            with open(config_file) as f:
                config = json.load(f)

            # Check if results directory exists
            results_dir = study_dir / "2_results"

            # Check for Optuna database (Protocol 10) or JSON history (other protocols)
            study_db = results_dir / "study.db"
            history_file = results_dir / "optimization_history_incremental.json"

            status = "not_started"
            trial_count = 0
            best_value = None

            # Protocol 10: Read from Optuna SQLite database
            if study_db.exists():
                try:
                    conn = sqlite3.connect(str(study_db))
                    cursor = conn.cursor()

                    # Get trial count and status
                    cursor.execute("SELECT COUNT(*) FROM trials WHERE state = 'COMPLETE'")
                    trial_count = cursor.fetchone()[0]

                    # Get best trial (for single-objective, or first objective for multi-objective)
                    if trial_count > 0:
                        cursor.execute("""
                            SELECT value FROM trial_values
                            WHERE trial_id IN (
                                SELECT trial_id FROM trials WHERE state = 'COMPLETE'
                            )
                            ORDER BY value ASC
                            LIMIT 1
                        """)
                        result = cursor.fetchone()
                        if result:
                            best_value = result[0]

                    conn.close()

                    # Determine status
                    total_trials = config.get('optimization_settings', {}).get('n_trials', 50)
                    if trial_count >= total_trials:
                        status = "completed"
                    else:
                        status = "running"  # Simplified - would need process check

                except Exception as e:
                    print(f"Warning: Failed to read Optuna database for {study_dir.name}: {e}")
                    status = "error"

            # Legacy: Read from JSON history
            elif history_file.exists():
                with open(history_file) as f:
                    history = json.load(f)
                    trial_count = len(history)
                    if history:
                        # Find best trial
                        best_trial = min(history, key=lambda x: x['objective'])
                        best_value = best_trial['objective']

                        # Determine status
                        total_trials = config.get('trials', {}).get('n_trials', 50)
                        if trial_count >= total_trials:
                            status = "completed"
                        else:
                            status = "running"  # Simplified - would need process check

            # Get total trials from config (supports both formats)
            total_trials = (
                config.get('optimization_settings', {}).get('n_trials') or
                config.get('trials', {}).get('n_trials', 50)
            )

            studies.append({
                "id": study_dir.name,
                "name": study_dir.name.replace("_", " ").title(),
                "status": status,
                "progress": {
                    "current": trial_count,
                    "total": total_trials
                },
                "best_value": best_value,
                "target": config.get('target', {}).get('value'),
                "path": str(study_dir)
            })

        return {"studies": studies}

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to list studies: {str(e)}")

@router.get("/studies/{study_id}/status")
async def get_study_status(study_id: str):
    """Get detailed status of a specific study"""
    try:
        study_dir = STUDIES_DIR / study_id
        if not study_dir.exists():
            raise HTTPException(status_code=404, detail=f"Study {study_id} not found")

        # Load config (check multiple locations)
        config_file = study_dir / "optimization_config.json"
        if not config_file.exists():
            config_file = study_dir / "1_setup" / "optimization_config.json"

        with open(config_file) as f:
            config = json.load(f)

        # Check for results
        results_dir = study_dir / "2_results"
        study_db = results_dir / "study.db"
        history_file = results_dir / "optimization_history_incremental.json"

        # Protocol 10: Read from Optuna database
        if study_db.exists():
            conn = sqlite3.connect(str(study_db))
            cursor = conn.cursor()

            # Get trial counts by state
            cursor.execute("SELECT COUNT(*) FROM trials WHERE state = 'COMPLETE'")
            trial_count = cursor.fetchone()[0]

            cursor.execute("SELECT COUNT(*) FROM trials WHERE state = 'PRUNED'")
            pruned_count = cursor.fetchone()[0]

            # Get best trial (first objective for multi-objective)
            best_trial = None
            if trial_count > 0:
                cursor.execute("""
                    SELECT t.trial_id, t.number, tv.value
                    FROM trials t
                    JOIN trial_values tv ON t.trial_id = tv.trial_id
                    WHERE t.state = 'COMPLETE'
                    ORDER BY tv.value ASC
                    LIMIT 1
                """)
                result = cursor.fetchone()
                if result:
                    trial_id, trial_number, best_value = result

                    # Get parameters for this trial
                    cursor.execute("""
                        SELECT param_name, param_value
                        FROM trial_params
                        WHERE trial_id = ?
                    """, (trial_id,))
                    params = {row[0]: row[1] for row in cursor.fetchall()}

                    best_trial = {
                        "trial_number": trial_number,
                        "objective": best_value,
                        "design_variables": params,
                        "results": {"first_frequency": best_value}
                    }

            conn.close()

            total_trials = config.get('optimization_settings', {}).get('n_trials', 50)
            status = "completed" if trial_count >= total_trials else "running"

            return {
                "study_id": study_id,
                "status": status,
                "progress": {
                    "current": trial_count,
                    "total": total_trials,
                    "percentage": (trial_count / total_trials * 100) if total_trials > 0 else 0
                },
                "best_trial": best_trial,
                "pruned_trials": pruned_count,
                "config": config
            }

        # Legacy: Read from JSON history
        if not history_file.exists():
            return {
                "study_id": study_id,
                "status": "not_started",
                "progress": {"current": 0, "total": config.get('trials', {}).get('n_trials', 50)},
                "config": config
            }

        with open(history_file) as f:
            history = json.load(f)

        trial_count = len(history)
        total_trials = config.get('trials', {}).get('n_trials', 50)

        # Find best trial
        best_trial = None
        if history:
            best_trial = min(history, key=lambda x: x['objective'])

        # Check for pruning data
        pruning_file = results_dir / "pruning_history.json"
        pruned_count = 0
        if pruning_file.exists():
            with open(pruning_file) as f:
                pruning_history = json.load(f)
                pruned_count = len(pruning_history)

        status = "completed" if trial_count >= total_trials else "running"

        return {
            "study_id": study_id,
            "status": status,
            "progress": {
                "current": trial_count,
                "total": total_trials,
                "percentage": (trial_count / total_trials * 100) if total_trials > 0 else 0
            },
            "best_trial": best_trial,
            "pruned_trials": pruned_count,
            "config": config
        }

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get study status: {str(e)}")

@router.get("/studies/{study_id}/history")
async def get_optimization_history(study_id: str, limit: Optional[int] = None):
    """Get optimization history (all trials)"""
    try:
        study_dir = STUDIES_DIR / study_id
        results_dir = study_dir / "2_results"
        study_db = results_dir / "study.db"
        history_file = results_dir / "optimization_history_incremental.json"

        # Protocol 10: Read from Optuna database
        if study_db.exists():
            conn = sqlite3.connect(str(study_db))
            cursor = conn.cursor()

            # Get all completed trials
            cursor.execute("""
                SELECT trial_id, number, datetime_start, datetime_complete
                FROM trials
                WHERE state = 'COMPLETE'
                ORDER BY number DESC
            """ + (f" LIMIT {limit}" if limit else ""))

            trial_rows = cursor.fetchall()

            trials = []
            for trial_id, trial_num, start_time, end_time in trial_rows:
                # Get objectives for this trial
                cursor.execute("""
                    SELECT value
                    FROM trial_values
                    WHERE trial_id = ?
                    ORDER BY objective
                """, (trial_id,))
                values = [row[0] for row in cursor.fetchall()]

                # Get parameters for this trial
                cursor.execute("""
                    SELECT param_name, param_value
                    FROM trial_params
                    WHERE trial_id = ?
                """, (trial_id,))
                params = {}
                for param_name, param_value in cursor.fetchall():
                    try:
                        params[param_name] = float(param_value) if param_value is not None else None
                    except (ValueError, TypeError):
                        params[param_name] = param_value

                trials.append({
                    "trial_number": trial_num,
                    "objective": values[0] if len(values) > 0 else None,  # Primary objective
                    "objectives": values if len(values) > 1 else None,  # All objectives for multi-objective
                    "design_variables": params,
                    "results": {"first_frequency": values[0]} if len(values) > 0 else {},
                    "start_time": start_time,
                    "end_time": end_time
                })

            conn.close()
            return {"trials": trials}

        # Legacy: Read from JSON history
        if not history_file.exists():
            return {"trials": []}

        with open(history_file) as f:
            history = json.load(f)

        # Apply limit if specified
        if limit:
            history = history[-limit:]

        return {"trials": history}

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get history: {str(e)}")

@router.get("/studies/{study_id}/pruning")
async def get_pruning_history(study_id: str):
    """Get pruning diagnostics"""
    try:
        study_dir = STUDIES_DIR / study_id
        pruning_file = study_dir / "2_results" / "pruning_history.json"

        if not pruning_file.exists():
            return {"pruned_trials": []}

        with open(pruning_file) as f:
            pruning_history = json.load(f)

        return {"pruned_trials": pruning_history}

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get pruning history: {str(e)}")

def _infer_objective_unit(objective: Dict) -> str:
    """Infer unit from objective name and description"""
    name = objective.get("name", "").lower()
    desc = objective.get("description", "").lower()

    # Common unit patterns
    if "frequency" in name or "hz" in desc:
        return "Hz"
    elif "stiffness" in name or "n/mm" in desc:
        return "N/mm"
    elif "mass" in name or "kg" in desc:
        return "kg"
    elif "stress" in name or "mpa" in desc or "pa" in desc:
        return "MPa"
    elif "displacement" in name or "mm" in desc:
        return "mm"
    elif "force" in name or "newton" in desc:
        return "N"
    elif "%" in desc or "percent" in desc:
        return "%"

    # Check if unit is explicitly mentioned in description (e.g., "(N/mm)")
    import re
    unit_match = re.search(r'\(([^)]+)\)', desc)
    if unit_match:
        return unit_match.group(1)

    return ""  # No unit found

@router.get("/studies/{study_id}/metadata")
async def get_study_metadata(study_id: str):
    """Read optimization_config.json for objectives, design vars, units (Protocol 13)"""
    try:
        study_dir = STUDIES_DIR / study_id
        if not study_dir.exists():
            raise HTTPException(status_code=404, detail=f"Study {study_id} not found")

        # Load config (check multiple locations)
        config_file = study_dir / "optimization_config.json"
        if not config_file.exists():
            config_file = study_dir / "1_setup" / "optimization_config.json"

        if not config_file.exists():
            raise HTTPException(status_code=404, detail=f"Config file not found for study {study_id}")

        with open(config_file) as f:
            config = json.load(f)

        # Enhance objectives with inferred units if not present
        objectives = config.get("objectives", [])
        for obj in objectives:
            if "unit" not in obj or not obj["unit"]:
                obj["unit"] = _infer_objective_unit(obj)

        return {
            "objectives": objectives,
            "design_variables": config.get("design_variables", []),
            "constraints": config.get("constraints", []),
            "study_name": config.get("study_name", study_id),
            "description": config.get("description", "")
        }

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get study metadata: {str(e)}")

@router.get("/studies/{study_id}/optimizer-state")
async def get_optimizer_state(study_id: str):
    """Read realtime optimizer state from intelligent_optimizer/ (Protocol 13)"""
    try:
        study_dir = STUDIES_DIR / study_id
        results_dir = study_dir / "2_results"
        state_file = results_dir / "intelligent_optimizer" / "optimizer_state.json"

        if not state_file.exists():
            return {"available": False}

        with open(state_file) as f:
            state = json.load(f)

        return {"available": True, **state}

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get optimizer state: {str(e)}")

@router.get("/studies/{study_id}/pareto-front")
async def get_pareto_front(study_id: str):
    """Get Pareto-optimal solutions for multi-objective studies (Protocol 13)"""
    try:
        study_dir = STUDIES_DIR / study_id
        results_dir = study_dir / "2_results"
        study_db = results_dir / "study.db"

        if not study_db.exists():
            return {"is_multi_objective": False, "pareto_front": []}

        # Import optuna here to avoid loading it for all endpoints
        import optuna

        storage = optuna.storages.RDBStorage(f"sqlite:///{study_db}")
        study = optuna.load_study(study_name=study_id, storage=storage)

        # Check if multi-objective
        if len(study.directions) == 1:
            return {"is_multi_objective": False, "pareto_front": []}

        # Get Pareto front
        pareto_trials = study.best_trials

        return {
            "is_multi_objective": True,
            "pareto_front": [
                {
                    "trial_number": t.number,
                    "values": t.values,
                    "params": t.params,
                    "user_attrs": dict(t.user_attrs),
                    "constraint_satisfied": t.user_attrs.get("constraint_satisfied", True)
                }
                for t in pareto_trials
            ]
        }

    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Study {study_id} not found")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to get Pareto front: {str(e)}")