diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 2d186066..de072aba 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -2,110 +2,42 @@ "permissions": { "allow": [ "Bash(dir:*)", - "Bash(sqlite3:*)", - "Bash(timeout /t 30 /nobreak)", - "Bash(npm install:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(git push:*)", + "Bash(powershell -Command:*)", "Bash(python:*)", - "Bash(conda activate:*)", - "Bash(C:/Users/Antoine/miniconda3/envs/atomizer/python.exe:*)", - "Bash(cat:*)", - "Bash(C:UsersAntoineminiconda3envsatomizerpython.exe run_adaptive_mirror_optimization.py --fea-budget 100 --batch-size 5 --strategy hybrid)", - "Bash(/c/Users/Antoine/miniconda3/envs/atomizer/python.exe:*)", - "Bash(npm run build:*)", - "Bash(npm uninstall:*)", + "Bash(git:*)", + "Bash(npm:*)", + "Bash(conda:*)", + "Bash(pip:*)", + "Bash(cmd /c:*)", + "Bash(tasklist:*)", + "Bash(taskkill:*)", + "Bash(robocopy:*)", + "Bash(xcopy:*)", + "Bash(del:*)", + "Bash(type:*)", + "Bash(where:*)", "Bash(netstat:*)", "Bash(findstr:*)", "Bash(curl:*)", - "Bash(npx tsc:*)", - "Bash(atomizer-dashboard/README.md )", - "Bash(atomizer-dashboard/backend/api/main.py )", - "Bash(atomizer-dashboard/backend/api/routes/optimization.py )", - "Bash(atomizer-dashboard/backend/api/routes/claude.py )", - "Bash(atomizer-dashboard/backend/api/routes/terminal.py )", - "Bash(atomizer-dashboard/backend/api/services/ )", - "Bash(atomizer-dashboard/backend/requirements.txt )", - "Bash(atomizer-dashboard/frontend/package.json )", - "Bash(atomizer-dashboard/frontend/package-lock.json )", - "Bash(atomizer-dashboard/frontend/src/components/ClaudeChat.tsx )", - "Bash(atomizer-dashboard/frontend/src/components/ClaudeTerminal.tsx )", - "Bash(atomizer-dashboard/frontend/src/components/dashboard/ControlPanel.tsx )", - "Bash(atomizer-dashboard/frontend/src/pages/Dashboard.tsx )", - "Bash(atomizer-dashboard/frontend/src/context/ )", - "Bash(atomizer-dashboard/frontend/src/pages/Home.tsx )", - "Bash(atomizer-dashboard/frontend/src/App.tsx )", - "Bash(atomizer-dashboard/frontend/src/api/client.ts )", - "Bash(atomizer-dashboard/frontend/src/components/layout/Sidebar.tsx )", - "Bash(atomizer-dashboard/frontend/src/index.css )", - "Bash(atomizer-dashboard/frontend/src/pages/Results.tsx )", - "Bash(atomizer-dashboard/frontend/tailwind.config.js )", - "Bash(docs/07_DEVELOPMENT/DASHBOARD_IMPROVEMENT_PLAN.md)", - "Bash(taskkill:*)", - "Bash(xargs:*)", - "Bash(cmd.exe /c:*)", - "Bash(powershell.exe -Command:*)", - "Bash(where:*)", - "Bash(type %USERPROFILE%.claude*)", - "Bash(conda create:*)", - "Bash(cmd /c \"conda create -n atomizer python=3.10 -y\")", - "Bash(cmd /c \"where conda\")", - "Bash(cmd /c \"dir /b C:\\Users\\antoi\\anaconda3\\Scripts\\conda.exe 2>nul || dir /b C:\\Users\\antoi\\miniconda3\\Scripts\\conda.exe 2>nul || dir /b C:\\ProgramData\\anaconda3\\Scripts\\conda.exe 2>nul || dir /b C:\\ProgramData\\miniconda3\\Scripts\\conda.exe 2>nul || echo NOT_FOUND\")", - "Bash(cmd /c \"if exist C:\\Users\\antoi\\anaconda3\\Scripts\\conda.exe (echo FOUND: anaconda3) else if exist C:\\Users\\antoi\\miniconda3\\Scripts\\conda.exe (echo FOUND: miniconda3) else if exist C:\\ProgramData\\anaconda3\\Scripts\\conda.exe (echo FOUND: ProgramData\\anaconda3) else (echo NOT_FOUND)\")", - "Bash(powershell:*)", - "Bash(C:Usersantoianaconda3Scriptsconda.exe create -n atomizer python=3.10 -y)", - "Bash(cmd /c \"C:\\Users\\antoi\\anaconda3\\Scripts\\conda.exe create -n atomizer python=3.10 -y\")", - "Bash(cmd /c \"set SPLM_LICENSE_SERVER=28000@dalidou;28000@100.80.199.40 && \"\"C:\\Program Files\\Siemens\\DesigncenterNX2512\\NXBIN\\run_journal.exe\"\" \"\"C:\\Users\\antoi\\Atomizer\\optimization_engine\\solve_simulation.py\"\" -args \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_adaptive_V15\\2_iterations\\iter2\\ASSY_M1_assyfem1_sim1.sim\"\" \"\"Solution 1\"\" 2>&1\")", - "Bash(cmd /c \"set SPLM_LICENSE_SERVER=28000@dalidou;28000@100.80.199.40 && \"C:Program FilesSiemensDesigncenterNX2512NXBINrun_journal.exe\" \"C:UsersantoiAtomizernx_journalsextract_part_mass_material.py\" -args \"C:UsersantoiAtomizerstudiesm1_mirror_cost_reduction1_setupmodelM1_Blank.prt\" \"C:UsersantoiAtomizerstudiesm1_mirror_cost_reduction1_setupmodel\" 2>&1\")", - "Bash(npm run dev:*)", - "Bash(cmd /c \"cd /d C:\\Users\\antoi\\Atomizer\\atomizer-dashboard\\frontend && npm run dev\")", - "Bash(cmd /c \"cd /d C:\\Users\\antoi\\Atomizer\\atomizer-dashboard\\frontend && dir package.json && npm --version\")", - "Bash(cmd /c \"set SPLM_LICENSE_SERVER=28000@dalidou;28000@100.80.199.40 && \"\"C:\\Program Files\\Siemens\\DesigncenterNX2512\\NXBIN\\run_journal.exe\"\" \"\"C:\\Users\\antoi\\Atomizer\\nx_journals\\extract_part_mass_material.py\"\" -args \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\\M1_Blank.prt\"\" \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\"\" 2>&1\")", - "Bash(cmd /c \"set SPLM_LICENSE_SERVER=28000@dalidou;28000@100.80.199.40 && \"\"C:\\Program Files\\Siemens\\DesigncenterNX2512\\NXBIN\\run_journal.exe\"\" \"\"C:\\Users\\antoi\\Atomizer\\nx_journals\\extract_expressions.py\"\" -args \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\\M1_Blank.prt\"\" \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\"\" 2>&1\")", - "Bash(cmd /c \"set SPLM_LICENSE_SERVER=28000@dalidou;28000@100.80.199.40 && \"\"C:\\Program Files\\Siemens\\DesigncenterNX2512\\NXBIN\\run_journal.exe\"\" \"\"C:\\Users\\antoi\\Atomizer\\nx_journals\\extract_expressions.py\"\" -args \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\\M1_Blank.prt\"\" \"\"C:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_cost_reduction\\1_setup\\model\"\"\")", - "Bash(cmd /c:*)", - "Bash(taskkill /F /FI \"WINDOWTITLE eq *uvicorn*\")", - "Bash(python -m uvicorn:*)", - "Bash(conda run:*)", - "Bash(/c/Users/antoi/miniconda3/envs/atomizer/python.exe -m uvicorn:*)", - "Bash(/c/Users/antoi/anaconda3/envs/atomizer/python.exe -m uvicorn:*)", - "Bash(/c/Users/antoi/anaconda3/envs/atomizer/python.exe:*)", - "Bash(tasklist:*)", - "Bash(wmic process where \"ProcessId=147068\" delete)", - "Bash(cmd.exe //c \"taskkill /F /PID 147068\")", - "Bash(pip show:*)", - "Bash(python3:*)", - "Bash(python extract_all_mirror_data.py:*)", - "Bash(C:Usersantoiminiconda3envsatomizerpython.exe extract_all_mirror_data.py)", - "Bash(/c/Users/antoi/miniconda3/envs/atomizer/python.exe:*)", - "Bash(grep:*)", - "Bash(python -c:*)", - "Bash(C:Usersantoianaconda3envsatomizerpython.exe -c \"\nimport pandas as pd\ndf = pd.read_csv(r''c:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_all_trials_export.csv'')\n\n# Check which columns have data\nprint(''=== Column data availability ==='')\nfor col in df.columns:\n non_null = df[col].notna().sum()\n print(f''{col}: {non_null}/{len(df)} ({100*non_null/len(df):.1f}%)'')\n\nprint(''\\n=== Studies in dataset ==='')\nprint(df[''study''].value_counts())\n\")", - "Bash(cmd /c \"C:\\Users\\antoi\\anaconda3\\envs\\atomizer\\python.exe -c \"\"import pandas as pd; df = pd.read_csv(r''c:\\Users\\antoi\\Atomizer\\studies\\m1_mirror_all_trials_export.csv''); print(''Rows:'', len(df)); print(df.columns.tolist())\"\"\")", - "Bash(robocopy:*)", - "Bash(xcopy:*)", - "Bash(ls:*)", - "Bash(dir \"c:\\Users\\antoi\\Atomizer\\studies\\*.png\")", - "Bash(powershell -Command \"Get-Process | Where-Object { $_Modules.FileName -like ''*study.db*'' } | Select-Object Id, ProcessName\")", - "Bash(powershell -Command:*)", - "Bash(C:/Users/antoi/miniconda3/envs/atomizer/python.exe -m uvicorn:*)", - "Bash(dir /s /b \"C:\\Users\\antoi\\*conda*\")", - "Bash(conda run -n atomizer python:*)", - "Bash(C:/ProgramData/anaconda3/condabin/conda.bat run -n atomizer python -c \"\nimport sqlite3\n\ndb_path = ''studies/M1_Mirror/m1_mirror_cost_reduction_V6/3_results/study.db''\nconn = sqlite3.connect(db_path)\ncursor = conn.cursor()\n\n# Get counts\ncursor.execute(''SELECT COUNT(*) FROM trials'')\ntotal = cursor.fetchone()[0]\n\ncursor.execute(\"\"SELECT COUNT(*) FROM trials WHERE state = ''COMPLETE''\"\")\ncomplete = cursor.fetchone()[0]\n\nprint(f''=== V6 Study Status ==='')\nprint(f''Total trials: {total}'')\nprint(f''Completed: {complete}'')\nprint(f''Failed/Pruned: {total - complete}'')\nprint(f''Progress: {complete}/200 ({100*complete/200:.1f}%)'')\n\n# Get objectives stats\nobjs = [''rel_filtered_rms_40_vs_20'', ''rel_filtered_rms_60_vs_20'', ''mfg_90_optician_workload'', ''mass_kg'']\nprint(f''\\n=== Objectives Stats ==='')\nfor obj in objs:\n cursor.execute(f\"\"SELECT MIN({obj}), MAX({obj}), AVG({obj}) FROM trials WHERE state = ''COMPLETE'' AND {obj} IS NOT NULL\"\")\n result = cursor.fetchone()\n if result and result[0] is not None:\n print(f''{obj}: min={result[0]:.4f}, max={result[1]:.4f}, mean={result[2]:.4f}'')\n\n# Design variables stats \ndvs = [''whiffle_min'', ''whiffle_outer_to_vertical'', ''whiffle_triangle_closeness'', ''blank_backface_angle'', ''Pocket_Radius'']\nprint(f''\\n=== Design Variables Explored ==='')\nfor dv in dvs:\n try:\n cursor.execute(f\"\"SELECT MIN({dv}), MAX({dv}), AVG({dv}) FROM trials WHERE state = ''COMPLETE''\"\")\n result = cursor.fetchone()\n if result and result[0] is not None:\n print(f''{dv}: min={result[0]:.3f}, max={result[1]:.3f}, mean={result[2]:.3f}'')\n except Exception as e:\n print(f''{dv}: error - {e}'')\n\nconn.close()\n\")", - "Bash(/c/Users/antoi/anaconda3/python.exe:*)", - "Bash(C:UsersantoiAtomizertemp_extract.bat)", - "Bash(dir /b \"C:\\Users\\antoi\\Atomizer\\knowledge_base\\lac\")", - "Bash(pip install:*)", - "Bash(dir \"C:\\Users\\antoi\\Atomizer\\studies\\M1_Mirror\\m1_mirror_cost_reduction_V7\\3_results\")", - "Bash(call \"%USERPROFILE%\\anaconda3\\Scripts\\activate.bat\" atomizer)", - "Bash(cmd /c \"cd /d c:\\Users\\antoi\\Atomizer && call %USERPROFILE%\\anaconda3\\Scripts\\activate.bat atomizer && python -c \"\"import sys; sys.path.insert(0, ''.''); from optimization_engine.extractors import ZernikeExtractor; print(''OK''); import inspect; print(inspect.signature(ZernikeExtractor.extract_relative))\"\"\")", - "Bash(cmd /c \"cd /d c:\\Users\\antoi\\Atomizer && c:\\Users\\antoi\\anaconda3\\envs\\atomizer\\python.exe -c \"\"import sys; sys.path.insert(0, ''.''); from optimization_engine.extractors import ZernikeExtractor; print(''Import OK''); import inspect; sig = inspect.signature(ZernikeExtractor.extract_relative); print(''Signature:'', sig)\"\"\")", - "Bash(c:Usersantoianaconda3envsatomizerpython.exe c:UsersantoiAtomizertoolstest_zernike_import.py)", - "Bash(dir \"C:\\Users\\antoi\\Atomizer\\studies\\M1_Mirror\\m1_mirror_cost_reduction_V7\\3_results\\best_design_archive\")", - "Bash(dir \"C:\\Users\\antoi\\Atomizer\\studies\\M1_Mirror\\m1_mirror_cost_reduction_V7\\3_results\\best_design_archive\\20251220_010128\")", - "Bash(dir /s /b \"C:\\Users\\antoi\\Atomizer\\studies\\M1_Mirror\\m1_mirror_cost_reduction_V8\")", - "Bash(c:/Users/antoi/anaconda3/envs/atomizer/python.exe:*)" + "Read", + "Skill(dashboard:*)", + "Bash(C:Usersantoianaconda3envsatomizerpython.exe:*)", + "Bash(del \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V5\\\\3_results\\\\study.db\")", + "Bash(C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe -c:*)", + "Bash(C:Usersantoianaconda3envsatomizerpython.exe run_optimization.py --trials 1)", + "Bash(C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe -m py_compile:*)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver analyze \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\")", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_V12\")", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_V2\")", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_V11\")", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_V11\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V3\")", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V3\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V6\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V1\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_flat_back_V5\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction_V12\" --execute)", + "Bash(\"C:\\\\Users\\\\antoi\\\\anaconda3\\\\envs\\\\atomizer\\\\python.exe\" -m optimization_engine.utils.study_archiver cleanup \"C:\\\\Users\\\\antoi\\\\Atomizer\\\\studies\\\\M1_Mirror\\\\m1_mirror_cost_reduction\" --execute)" ], "deny": [], "ask": [] diff --git a/.claude/skills/00_BOOTSTRAP.md b/.claude/skills/00_BOOTSTRAP.md index bc1cbc76..4eb49b08 100644 --- a/.claude/skills/00_BOOTSTRAP.md +++ b/.claude/skills/00_BOOTSTRAP.md @@ -84,6 +84,10 @@ User Request │ ├─ "error", "failed", "not working", "crashed" │ └─► Load: OP_06_TROUBLESHOOT.md │ + ├─► MANAGE disk space? + │ ├─ "disk", "space", "cleanup", "archive", "storage" + │ └─► Load: OP_07_DISK_OPTIMIZATION.md + │ ├─► CONFIGURE settings? │ ├─ "change", "modify", "settings", "parameters" │ └─► Load relevant SYS_* protocol @@ -109,6 +113,7 @@ User Request | Analyze results | "results", "best", "compare", "pareto" | OP_04 | - | user | | Export training data | "export", "training data", "neural" | OP_05 | modules/neural-acceleration.md | user | | Debug issues | "error", "failed", "not working", "help" | OP_06 | - | user | +| **Disk management** | "disk", "space", "cleanup", "archive" | **OP_07** | modules/study-disk-optimization.md | user | | Understand IMSO | "protocol 10", "IMSO", "adaptive" | SYS_10 | - | user | | Multi-objective | "pareto", "NSGA", "multi-objective" | SYS_11 | - | user | | Extractors | "extractor", "displacement", "stress" | SYS_12 | modules/extractors-catalog.md | user | diff --git a/.claude/skills/01_CHEATSHEET.md b/.claude/skills/01_CHEATSHEET.md index f9972285..a34c6ac8 100644 --- a/.claude/skills/01_CHEATSHEET.md +++ b/.claude/skills/01_CHEATSHEET.md @@ -30,6 +30,7 @@ requires_skills: | See best results | OP_04 | `optuna-dashboard sqlite:///study.db` or dashboard | | Export neural training data | OP_05 | `python run_optimization.py --export-training` | | Fix an error | OP_06 | Read error log → follow diagnostic tree | +| **Free disk space** | **OP_07** | `archive_study.bat cleanup --execute` | | Add custom physics extractor | EXT_01 | Create in `optimization_engine/extractors/` | | Add lifecycle hook | EXT_02 | Create in `optimization_engine/plugins/` | | Generate physics insight | SYS_16 | `python -m optimization_engine.insights generate ` | @@ -219,6 +220,48 @@ python -c "import optuna; s=optuna.load_study('my_study', 'sqlite:///3_results/s --- +## Disk Space Management (OP_07) + +FEA studies consume massive disk space. After completion, clean up regenerable files: + +### Quick Commands + +```bash +# Analyze disk usage +archive_study.bat analyze studies\M1_Mirror + +# Cleanup completed study (dry run first!) +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute + +# Archive to dalidou server +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute + +# List remote archives +archive_study.bat list +``` + +### What Gets Deleted vs Kept + +| KEEP | DELETE | +|------|--------| +| `.op2` (Nastran results) | `.prt, .fem, .sim` (copies of master) | +| `.json` (params/metadata) | `.dat` (solver input) | +| `1_setup/` (master files) | `.f04, .f06, .log` (solver logs) | +| `3_results/` (database) | `.afm, .diag, .bak` (temp files) | + +### Typical Savings + +| Stage | M1_Mirror Example | +|-------|-------------------| +| Full | 194 GB | +| After cleanup | 114 GB (41% saved) | +| Archived to server | 5 GB local (97% saved) | + +**Full details**: `docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md` + +--- + ## LAC (Learning Atomizer Core) Commands ```bash diff --git a/.claude/skills/modules/OPTIMIZATION_ENGINE_MIGRATION_PLAN.md b/.claude/skills/modules/OPTIMIZATION_ENGINE_MIGRATION_PLAN.md index 7a095c08..22ee06bc 100644 --- a/.claude/skills/modules/OPTIMIZATION_ENGINE_MIGRATION_PLAN.md +++ b/.claude/skills/modules/OPTIMIZATION_ENGINE_MIGRATION_PLAN.md @@ -1,485 +1,682 @@ -# Optimization Engine Reorganization - Migration Plan +# Optimization Engine Reorganization - DEFINITIVE Migration Plan -## Comprehensive Guide for Safe Codebase Restructuring +## Complete Guide for Safe Codebase Restructuring -**Document Version**: 1.0 +**Document Version**: 2.1 (EXHAUSTIVE + CONTEXT ENGINEERING ALIGNED) **Created**: 2025-12-23 -**Status**: PLANNING - Do Not Execute Without Review -**Risk Level**: HIGH - Affects 557+ locations across 276 files +**Updated**: 2025-12-28 +**Status**: READY FOR EXECUTION +**Risk Level**: HIGH - Affects 500+ files across entire codebase +**Next Phase**: ATOMIZER_CONTEXT_ENGINEERING_PLAN.md (execute AFTER this migration) + +--- + +## Execution Order (IMPORTANT) + +This migration plan should be executed **BEFORE** the Context Engineering plan. + +**Why Migration First?** +1. Creates clean, organized module structure +2. Context Engineering adds NEW modules (`context/`) that won't be moved +3. Avoids double-updating imports (migrate first, then integrate new code) +4. Documentation written once with final paths + +**After Migration Completes:** +- Execute `ATOMIZER_CONTEXT_ENGINEERING_PLAN.md` +- Add `optimization_engine/context/` module (playbook, reflector, session_state) +- Integrate with `optimization_engine/core/runner.py` (already at final path) --- ## Executive Summary -This document provides a complete migration plan for reorganizing `optimization_engine/` from 50+ loose files to a clean modular structure. This is a **high-impact refactoring** that requires careful execution. +This is the **DEFINITIVE** migration plan covering every single file that needs updating. The scope is significantly larger than initially estimated. -### Impact Summary +### True Impact Summary -| Category | Files Affected | Lines to Change | -|----------|----------------|-----------------| -| Python imports (internal) | 90+ files | ~145 changes | -| Python imports (studies) | 30+ folders | ~153 changes | -| Python imports (tests) | 30+ files | ~79 changes | -| Python imports (dashboard) | 3 files | ~11 changes | -| Documentation (protocols, skills) | 119 files | ~200 changes | -| JSON configs | 4 files | ~50 changes | -| **TOTAL** | **276 files** | **~640 changes** | +| Category | Files | Estimated Changes | +|----------|-------|-------------------| +| Python files (optimization_engine internal) | 90+ | ~300 import changes | +| Python files (studies) | 80+ | ~200 import changes | +| Python files (tests) | 45+ | ~150 import changes | +| Python files (tools, examples, root) | 10+ | ~50 import changes | +| Python files (dashboard backend) | 3 | ~15 import changes | +| Python files (archive - optional) | 50+ | ~100 import changes | +| Documentation (markdown) | 100+ | ~500 path references | +| JSON configuration files | 5 | ~50 path entries | +| Build config (pyproject.toml) | 1 | 2 lines | +| **TOTAL** | **500+ files** | **~1,400 changes** | + +### Estimated Time: 8-12 hours (with automation scripts) --- -## Part 1: Current State Analysis +## Part 1: Complete File Inventory -### 1.1 Top-Level Files Requiring Migration +### 1.1 Files to MOVE (Source Files) -These 50+ files at `optimization_engine/` root need to move: +These files move from `optimization_engine/` root to new subdirectories: ``` -SURROGATES (6 files) → processors/surrogates/ +MOVE TO: optimization_engine/core/ +├── runner.py +├── base_runner.py +├── runner_with_neural.py +├── intelligent_optimizer.py +├── method_selector.py +├── strategy_selector.py +├── strategy_portfolio.py +└── gradient_optimizer.py + +MOVE TO: optimization_engine/processors/surrogates/ ├── neural_surrogate.py ├── generic_surrogate.py ├── adaptive_surrogate.py ├── simple_mlp_surrogate.py ├── active_learning_surrogate.py -└── surrogate_tuner.py +├── surrogate_tuner.py +├── auto_trainer.py +└── training_data_exporter.py -OPTIMIZATION CORE (7 files) → core/ -├── runner.py -├── runner_with_neural.py -├── base_runner.py -├── intelligent_optimizer.py -├── method_selector.py -├── strategy_selector.py -└── strategy_portfolio.py - -NX INTEGRATION (6 files) → nx/ -├── nx_solver.py -├── nx_updater.py -├── nx_session_manager.py +MOVE TO: optimization_engine/nx/ +├── nx_solver.py → solver.py +├── nx_updater.py → updater.py +├── nx_session_manager.py → session_manager.py ├── solve_simulation.py ├── solve_simulation_simple.py -└── model_cleanup.py +├── model_cleanup.py +├── export_expressions.py +├── import_expressions.py +└── mesh_converter.py -STUDY MANAGEMENT (5 files) → study/ -├── study_creator.py -├── study_wizard.py -├── study_state.py -├── study_reset.py -└── study_continuation.py +MOVE TO: optimization_engine/study/ +├── study_creator.py → creator.py +├── study_wizard.py → wizard.py +├── study_state.py → state.py +├── study_reset.py → reset.py +├── study_continuation.py → continuation.py +├── benchmarking_substudy.py → benchmarking.py +└── generate_history_from_trials.py → history_generator.py -REPORTING (5 files) → reporting/ -├── generate_report.py -├── generate_report_markdown.py -├── comprehensive_results_analyzer.py +MOVE TO: optimization_engine/reporting/ +├── generate_report.py → report_generator.py +├── generate_report_markdown.py → markdown_report.py +├── comprehensive_results_analyzer.py → results_analyzer.py ├── visualizer.py └── landscape_analyzer.py -CONFIG (4 files) → config/ -├── config_manager.py -├── optimization_config_builder.py -├── optimization_setup_wizard.py -└── capability_matcher.py +MOVE TO: optimization_engine/config/ +├── config_manager.py → manager.py +├── optimization_config_builder.py → builder.py +├── optimization_setup_wizard.py → setup_wizard.py +├── capability_matcher.py +└── template_loader.py -AGENTS/RESEARCH (5 files) → agents/ or future/ +MOVE TO: optimization_engine/utils/ (existing, add to it) +├── logger.py +├── auto_doc.py +├── realtime_tracking.py +├── codebase_analyzer.py +└── pruning_logger.py + +MOVE TO: optimization_engine/future/ (existing, add to it) ├── research_agent.py ├── pynastran_research_agent.py ├── targeted_research_planner.py ├── workflow_decomposer.py -└── step_classifier.py +├── step_classifier.py +├── llm_optimization_runner.py +├── llm_workflow_analyzer.py +├── extractor_orchestrator.py +├── inline_code_generator.py +└── hook_generator.py -MISC (remaining ~15 files) - evaluate individually -├── logger.py → utils/ -├── op2_extractor.py → extractors/ -├── extractor_library.py → extractors/ -├── export_expressions.py → nx/ -├── import_expressions.py → nx/ -├── mesh_converter.py → nx/ -├── simulation_validator.py → validators/ -├── auto_doc.py → utils/ -├── auto_trainer.py → processors/surrogates/ -├── realtime_tracking.py → utils/ -├── benchmarking_substudy.py → study/ -├── codebase_analyzer.py → utils/ -├── training_data_exporter.py → processors/surrogates/ -├── pruning_logger.py → utils/ -├── adaptive_characterization.py → processors/ -└── generate_history_from_trials.py → study/ +MOVE TO: optimization_engine/extractors/ (existing, add to it) +├── op2_extractor.py +└── extractor_library.py + +MOVE TO: optimization_engine/validators/ (existing, add to it) +└── simulation_validator.py + +MOVE TO: optimization_engine/processors/ (new) +└── adaptive_characterization.py ``` -### 1.2 Well-Organized Directories (Keep As-Is) +### 1.2 Files to UPDATE (Import Changes Required) -These are already properly organized: +#### A. OPTIMIZATION_ENGINE INTERNAL (90+ files) + +**Core/Root Files:** +``` +optimization_engine/run_optimization.py +optimization_engine/__init__.py +``` + +**Extractors (20 files):** +``` +optimization_engine/extractors/__init__.py (31 imports) +optimization_engine/extractors/extract_displacement.py +optimization_engine/extractors/extract_frequency.py +optimization_engine/extractors/extract_von_mises_stress.py +optimization_engine/extractors/bdf_mass_extractor.py +optimization_engine/extractors/extract_mass_from_bdf.py +optimization_engine/extractors/extract_mass_from_expression.py +optimization_engine/extractors/extract_part_mass_material.py +optimization_engine/extractors/extract_zernike.py +optimization_engine/extractors/extract_zernike_opd.py +optimization_engine/extractors/extract_zernike_figure.py +optimization_engine/extractors/extract_zernike_surface.py +optimization_engine/extractors/zernike_helpers.py +optimization_engine/extractors/extract_principal_stress.py +optimization_engine/extractors/extract_strain_energy.py +optimization_engine/extractors/extract_spc_forces.py +optimization_engine/extractors/extract_temperature.py +optimization_engine/extractors/extract_modal_mass.py +optimization_engine/extractors/introspect_part.py +optimization_engine/extractors/field_data_extractor.py +optimization_engine/extractors/stiffness_calculator.py +optimization_engine/extractors/test_phase3_extractors.py +``` + +**GNN Module (12 files):** +``` +optimization_engine/gnn/__init__.py +optimization_engine/gnn/zernike_gnn.py +optimization_engine/gnn/polar_graph.py +optimization_engine/gnn/train_zernike_gnn.py +optimization_engine/gnn/gnn_optimizer.py +optimization_engine/gnn/extract_displacement_field.py +optimization_engine/gnn/differentiable_zernike.py +optimization_engine/gnn/backfill_field_data.py +optimization_engine/gnn/test_polar_graph.py +optimization_engine/gnn/test_field_extraction.py +optimization_engine/gnn/test_new_extraction.py +``` + +**Hooks Module (12 files):** +``` +optimization_engine/hooks/__init__.py +optimization_engine/hooks/examples.py +optimization_engine/hooks/test_introspection.py +optimization_engine/hooks/test_hooks.py +optimization_engine/hooks/nx_cad/__init__.py +optimization_engine/hooks/nx_cad/model_introspection.py +optimization_engine/hooks/nx_cad/part_manager.py +optimization_engine/hooks/nx_cad/expression_manager.py +optimization_engine/hooks/nx_cad/geometry_query.py +optimization_engine/hooks/nx_cad/feature_manager.py +optimization_engine/hooks/nx_cae/__init__.py +optimization_engine/hooks/nx_cae/solver_manager.py +``` + +**Plugins Module (5 files):** +``` +optimization_engine/plugins/__init__.py +optimization_engine/plugins/hook_manager.py +optimization_engine/plugins/post_calculation/safety_factor_constraint.py +optimization_engine/plugins/pre_solve/*.py +optimization_engine/plugins/post_solve/*.py +``` + +**Insights Module (12 files):** +``` +optimization_engine/insights/__init__.py +optimization_engine/insights/base.py +optimization_engine/insights/zernike_wfe.py +optimization_engine/insights/zernike_opd_comparison.py +optimization_engine/insights/zernike_dashboard.py +optimization_engine/insights/msf_zernike.py +optimization_engine/insights/stress_field.py +optimization_engine/insights/modal_analysis.py +optimization_engine/insights/thermal_field.py +optimization_engine/insights/design_space.py +``` + +**Utils Module (6 files):** +``` +optimization_engine/utils/__init__.py +optimization_engine/utils/trial_manager.py +optimization_engine/utils/dashboard_db.py +optimization_engine/utils/nx_session_manager.py +optimization_engine/utils/nx_file_discovery.py +optimization_engine/utils/study_cleanup.py +``` + +**Validators Module (5 files):** +``` +optimization_engine/validators/__init__.py +optimization_engine/validators/study_validator.py +optimization_engine/validators/results_validator.py +optimization_engine/validators/model_validator.py +optimization_engine/validators/config_validator.py +``` + +**Templates (3 files):** +``` +optimization_engine/templates/__init__.py +optimization_engine/templates/run_optimization_template.py +optimization_engine/templates/run_nn_optimization_template.py +``` + +**Model Discovery (2 files):** +``` +optimization_engine/model_discovery/__init__.py +optimization_engine/model_discovery/discover_parts.py +``` + +#### B. STUDIES DIRECTORY (80+ files) + +**M1_Mirror Studies (35+ files):** +``` +studies/M1_Mirror/m1_mirror_cost_reduction/run_optimization.py +studies/M1_Mirror/m1_mirror_zernike_optimization/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_flat_back_V1/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_flat_back_V3/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_flat_back_V4/run_turbo_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_flat_back_V5/run_turbo_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_flat_back_V6/run_optimization.py +studies/M1_Mirror/m1_mirror_turbo_V1/run_optimization.py +studies/M1_Mirror/m1_mirror_turbo_V2/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V2/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V3/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V4/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V5/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V5/_analyze_baseline.py +studies/M1_Mirror/m1_mirror_cost_reduction_V6/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V6/_analyze_baseline.py +studies/M1_Mirror/m1_mirror_cost_reduction_V7/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V8/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V9/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V10/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V11/run_optimization.py +studies/M1_Mirror/m1_mirror_cost_reduction_V12/run_optimization.py +studies/M1_Mirror/m1_mirror_adaptive_V11/run_optimization.py +studies/M1_Mirror/m1_mirror_adaptive_V12/run_optimization.py +studies/M1_Mirror/m1_mirror_adaptive_V12/run_gnn_turbo.py +studies/M1_Mirror/m1_mirror_adaptive_V12/validate_gnn_best.py +studies/M1_Mirror/m1_mirror_adaptive_V12/compute_full_calibration.py +studies/M1_Mirror/m1_mirror_adaptive_V13/run_optimization.py +studies/M1_Mirror/m1_mirror_adaptive_V14/run_optimization.py +studies/M1_Mirror/m1_mirror_adaptive_V14/run_lbfgs_polish.py +studies/M1_Mirror/m1_mirror_adaptive_V15/run_optimization.py +studies/M1_Mirror/m1_mirror_surrogate_turbo/scripts/01_reextract_studies.py +studies/M1_Mirror/m1_mirror_surrogate_turbo/scripts/02_aggregate_gnn_data.py +studies/M1_Mirror/m1_mirror_surrogate_turbo/scripts/03_train_gnn.py +``` + +**Simple_Bracket Studies (12+ files):** +``` +studies/Simple_Bracket/bracket_stiffness_optimization/run_optimization.py +studies/Simple_Bracket/bracket_stiffness_optimization/bracket_stiffness_extractor.py +studies/Simple_Bracket/bracket_stiffness_optimization_V2/run_optimization.py +studies/Simple_Bracket/bracket_stiffness_optimization_V2/bracket_stiffness_extractor.py +studies/Simple_Bracket/bracket_stiffness_optimization_V3/run_optimization.py +studies/Simple_Bracket/bracket_stiffness_optimization_V3/bracket_stiffness_extractor.py +studies/Simple_Bracket/bracket_stiffness_optimization_atomizerfield/run_optimization.py +studies/Simple_Bracket/bracket_pareto_3obj/run_optimization.py +studies/Simple_Bracket/bracket_pareto_3obj/run_nn_optimization.py +``` + +**Other Studies (15+ files):** +``` +studies/Simple_Beam/simple_beam_optimization/run_optimization.py +studies/UAV_Arm/uav_arm_optimization/run_optimization.py +studies/UAV_Arm/uav_arm_optimization_V2/run_optimization.py +studies/UAV_Arm/uav_arm_atomizerfield_test/run_optimization.py +studies/Drone_Gimbal/drone_gimbal_arm_optimization/run_optimization.py +studies/bracket_displacement_maximizing/run_optimization.py +studies/bracket_displacement_maximizing/run_optimization_old.py +studies/bracket_displacement_maximizing/run_substudy.py +studies/bracket_displacement_maximizing/test_fix.py +studies/_Other/training_data_export_test/run_optimization.py +``` + +#### C. TESTS DIRECTORY (45+ files) ``` -extractors/ ✓ 20+ extractors, clean __init__.py -insights/ ✓ 8 insight types, registry pattern -hooks/ ✓ nx_cad/, nx_cae/ subdirs -gnn/ ✓ Neural surrogate for Zernike -templates/ ✓ Config templates -schemas/ ✓ JSON schemas -validators/ ✓ Validation code -plugins/ ✓ Hook manager system -utils/ ✓ Utility functions -custom_functions/ ✓ NX material generator -model_discovery/ ✓ Model introspection -future/ ✓ Experimental code +tests/audit_v10_wfe.py +tests/audit_v10_method_diff.py +tests/audit_v10_fix.py +tests/analyze_v11.py +tests/debug_insights.py +tests/debug_figure_coords.py +tests/compare_v8_zernike_methods.py +tests/test_zernike_opd_with_prescription.py +tests/test_zernike_opd_comparison.py +tests/test_zernike_methods_comparison.py +tests/test_zernike_insight.py +tests/test_zernike_import.py +tests/test_training_data_exporter.py +tests/test_timestamp_verification.py +tests/test_task_1_2_integration.py +tests/test_step_classifier.py +tests/test_research_agent.py +tests/test_phase_3_2_llm_mode.py +tests/test_phase_3_1_integration.py +tests/test_phase_2_5_intelligent_gap_detection.py +tests/test_plugin_system.py +tests/test_optimization_setup_wizard.py +tests/test_modal_deformation_request.py +tests/test_llm_runner_init.py +tests/test_llm_complex_request.py +tests/test_lifecycle_hook_integration.py +tests/test_knowledge_base_search.py +tests/test_journal_optimization.py +tests/test_interactive_session.py +tests/test_hooks_with_bracket.py +tests/test_complex_multiobj_request.py +tests/test_complete_research_workflow.py +tests/test_code_generation.py +tests/test_cbar_genetic_algorithm.py +tests/test_cbush_optimization.py +tests/test_bracket_llm_runner.py +tests/test_bracket_full_optimization.py +tests/test_beam_workflow.py +tests/test_api_verification.py +tests/interactive_optimization_setup.py +tests/demo_research_agent.py +tests/run_beam_benchmarking.py +tests/setup_beam_optimization.py +tests/run_5trial_test.py +tests/run_benchmarking_simple.py +tests/run_beam_benchmarking_clean.py +``` + +#### D. ROOT & TOOLS (10+ files) + +``` +atomizer.py +run_cleanup.py +run_training_fea.py +tools/zernike_html_generator.py +tools/extract_mirror_optical_specs.py +examples/interactive_research_session.py +``` + +#### E. DASHBOARD BACKEND (3 files) + +``` +atomizer-dashboard/backend/api/main.py +atomizer-dashboard/backend/api/routes/optimization.py +atomizer-dashboard/backend/api/routes/insights.py +``` + +#### F. ARCHIVE (50+ files - OPTIONAL) + +``` +archive/deprecated/result_extractors/extractors.py +archive/deprecated/result_extractors/generated/extract_expression.py +archive/scripts/create_intelligent_study.py +archive/scripts/create_circular_plate_study_v2.py +archive/scripts/create_circular_plate_study.py +archive/scripts/run_nn_optimization.py +archive/scripts/run_calibration_loop.py +archive/scripts/run_validated_nn_optimization.py +archive/scripts/validate_surrogate_real_data.py +archive/scripts/visualize_nn_optimization.py +archive/test_scripts/*.py (10+ files) +``` + +### 1.3 Documentation Files to UPDATE (100+ files) + +#### A. ROOT DOCUMENTATION + +``` +CLAUDE.md (lines 141, 152, 166, 200, 221, 235, 253, 283, 592-603) +README.md (lines 98, 193, 213, 272) +CHANGELOG.md (lines 31, 40, 82) +DEVELOPMENT.md (lines 40, 64, 77, 90, 215-277, 398, 532, 535) +PROTOCOL.md (lines 481, 704-705, 760, 812, 924-939, 1118, 1262, 1380, 1406, 1501-1596, 1851) +``` + +#### B. .CLAUDE DIRECTORY + +``` +.claude/ATOMIZER_CONTEXT.md (lines 37, 40, 71, 235, 238, 248, 273, 282, 288, 421-472) +.claude/CHEATSHEET.md (lines 7-10, 33-35, 144, 156, 353-472) +.claude/skills/00_BOOTSTRAP.md (line 236) +.claude/skills/01_CHEATSHEET.md (lines 7-10, 33-35, 144, 156, 353-472) +.claude/skills/analyze-workflow.md (line 37) +.claude/skills/analyze-model.md (lines 52, 99, 183) +.claude/skills/atomizer.md (lines 15, 27, 62, 193, 286, 317, 323) +.claude/skills/DEV_DOCUMENTATION.md (lines 115-117, 141) +.claude/skills/core/study-creation-core.md (lines 7-9, 43, 157, 160, 259, 279, 454-459, 643, 706) +.claude/skills/modules/extractors-catalog.md (lines 24-31, 42, 63, 86, 110-271) +.claude/skills/modules/insights-catalog.md (lines 53-237) +.claude/skills/modules/neural-acceleration.md (lines 48, 318) +.claude/skills/modules/nx-docs-lookup.md (line 190) +.claude/skills/modules/zernike-optimization.md (lines 35, 73, 111) +.claude/skills/modules/atomizer_fast_solver_technologies.md (lines 739, 744) +.claude/skills/modules/learning-atomizer-core.md +.claude/skills/archive/create-study-wizard.md (lines 13, 74, 177, 360, 402) +``` + +#### C. DOCS DIRECTORY - PROTOCOLS + +``` +docs/00_INDEX.md (line 284) +docs/01_PROTOCOLS.md (lines 277, 352-355, 528, 535-600, 1355, 1434, 1460) +docs/02_ARCHITECTURE.md (lines 68-69, 104, 123, 193, 254, 279, 299, 376-495) +docs/03_GETTING_STARTED.md (line 286) +docs/protocols/operations/OP_01_CREATE_STUDY.md (lines 162, 301-303) +docs/protocols/operations/OP_02_RUN_OPTIMIZATION.md +docs/protocols/operations/OP_03_MONITOR_PROGRESS.md +docs/protocols/operations/OP_04_ANALYZE_RESULTS.md +docs/protocols/operations/OP_05_EXPORT_TRAINING_DATA.md +docs/protocols/operations/OP_06_TROUBLESHOOT.md +docs/protocols/system/SYS_10_IMSO.md (lines 224, 302-305) +docs/protocols/system/SYS_11_MULTI_OBJECTIVE.md (lines 322-330) +docs/protocols/system/SYS_12_EXTRACTOR_LIBRARY.md (lines 71-851 - MANY references) +docs/protocols/system/SYS_13_DASHBOARD_TRACKING.md (lines 60, 85, 315) +docs/protocols/system/SYS_14_NEURAL_ACCELERATION.md (lines 231-1080) +docs/protocols/system/SYS_15_METHOD_SELECTOR.md (lines 42-422) +docs/protocols/system/SYS_16_STUDY_INSIGHTS.md (lines 62-498) +docs/protocols/extensions/EXT_01_CREATE_EXTRACTOR.md (lines 35-287) +docs/protocols/extensions/EXT_02_CREATE_HOOK.md (lines 45-357) +docs/protocols/extensions/EXT_03_CREATE_INSIGHT.md +docs/protocols/extensions/EXT_04_CREATE_SKILL.md (lines 263, 274) +docs/protocols/extensions/templates/hook_template.py (lines 128, 164) +``` + +#### D. DOCS SUBDIRECTORIES + +``` +docs/04_USER_GUIDES/*.md (estimated 20+ files) +docs/05_API_REFERENCE/*.md (estimated 10+ files) +docs/06_PHYSICS/*.md (estimated 15+ files) +docs/07_DEVELOPMENT/*.md (estimated 20+ files) +docs/08_ARCHIVE/*.md (estimated 30+ files) +``` + +### 1.4 Configuration Files to UPDATE + +``` +optimization_engine/feature_registry.json (50+ path entries on lines 21-683) +optimization_engine/schemas/README.md (lines 19, 26) +studies/Simple_Bracket/bracket_stiffness_optimization_atomizerfield/1_setup/optimization_config.json +studies/Simple_Bracket/bracket_pareto_3obj/1_setup/optimization_config.json +pyproject.toml (lines 53, 76) ``` --- -## Part 2: Target Structure +## Part 2: Import Mapping Reference -### 2.1 Proposed Final Structure - -``` -optimization_engine/ -│ -├── __init__.py # Updated with backwards-compat aliases -│ -├── core/ # NEW - Optimization engine core -│ ├── __init__.py -│ ├── runner.py -│ ├── base_runner.py -│ ├── runner_with_neural.py -│ ├── intelligent_optimizer.py -│ ├── method_selector.py -│ ├── strategy_selector.py -│ └── strategy_portfolio.py -│ -├── processors/ # NEW - Data processing & algorithms -│ ├── __init__.py -│ ├── surrogates/ -│ │ ├── __init__.py -│ │ ├── neural_surrogate.py -│ │ ├── generic_surrogate.py -│ │ ├── adaptive_surrogate.py -│ │ ├── simple_mlp_surrogate.py -│ │ ├── active_learning_surrogate.py -│ │ ├── surrogate_tuner.py -│ │ ├── auto_trainer.py -│ │ └── training_data_exporter.py -│ │ -│ └── dynamic_response/ # NEW - From master plan -│ ├── __init__.py -│ ├── modal_database.py -│ ├── transfer_functions.py -│ ├── random_vibration.py -│ ├── psd_profiles.py -│ └── utils/ -│ -├── nx/ # NEW - NX/Nastran integration -│ ├── __init__.py -│ ├── solver.py # Was nx_solver.py -│ ├── updater.py # Was nx_updater.py -│ ├── session_manager.py # Was nx_session_manager.py -│ ├── solve_simulation.py -│ ├── solve_simulation_simple.py -│ ├── model_cleanup.py -│ ├── export_expressions.py -│ ├── import_expressions.py -│ └── mesh_converter.py -│ -├── study/ # NEW - Study management -│ ├── __init__.py -│ ├── creator.py # Was study_creator.py -│ ├── wizard.py # Was study_wizard.py -│ ├── state.py # Was study_state.py -│ ├── reset.py # Was study_reset.py -│ ├── continuation.py # Was study_continuation.py -│ ├── benchmarking.py # Was benchmarking_substudy.py -│ └── history_generator.py # Was generate_history_from_trials.py -│ -├── reporting/ # NEW - Reports and analysis -│ ├── __init__.py -│ ├── report_generator.py -│ ├── markdown_report.py -│ ├── results_analyzer.py -│ ├── visualizer.py -│ └── landscape_analyzer.py -│ -├── config/ # NEW - Configuration -│ ├── __init__.py -│ ├── manager.py -│ ├── builder.py -│ ├── setup_wizard.py -│ └── capability_matcher.py -│ -├── extractors/ # EXISTING - Add op2_extractor, extractor_library -├── insights/ # EXISTING -├── hooks/ # EXISTING -├── gnn/ # EXISTING -├── templates/ # EXISTING -├── schemas/ # EXISTING -├── validators/ # EXISTING - Add simulation_validator -├── plugins/ # EXISTING -├── utils/ # EXISTING - Add logger, auto_doc, etc. -├── custom_functions/ # EXISTING -├── model_discovery/ # EXISTING -└── future/ # EXISTING - Move agents here - ├── research_agent.py - ├── pynastran_research_agent.py - ├── targeted_research_planner.py - ├── workflow_decomposer.py - └── step_classifier.py -``` - ---- - -## Part 3: Import Mapping Tables - -### 3.1 Old → New Import Mapping - -This is the critical reference for all updates: +### 2.1 Complete Old → New Import Mapping ```python -# CORE OPTIMIZATION -"from optimization_engine.runner" → "from optimization_engine.core.runner" -"from optimization_engine.base_runner" → "from optimization_engine.core.base_runner" -"from optimization_engine.runner_with_neural" → "from optimization_engine.core.runner_with_neural" -"from optimization_engine.intelligent_optimizer" → "from optimization_engine.core.intelligent_optimizer" -"from optimization_engine.method_selector" → "from optimization_engine.core.method_selector" -"from optimization_engine.strategy_selector" → "from optimization_engine.core.strategy_selector" -"from optimization_engine.strategy_portfolio" → "from optimization_engine.core.strategy_portfolio" +# ============================================================================= +# CORE MODULE +# ============================================================================= +"from optimization_engine.runner" → "from optimization_engine.core.runner" +"from optimization_engine.base_runner" → "from optimization_engine.core.base_runner" +"from optimization_engine.runner_with_neural" → "from optimization_engine.core.runner_with_neural" +"from optimization_engine.intelligent_optimizer" → "from optimization_engine.core.intelligent_optimizer" +"from optimization_engine.method_selector" → "from optimization_engine.core.method_selector" +"from optimization_engine.strategy_selector" → "from optimization_engine.core.strategy_selector" +"from optimization_engine.strategy_portfolio" → "from optimization_engine.core.strategy_portfolio" +"from optimization_engine.gradient_optimizer" → "from optimization_engine.core.gradient_optimizer" -# SURROGATES -"from optimization_engine.neural_surrogate" → "from optimization_engine.processors.surrogates.neural_surrogate" -"from optimization_engine.generic_surrogate" → "from optimization_engine.processors.surrogates.generic_surrogate" -"from optimization_engine.adaptive_surrogate" → "from optimization_engine.processors.surrogates.adaptive_surrogate" -"from optimization_engine.simple_mlp_surrogate" → "from optimization_engine.processors.surrogates.simple_mlp_surrogate" -"from optimization_engine.active_learning_surrogate" → "from optimization_engine.processors.surrogates.active_learning_surrogate" -"from optimization_engine.surrogate_tuner" → "from optimization_engine.processors.surrogates.surrogate_tuner" +"import optimization_engine.runner" → "import optimization_engine.core.runner" +"import optimization_engine.intelligent_optimizer" → "import optimization_engine.core.intelligent_optimizer" -# NX INTEGRATION -"from optimization_engine.nx_solver" → "from optimization_engine.nx.solver" -"from optimization_engine.nx_updater" → "from optimization_engine.nx.updater" -"from optimization_engine.nx_session_manager" → "from optimization_engine.nx.session_manager" -"from optimization_engine.solve_simulation" → "from optimization_engine.nx.solve_simulation" -"from optimization_engine.model_cleanup" → "from optimization_engine.nx.model_cleanup" -"from optimization_engine.export_expressions" → "from optimization_engine.nx.export_expressions" -"from optimization_engine.import_expressions" → "from optimization_engine.nx.import_expressions" -"from optimization_engine.mesh_converter" → "from optimization_engine.nx.mesh_converter" +# ============================================================================= +# SURROGATES MODULE +# ============================================================================= +"from optimization_engine.neural_surrogate" → "from optimization_engine.processors.surrogates.neural_surrogate" +"from optimization_engine.generic_surrogate" → "from optimization_engine.processors.surrogates.generic_surrogate" +"from optimization_engine.adaptive_surrogate" → "from optimization_engine.processors.surrogates.adaptive_surrogate" +"from optimization_engine.simple_mlp_surrogate" → "from optimization_engine.processors.surrogates.simple_mlp_surrogate" +"from optimization_engine.active_learning_surrogate" → "from optimization_engine.processors.surrogates.active_learning_surrogate" +"from optimization_engine.surrogate_tuner" → "from optimization_engine.processors.surrogates.surrogate_tuner" +"from optimization_engine.auto_trainer" → "from optimization_engine.processors.surrogates.auto_trainer" +"from optimization_engine.training_data_exporter" → "from optimization_engine.processors.surrogates.training_data_exporter" -# STUDY MANAGEMENT -"from optimization_engine.study_creator" → "from optimization_engine.study.creator" -"from optimization_engine.study_wizard" → "from optimization_engine.study.wizard" -"from optimization_engine.study_state" → "from optimization_engine.study.state" -"from optimization_engine.study_reset" → "from optimization_engine.study.reset" -"from optimization_engine.study_continuation" → "from optimization_engine.study.continuation" +# ============================================================================= +# NX MODULE +# ============================================================================= +"from optimization_engine.nx_solver" → "from optimization_engine.nx.solver" +"from optimization_engine.nx_updater" → "from optimization_engine.nx.updater" +"from optimization_engine.nx_session_manager" → "from optimization_engine.nx.session_manager" +"from optimization_engine.solve_simulation" → "from optimization_engine.nx.solve_simulation" +"from optimization_engine.solve_simulation_simple" → "from optimization_engine.nx.solve_simulation_simple" +"from optimization_engine.model_cleanup" → "from optimization_engine.nx.model_cleanup" +"from optimization_engine.export_expressions" → "from optimization_engine.nx.export_expressions" +"from optimization_engine.import_expressions" → "from optimization_engine.nx.import_expressions" +"from optimization_engine.mesh_converter" → "from optimization_engine.nx.mesh_converter" -# REPORTING -"from optimization_engine.generate_report" → "from optimization_engine.reporting.report_generator" -"from optimization_engine.generate_report_markdown" → "from optimization_engine.reporting.markdown_report" -"from optimization_engine.comprehensive_results" → "from optimization_engine.reporting.results_analyzer" -"from optimization_engine.visualizer" → "from optimization_engine.reporting.visualizer" -"from optimization_engine.landscape_analyzer" → "from optimization_engine.reporting.landscape_analyzer" +"import optimization_engine.nx_solver" → "import optimization_engine.nx.solver" -# CONFIG -"from optimization_engine.config_manager" → "from optimization_engine.config.manager" +# ============================================================================= +# STUDY MODULE +# ============================================================================= +"from optimization_engine.study_creator" → "from optimization_engine.study.creator" +"from optimization_engine.study_wizard" → "from optimization_engine.study.wizard" +"from optimization_engine.study_state" → "from optimization_engine.study.state" +"from optimization_engine.study_reset" → "from optimization_engine.study.reset" +"from optimization_engine.study_continuation" → "from optimization_engine.study.continuation" +"from optimization_engine.benchmarking_substudy" → "from optimization_engine.study.benchmarking" +"from optimization_engine.generate_history_from_trials" → "from optimization_engine.study.history_generator" + +# ============================================================================= +# REPORTING MODULE +# ============================================================================= +"from optimization_engine.generate_report" → "from optimization_engine.reporting.report_generator" +"from optimization_engine.generate_report_markdown" → "from optimization_engine.reporting.markdown_report" +"from optimization_engine.comprehensive_results_analyzer" → "from optimization_engine.reporting.results_analyzer" +"from optimization_engine.visualizer" → "from optimization_engine.reporting.visualizer" +"from optimization_engine.landscape_analyzer" → "from optimization_engine.reporting.landscape_analyzer" + +# ============================================================================= +# CONFIG MODULE +# ============================================================================= +"from optimization_engine.config_manager" → "from optimization_engine.config.manager" "from optimization_engine.optimization_config_builder" → "from optimization_engine.config.builder" -"from optimization_engine.optimization_setup_wizard" → "from optimization_engine.config.setup_wizard" -"from optimization_engine.capability_matcher" → "from optimization_engine.config.capability_matcher" +"from optimization_engine.optimization_setup_wizard" → "from optimization_engine.config.setup_wizard" +"from optimization_engine.capability_matcher" → "from optimization_engine.config.capability_matcher" +"from optimization_engine.template_loader" → "from optimization_engine.config.template_loader" -# UTILITIES (moving to utils/) -"from optimization_engine.logger" → "from optimization_engine.utils.logger" -"from optimization_engine.auto_doc" → "from optimization_engine.utils.auto_doc" -"from optimization_engine.realtime_tracking" → "from optimization_engine.utils.realtime_tracking" -"from optimization_engine.codebase_analyzer" → "from optimization_engine.utils.codebase_analyzer" -"from optimization_engine.pruning_logger" → "from optimization_engine.utils.pruning_logger" +# ============================================================================= +# UTILS MODULE (moving to existing utils/) +# ============================================================================= +"from optimization_engine.logger" → "from optimization_engine.utils.logger" +"from optimization_engine.auto_doc" → "from optimization_engine.utils.auto_doc" +"from optimization_engine.realtime_tracking" → "from optimization_engine.utils.realtime_tracking" +"from optimization_engine.codebase_analyzer" → "from optimization_engine.utils.codebase_analyzer" +"from optimization_engine.pruning_logger" → "from optimization_engine.utils.pruning_logger" -# RESEARCH/AGENTS (moving to future/) -"from optimization_engine.research_agent" → "from optimization_engine.future.research_agent" -"from optimization_engine.workflow_decomposer" → "from optimization_engine.future.workflow_decomposer" -"from optimization_engine.step_classifier" → "from optimization_engine.future.step_classifier" +# ============================================================================= +# FUTURE MODULE (research/experimental) +# ============================================================================= +"from optimization_engine.research_agent" → "from optimization_engine.future.research_agent" +"from optimization_engine.pynastran_research_agent" → "from optimization_engine.future.pynastran_research_agent" +"from optimization_engine.targeted_research_planner" → "from optimization_engine.future.targeted_research_planner" +"from optimization_engine.workflow_decomposer" → "from optimization_engine.future.workflow_decomposer" +"from optimization_engine.step_classifier" → "from optimization_engine.future.step_classifier" +"from optimization_engine.llm_optimization_runner" → "from optimization_engine.future.llm_optimization_runner" +"from optimization_engine.llm_workflow_analyzer" → "from optimization_engine.future.llm_workflow_analyzer" +"from optimization_engine.extractor_orchestrator" → "from optimization_engine.future.extractor_orchestrator" +"from optimization_engine.inline_code_generator" → "from optimization_engine.future.inline_code_generator" +"from optimization_engine.hook_generator" → "from optimization_engine.future.hook_generator" + +# ============================================================================= +# EXTRACTORS MODULE (moving to existing extractors/) +# ============================================================================= +"from optimization_engine.op2_extractor" → "from optimization_engine.extractors.op2_extractor" +"from optimization_engine.extractor_library" → "from optimization_engine.extractors.extractor_library" + +# ============================================================================= +# VALIDATORS MODULE (moving to existing validators/) +# ============================================================================= +"from optimization_engine.simulation_validator" → "from optimization_engine.validators.simulation_validator" + +# ============================================================================= +# PROCESSORS MODULE (new) +# ============================================================================= +"from optimization_engine.adaptive_characterization" → "from optimization_engine.processors.adaptive_characterization" + +# ============================================================================= +# UNCHANGED (these modules stay in place) +# ============================================================================= +# extractors/ - stays as optimization_engine.extractors.* +# gnn/ - stays as optimization_engine.gnn.* +# hooks/ - stays as optimization_engine.hooks.* +# insights/ - stays as optimization_engine.insights.* +# plugins/ - stays as optimization_engine.plugins.* +# validators/ - stays as optimization_engine.validators.* +# utils/ - stays as optimization_engine.utils.* +# templates/ - stays as optimization_engine.templates.* +# schemas/ - stays as optimization_engine.schemas.* +# model_discovery/ - stays as optimization_engine.model_discovery.* ``` -### 3.2 Backwards Compatibility Aliases +### 2.2 Documentation Path Mapping -Add to `optimization_engine/__init__.py` for transition period: +``` +# In markdown files, update these path references: -```python -# BACKWARDS COMPATIBILITY ALIASES -# These allow old imports to work during migration period -# Remove after all code is updated - -# Core -from optimization_engine.core.runner import * -from optimization_engine.core.base_runner import * -from optimization_engine.core.intelligent_optimizer import * - -# NX -from optimization_engine.nx import solver as nx_solver -from optimization_engine.nx import updater as nx_updater -from optimization_engine.nx import session_manager as nx_session_manager -from optimization_engine.nx import solve_simulation - -# Study -from optimization_engine.study import creator as study_creator -from optimization_engine.study import wizard as study_wizard -from optimization_engine.study import state as study_state - -# Surrogates -from optimization_engine.processors.surrogates import neural_surrogate -from optimization_engine.processors.surrogates import generic_surrogate - -# Config -from optimization_engine.config import manager as config_manager - -# Utils -from optimization_engine.utils import logger - -# Deprecation warnings (optional) -import warnings -def __getattr__(name): - deprecated = { - 'nx_solver': 'optimization_engine.nx.solver', - 'study_creator': 'optimization_engine.study.creator', - # ... more mappings - } - if name in deprecated: - warnings.warn( - f"Importing {name} from optimization_engine is deprecated. " - f"Use {deprecated[name]} instead.", - DeprecationWarning, - stacklevel=2 - ) - # Return the module anyway for compatibility - raise AttributeError(f"module 'optimization_engine' has no attribute '{name}'") +optimization_engine/runner.py → optimization_engine/core/runner.py +optimization_engine/nx_solver.py → optimization_engine/nx/solver.py +optimization_engine/nx_updater.py → optimization_engine/nx/updater.py +optimization_engine/study_creator.py → optimization_engine/study/creator.py +optimization_engine/study_wizard.py → optimization_engine/study/wizard.py +optimization_engine/neural_surrogate.py → optimization_engine/processors/surrogates/neural_surrogate.py +optimization_engine/config_manager.py → optimization_engine/config/manager.py +optimization_engine/logger.py → optimization_engine/utils/logger.py +# ... etc for all moved files ``` --- -## Part 4: Files Requiring Updates +## Part 3: Execution Plan -### 4.1 Internal optimization_engine Files (90+ files) - -**Highest-impact internal files:** - -| File | Imports to Update | Priority | -|------|-------------------|----------| -| `runner.py` | nx_solver, config_manager, extractors | Critical | -| `base_runner.py` | config_manager, validators | Critical | -| `intelligent_optimizer.py` | runner, neural_surrogate, method_selector | Critical | -| `runner_with_neural.py` | runner, neural_surrogate | High | -| `gnn/gnn_optimizer.py` | polar_graph, zernike_gnn, nx_solver | High | -| `hooks/nx_cad/*.py` | nx_session_manager | High | -| `plugins/hook_manager.py` | validators, config | Medium | - -### 4.2 Study Scripts (30+ folders, 153 imports) - -**Pattern in every study's `run_optimization.py`:** - -```python -# BEFORE -from optimization_engine.nx_solver import run_nx_simulation -from optimization_engine.extractors import ZernikeExtractor -from optimization_engine.gnn.gnn_optimizer import ZernikeGNNOptimizer - -# AFTER -from optimization_engine.nx.solver import run_nx_simulation -from optimization_engine.extractors import ZernikeExtractor # unchanged -from optimization_engine.gnn.gnn_optimizer import ZernikeGNNOptimizer # unchanged -``` - -**Studies requiring updates:** - -``` -studies/ -├── M1_Mirror/ -│ ├── m1_mirror_adaptive_V*/run_optimization.py (12 files) -│ └── [other variants] -├── Simple_Bracket/ -│ ├── bracket_*/run_optimization.py (8 files) -├── UAV_Arm/ -│ ├── */run_optimization.py (4 files) -├── Drone_Gimbal/ -├── Simple_Beam/ -└── [others] -``` - -### 4.3 Test Files (30+ files, 79 imports) - -Located in `tests/`: - -``` -tests/ -├── test_extractors.py -├── test_zernike_*.py (5+ files) -├── test_neural_surrogate.py -├── test_gnn_*.py -├── test_nx_solver.py -├── test_study_*.py -└── [others] -``` - -### 4.4 Dashboard Backend (3 files, 11 imports) - -``` -atomizer-dashboard/backend/api/ -├── main.py # sys.path setup -├── routes/optimization.py # study management imports -└── routes/insights.py # insight imports -``` - -### 4.5 Documentation Files (119 files) - -**Protocols requiring updates:** - -| Protocol | References | Changes Needed | -|----------|------------|----------------| -| SYS_10_IMSO.md | intelligent_optimizer | core.intelligent_optimizer | -| SYS_12_EXTRACTOR_LIBRARY.md | extractors/* | None (unchanged) | -| SYS_14_NEURAL_ACCELERATION.md | neural_surrogate, gnn/* | processors.surrogates.* | -| SYS_15_METHOD_SELECTOR.md | method_selector | core.method_selector | -| OP_01_CREATE_STUDY.md | study_creator, study_wizard | study.creator, study.wizard | -| OP_02_RUN_OPTIMIZATION.md | runner, nx_solver | core.runner, nx.solver | - -**Skill files requiring updates:** - -| Skill File | Changes | -|------------|---------| -| 01_CHEATSHEET.md | Path references | -| core/study-creation-core.md | Import examples | -| modules/extractors-catalog.md | Directory paths | -| modules/neural-acceleration.md | Surrogate imports | - -### 4.6 JSON Configuration Files (4 files) - -**feature_registry.json** (878 lines, 50+ path references): - -```json -// BEFORE -"file_path": "optimization_engine/neural_surrogate.py" - -// AFTER -"file_path": "optimization_engine/processors/surrogates/neural_surrogate.py" -``` - ---- - -## Part 5: Migration Execution Plan - -### Phase 0: Pre-Migration (30 min) - -1. **Create full backup** - ```bash - git stash # Save any uncommitted changes - git checkout -b refactor/optimization-engine-reorganization - cp -r optimization_engine optimization_engine_backup - ``` - -2. **Run baseline tests** - ```bash - python -m pytest tests/ -v --tb=short > baseline_tests.log 2>&1 - ``` - -3. **Document current working state** - - Run one study end-to-end - - Verify dashboard loads - - Note any existing failures - -### Phase 1: Create Directory Structure (15 min) +### Phase 0: Preparation (30 minutes) ```bash -# Create new directories +# 1. Create backup branch +git checkout -b backup/pre-reorganization +git push origin backup/pre-reorganization +git checkout main +git checkout -b refactor/optimization-engine-v2 + +# 2. Create full backup +cp -r optimization_engine optimization_engine_BACKUP_$(date +%Y%m%d) + +# 3. Run baseline tests and save output +python -m pytest tests/ -v --tb=short 2>&1 | tee baseline_tests_$(date +%Y%m%d).log + +# 4. Count current test status +echo "Baseline test count:" +grep -E "passed|failed|error" baseline_tests_*.log | tail -5 + +# 5. Verify NX is not running (to prevent file locks) +tasklist | grep -i ugraf || echo "NX not running - safe to proceed" +``` + +### Phase 1: Create Directory Structure (15 minutes) + +```bash +# Create all new directories mkdir -p optimization_engine/core mkdir -p optimization_engine/processors/surrogates mkdir -p optimization_engine/processors/dynamic_response @@ -488,7 +685,7 @@ mkdir -p optimization_engine/study mkdir -p optimization_engine/reporting mkdir -p optimization_engine/config -# Create __init__.py files +# Create __init__.py files for new directories touch optimization_engine/core/__init__.py touch optimization_engine/processors/__init__.py touch optimization_engine/processors/surrogates/__init__.py @@ -499,25 +696,32 @@ touch optimization_engine/reporting/__init__.py touch optimization_engine/config/__init__.py ``` -### Phase 2: Move Files (30 min) +### Phase 2: Move Files (45 minutes) -**Execute in this order to minimize circular import issues:** +**Execute in this specific order to minimize issues:** ```bash -# 1. UTILITIES FIRST (no dependencies) +# ============================================================================= +# STEP 2.1: UTILITIES FIRST (no dependencies on other moved files) +# ============================================================================= mv optimization_engine/logger.py optimization_engine/utils/ mv optimization_engine/auto_doc.py optimization_engine/utils/ mv optimization_engine/realtime_tracking.py optimization_engine/utils/ mv optimization_engine/codebase_analyzer.py optimization_engine/utils/ mv optimization_engine/pruning_logger.py optimization_engine/utils/ -# 2. CONFIG (low dependencies) +# ============================================================================= +# STEP 2.2: CONFIG (depends only on logger) +# ============================================================================= mv optimization_engine/config_manager.py optimization_engine/config/manager.py mv optimization_engine/optimization_config_builder.py optimization_engine/config/builder.py mv optimization_engine/optimization_setup_wizard.py optimization_engine/config/setup_wizard.py mv optimization_engine/capability_matcher.py optimization_engine/config/capability_matcher.py +mv optimization_engine/template_loader.py optimization_engine/config/template_loader.py -# 3. NX INTEGRATION +# ============================================================================= +# STEP 2.3: NX INTEGRATION (core infrastructure) +# ============================================================================= mv optimization_engine/nx_solver.py optimization_engine/nx/solver.py mv optimization_engine/nx_updater.py optimization_engine/nx/updater.py mv optimization_engine/nx_session_manager.py optimization_engine/nx/session_manager.py @@ -528,7 +732,9 @@ mv optimization_engine/export_expressions.py optimization_engine/nx/ mv optimization_engine/import_expressions.py optimization_engine/nx/ mv optimization_engine/mesh_converter.py optimization_engine/nx/ -# 4. SURROGATES +# ============================================================================= +# STEP 2.4: SURROGATES (ML components) +# ============================================================================= mv optimization_engine/neural_surrogate.py optimization_engine/processors/surrogates/ mv optimization_engine/generic_surrogate.py optimization_engine/processors/surrogates/ mv optimization_engine/adaptive_surrogate.py optimization_engine/processors/surrogates/ @@ -538,7 +744,9 @@ mv optimization_engine/surrogate_tuner.py optimization_engine/processors/surroga mv optimization_engine/auto_trainer.py optimization_engine/processors/surrogates/ mv optimization_engine/training_data_exporter.py optimization_engine/processors/surrogates/ -# 5. STUDY MANAGEMENT +# ============================================================================= +# STEP 2.5: STUDY MANAGEMENT +# ============================================================================= mv optimization_engine/study_creator.py optimization_engine/study/creator.py mv optimization_engine/study_wizard.py optimization_engine/study/wizard.py mv optimization_engine/study_state.py optimization_engine/study/state.py @@ -547,14 +755,18 @@ mv optimization_engine/study_continuation.py optimization_engine/study/continuat mv optimization_engine/benchmarking_substudy.py optimization_engine/study/benchmarking.py mv optimization_engine/generate_history_from_trials.py optimization_engine/study/history_generator.py -# 6. REPORTING +# ============================================================================= +# STEP 2.6: REPORTING +# ============================================================================= mv optimization_engine/generate_report.py optimization_engine/reporting/report_generator.py mv optimization_engine/generate_report_markdown.py optimization_engine/reporting/markdown_report.py mv optimization_engine/comprehensive_results_analyzer.py optimization_engine/reporting/results_analyzer.py mv optimization_engine/visualizer.py optimization_engine/reporting/ mv optimization_engine/landscape_analyzer.py optimization_engine/reporting/ -# 7. CORE (depends on many things, do last) +# ============================================================================= +# STEP 2.7: CORE (depends on many things, do after others) +# ============================================================================= mv optimization_engine/runner.py optimization_engine/core/ mv optimization_engine/base_runner.py optimization_engine/core/ mv optimization_engine/runner_with_neural.py optimization_engine/core/ @@ -562,371 +774,907 @@ mv optimization_engine/intelligent_optimizer.py optimization_engine/core/ mv optimization_engine/method_selector.py optimization_engine/core/ mv optimization_engine/strategy_selector.py optimization_engine/core/ mv optimization_engine/strategy_portfolio.py optimization_engine/core/ +mv optimization_engine/gradient_optimizer.py optimization_engine/core/ -# 8. RESEARCH/FUTURE +# ============================================================================= +# STEP 2.8: FUTURE/RESEARCH (experimental) +# ============================================================================= mv optimization_engine/research_agent.py optimization_engine/future/ mv optimization_engine/pynastran_research_agent.py optimization_engine/future/ mv optimization_engine/targeted_research_planner.py optimization_engine/future/ mv optimization_engine/workflow_decomposer.py optimization_engine/future/ mv optimization_engine/step_classifier.py optimization_engine/future/ +# Check if these exist before moving: +test -f optimization_engine/llm_optimization_runner.py && mv optimization_engine/llm_optimization_runner.py optimization_engine/future/ +test -f optimization_engine/llm_workflow_analyzer.py && mv optimization_engine/llm_workflow_analyzer.py optimization_engine/future/ +test -f optimization_engine/extractor_orchestrator.py && mv optimization_engine/extractor_orchestrator.py optimization_engine/future/ +test -f optimization_engine/inline_code_generator.py && mv optimization_engine/inline_code_generator.py optimization_engine/future/ +test -f optimization_engine/hook_generator.py && mv optimization_engine/hook_generator.py optimization_engine/future/ -# 9. REMAINING MISC +# ============================================================================= +# STEP 2.9: MOVE TO EXISTING DIRECTORIES +# ============================================================================= mv optimization_engine/op2_extractor.py optimization_engine/extractors/ mv optimization_engine/extractor_library.py optimization_engine/extractors/ mv optimization_engine/simulation_validator.py optimization_engine/validators/ mv optimization_engine/adaptive_characterization.py optimization_engine/processors/ ``` -### Phase 3: Update Internal Imports (1-2 hours) +### Phase 3: Update Imports with Automation Script (2-3 hours) -**Use sed/grep for bulk updates:** - -```bash -# Example sed commands (run from project root) - -# NX imports -find . -name "*.py" -exec sed -i 's/from optimization_engine\.nx_solver/from optimization_engine.nx.solver/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.nx_updater/from optimization_engine.nx.updater/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.nx_session_manager/from optimization_engine.nx.session_manager/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.solve_simulation/from optimization_engine.nx.solve_simulation/g' {} + - -# Study imports -find . -name "*.py" -exec sed -i 's/from optimization_engine\.study_creator/from optimization_engine.study.creator/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.study_wizard/from optimization_engine.study.wizard/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.study_state/from optimization_engine.study.state/g' {} + - -# Config imports -find . -name "*.py" -exec sed -i 's/from optimization_engine\.config_manager/from optimization_engine.config.manager/g' {} + - -# Core imports -find . -name "*.py" -exec sed -i 's/from optimization_engine\.runner import/from optimization_engine.core.runner import/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.base_runner/from optimization_engine.core.base_runner/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.intelligent_optimizer/from optimization_engine.core.intelligent_optimizer/g' {} + - -# Surrogate imports -find . -name "*.py" -exec sed -i 's/from optimization_engine\.neural_surrogate/from optimization_engine.processors.surrogates.neural_surrogate/g' {} + -find . -name "*.py" -exec sed -i 's/from optimization_engine\.generic_surrogate/from optimization_engine.processors.surrogates.generic_surrogate/g' {} + - -# Logger -find . -name "*.py" -exec sed -i 's/from optimization_engine\.logger/from optimization_engine.utils.logger/g' {} + -``` - -**Handle edge cases manually:** -- `import optimization_engine.nx_solver` (without `from`) -- Dynamic imports using `importlib` -- String references in configs - -### Phase 4: Create __init__.py Files (30 min) - -**Example: `optimization_engine/core/__init__.py`** +Create and run a Python migration script: ```python -""" -Optimization Engine Core - -Main optimization runners and algorithm selection. -""" - -from .runner import OptimizationRunner -from .base_runner import BaseRunner -from .intelligent_optimizer import IntelligentOptimizer, IMSO -from .method_selector import MethodSelector -from .strategy_selector import StrategySelector -from .strategy_portfolio import StrategyPortfolio - -__all__ = [ - 'OptimizationRunner', - 'BaseRunner', - 'IntelligentOptimizer', - 'IMSO', - 'MethodSelector', - 'StrategySelector', - 'StrategyPortfolio', -] -``` - -**Create similar for each new directory.** - -### Phase 5: Add Backwards Compatibility (30 min) - -Update `optimization_engine/__init__.py`: - -```python -""" -Optimization Engine for Atomizer - -Reorganized structure (v2.0): -- core/ - Optimization runners -- processors/ - Data processing (surrogates, dynamic response) -- nx/ - NX/Nastran integration -- study/ - Study management -- reporting/ - Reports and analysis -- config/ - Configuration -- extractors/ - Physics extraction -- insights/ - Visualizations -- gnn/ - Graph neural networks -- hooks/ - NX hooks -""" - -# Re-export commonly used items at top level for convenience -from optimization_engine.core.runner import OptimizationRunner -from optimization_engine.core.intelligent_optimizer import IMSO -from optimization_engine.nx.solver import run_nx_simulation, NXSolver -from optimization_engine.study.creator import create_study -from optimization_engine.config.manager import ConfigManager - -# Backwards compatibility aliases (deprecated) -# These will be removed in a future version -import warnings as _warnings - -def _deprecated_import(old_name, new_location): - _warnings.warn( - f"Importing '{old_name}' directly from optimization_engine is deprecated. " - f"Use '{new_location}' instead.", - DeprecationWarning, - stacklevel=3 - ) - -# Lazy loading for backwards compatibility -def __getattr__(name): - # Map old names to new locations - _compat_map = { - 'nx_solver': ('optimization_engine.nx.solver', 'nx_solver'), - 'nx_updater': ('optimization_engine.nx.updater', 'nx_updater'), - 'study_creator': ('optimization_engine.study.creator', 'study_creator'), - 'config_manager': ('optimization_engine.config.manager', 'config_manager'), - 'runner': ('optimization_engine.core.runner', 'runner'), - 'neural_surrogate': ('optimization_engine.processors.surrogates.neural_surrogate', 'neural_surrogate'), - } - - if name in _compat_map: - module_path, attr = _compat_map[name] - _deprecated_import(name, module_path) - import importlib - module = importlib.import_module(module_path) - return module - - raise AttributeError(f"module 'optimization_engine' has no attribute '{name}'") -``` - -### Phase 6: Update Documentation (1 hour) - -**Use sed for bulk updates in markdown:** - -```bash -# Update protocol files -find docs/protocols -name "*.md" -exec sed -i 's/optimization_engine\/nx_solver/optimization_engine\/nx\/solver/g' {} + -find docs/protocols -name "*.md" -exec sed -i 's/optimization_engine\/study_creator/optimization_engine\/study\/creator/g' {} + -# ... etc for all mappings - -# Update .claude skills -find .claude -name "*.md" -exec sed -i 's/optimization_engine\.nx_solver/optimization_engine.nx.solver/g' {} + -# ... etc -``` - -**Manual review needed for:** -- Code examples in markdown -- Directory structure diagrams -- Command-line examples - -### Phase 7: Update JSON Configs (30 min) - -**feature_registry.json** - Use a Python script: - -```python -import json - -with open('optimization_engine/feature_registry.json', 'r') as f: - registry = json.load(f) - -# Define path mappings -path_map = { - 'optimization_engine/neural_surrogate.py': 'optimization_engine/processors/surrogates/neural_surrogate.py', - 'optimization_engine/nx_solver.py': 'optimization_engine/nx/solver.py', - # ... all other mappings -} - -def update_paths(obj): - if isinstance(obj, dict): - for key, value in obj.items(): - if key == 'file_path' and value in path_map: - obj[key] = path_map[value] - else: - update_paths(value) - elif isinstance(obj, list): - for item in obj: - update_paths(item) - -update_paths(registry) - -with open('optimization_engine/feature_registry.json', 'w') as f: - json.dump(registry, f, indent=2) -``` - -### Phase 8: Testing & Validation (1-2 hours) - -```bash -# 1. Run Python import tests -python -c "from optimization_engine.core.runner import OptimizationRunner" -python -c "from optimization_engine.nx.solver import run_nx_simulation" -python -c "from optimization_engine.study.creator import create_study" -python -c "from optimization_engine.processors.surrogates.neural_surrogate import NeuralSurrogate" - -# 2. Run backwards compatibility tests -python -c "from optimization_engine import nx_solver" # Should work with deprecation warning - -# 3. Run test suite -python -m pytest tests/ -v --tb=short - -# 4. Test a study end-to-end -cd studies/Simple_Bracket/bracket_displacement_maximizing -python run_optimization.py --trials 2 --dry-run - -# 5. Test dashboard -cd atomizer-dashboard -python backend/api/main.py # Should start without import errors -``` - -### Phase 9: Cleanup (15 min) - -```bash -# Remove backup after successful testing -rm -rf optimization_engine_backup - -# Remove any .pyc files that might cache old imports -find . -name "*.pyc" -delete -find . -name "__pycache__" -type d -exec rm -rf {} + - -# Commit -git add -A -git commit -m "refactor: Reorganize optimization_engine into modular structure - -- Move 50+ top-level files into logical subdirectories -- Create core/, processors/, nx/, study/, reporting/, config/ -- Add backwards compatibility aliases with deprecation warnings -- Update all imports across codebase (640+ changes) -- Update documentation and protocols - -BREAKING: Direct imports from optimization_engine.* are deprecated. -Use new paths like optimization_engine.core.runner instead. - -🤖 Generated with [Claude Code](https://claude.com/claude-code)" -``` - ---- - -## Part 6: Rollback Plan - -If migration fails: - -```bash -# Option 1: Git reset (if not committed) -git checkout -- . - -# Option 2: Restore backup -rm -rf optimization_engine -mv optimization_engine_backup optimization_engine - -# Option 3: Git revert (if committed) -git revert HEAD -``` - ---- - -## Part 7: Post-Migration Tasks - -### 7.1 Update LAC Knowledge Base - -Record the migration in LAC: - -```python -from knowledge_base.lac import get_lac -lac = get_lac() -lac.record_insight( - category="architecture", - context="optimization_engine reorganization", - insight="Migrated 50+ files into modular structure. New paths: " - "core/, processors/, nx/, study/, reporting/, config/. " - "Backwards compat aliases provided with deprecation warnings.", - confidence=1.0, - tags=["refactoring", "architecture", "breaking-change"] -) -``` - -### 7.2 Update CLAUDE.md - -Add section about new structure. - -### 7.3 Schedule Deprecation Removal - -After 2-4 weeks of stable operation: -- Remove backwards compatibility aliases -- Update remaining old imports -- Clean up __init__.py files - ---- - -## Part 8: Time Estimate Summary - -| Phase | Task | Time | -|-------|------|------| -| 0 | Pre-migration (backup, baseline) | 30 min | -| 1 | Create directory structure | 15 min | -| 2 | Move files | 30 min | -| 3 | Update internal imports | 1-2 hours | -| 4 | Create __init__.py files | 30 min | -| 5 | Add backwards compatibility | 30 min | -| 6 | Update documentation | 1 hour | -| 7 | Update JSON configs | 30 min | -| 8 | Testing & validation | 1-2 hours | -| 9 | Cleanup & commit | 15 min | -| **TOTAL** | | **6-8 hours** | - ---- - -## Part 9: Decision Points - -### Option A: Full Migration Now -- **Pros**: Clean structure, enables dynamic_response cleanly -- **Cons**: 6-8 hours work, risk of breakage -- **When**: When you have a dedicated half-day - -### Option B: Minimal Migration (processors/ only) -- **Pros**: Low risk, enables dynamic_response -- **Cons**: Leaves technical debt, inconsistent structure -- **When**: If you need dynamic_response urgently - -### Option C: Defer Migration -- **Pros**: Zero risk now -- **Cons**: Dynamic_response goes in awkward location -- **When**: If stability is critical - ---- - -## Appendix A: Automated Migration Script - -A Python script to automate most of the migration: - -```python -#!/usr/bin/env python +#!/usr/bin/env python3 """ optimization_engine Migration Script +===================================== +Automatically updates all imports across the codebase. -Run with: python migrate_optimization_engine.py --dry-run -Then: python migrate_optimization_engine.py --execute +Usage: + python migrate_imports.py --dry-run # Preview changes + python migrate_imports.py --execute # Apply changes """ import os import re -import shutil +import sys from pathlib import Path +from typing import Dict, List, Tuple -# ... full script would be ~200 lines -# Handles: directory creation, file moves, import updates, __init__ generation +# Import mappings (old -> new) +IMPORT_MAPPINGS = { + # Core + r'from optimization_engine\.runner\b': 'from optimization_engine.core.runner', + r'from optimization_engine\.base_runner\b': 'from optimization_engine.core.base_runner', + r'from optimization_engine\.runner_with_neural\b': 'from optimization_engine.core.runner_with_neural', + r'from optimization_engine\.intelligent_optimizer\b': 'from optimization_engine.core.intelligent_optimizer', + r'from optimization_engine\.method_selector\b': 'from optimization_engine.core.method_selector', + r'from optimization_engine\.strategy_selector\b': 'from optimization_engine.core.strategy_selector', + r'from optimization_engine\.strategy_portfolio\b': 'from optimization_engine.core.strategy_portfolio', + r'from optimization_engine\.gradient_optimizer\b': 'from optimization_engine.core.gradient_optimizer', + r'import optimization_engine\.runner\b': 'import optimization_engine.core.runner', + r'import optimization_engine\.intelligent_optimizer\b': 'import optimization_engine.core.intelligent_optimizer', + + # Surrogates + r'from optimization_engine\.neural_surrogate\b': 'from optimization_engine.processors.surrogates.neural_surrogate', + r'from optimization_engine\.generic_surrogate\b': 'from optimization_engine.processors.surrogates.generic_surrogate', + r'from optimization_engine\.adaptive_surrogate\b': 'from optimization_engine.processors.surrogates.adaptive_surrogate', + r'from optimization_engine\.simple_mlp_surrogate\b': 'from optimization_engine.processors.surrogates.simple_mlp_surrogate', + r'from optimization_engine\.active_learning_surrogate\b': 'from optimization_engine.processors.surrogates.active_learning_surrogate', + r'from optimization_engine\.surrogate_tuner\b': 'from optimization_engine.processors.surrogates.surrogate_tuner', + r'from optimization_engine\.auto_trainer\b': 'from optimization_engine.processors.surrogates.auto_trainer', + r'from optimization_engine\.training_data_exporter\b': 'from optimization_engine.processors.surrogates.training_data_exporter', + + # NX + r'from optimization_engine\.nx_solver\b': 'from optimization_engine.nx.solver', + r'from optimization_engine\.nx_updater\b': 'from optimization_engine.nx.updater', + r'from optimization_engine\.nx_session_manager\b': 'from optimization_engine.nx.session_manager', + r'from optimization_engine\.solve_simulation\b': 'from optimization_engine.nx.solve_simulation', + r'from optimization_engine\.solve_simulation_simple\b': 'from optimization_engine.nx.solve_simulation_simple', + r'from optimization_engine\.model_cleanup\b': 'from optimization_engine.nx.model_cleanup', + r'from optimization_engine\.export_expressions\b': 'from optimization_engine.nx.export_expressions', + r'from optimization_engine\.import_expressions\b': 'from optimization_engine.nx.import_expressions', + r'from optimization_engine\.mesh_converter\b': 'from optimization_engine.nx.mesh_converter', + r'import optimization_engine\.nx_solver\b': 'import optimization_engine.nx.solver', + + # Study + r'from optimization_engine\.study_creator\b': 'from optimization_engine.study.creator', + r'from optimization_engine\.study_wizard\b': 'from optimization_engine.study.wizard', + r'from optimization_engine\.study_state\b': 'from optimization_engine.study.state', + r'from optimization_engine\.study_reset\b': 'from optimization_engine.study.reset', + r'from optimization_engine\.study_continuation\b': 'from optimization_engine.study.continuation', + r'from optimization_engine\.benchmarking_substudy\b': 'from optimization_engine.study.benchmarking', + r'from optimization_engine\.generate_history_from_trials\b': 'from optimization_engine.study.history_generator', + + # Reporting + r'from optimization_engine\.generate_report\b': 'from optimization_engine.reporting.report_generator', + r'from optimization_engine\.generate_report_markdown\b': 'from optimization_engine.reporting.markdown_report', + r'from optimization_engine\.comprehensive_results_analyzer\b': 'from optimization_engine.reporting.results_analyzer', + r'from optimization_engine\.visualizer\b': 'from optimization_engine.reporting.visualizer', + r'from optimization_engine\.landscape_analyzer\b': 'from optimization_engine.reporting.landscape_analyzer', + + # Config + r'from optimization_engine\.config_manager\b': 'from optimization_engine.config.manager', + r'from optimization_engine\.optimization_config_builder\b': 'from optimization_engine.config.builder', + r'from optimization_engine\.optimization_setup_wizard\b': 'from optimization_engine.config.setup_wizard', + r'from optimization_engine\.capability_matcher\b': 'from optimization_engine.config.capability_matcher', + r'from optimization_engine\.template_loader\b': 'from optimization_engine.config.template_loader', + + # Utils + r'from optimization_engine\.logger\b': 'from optimization_engine.utils.logger', + r'from optimization_engine\.auto_doc\b': 'from optimization_engine.utils.auto_doc', + r'from optimization_engine\.realtime_tracking\b': 'from optimization_engine.utils.realtime_tracking', + r'from optimization_engine\.codebase_analyzer\b': 'from optimization_engine.utils.codebase_analyzer', + r'from optimization_engine\.pruning_logger\b': 'from optimization_engine.utils.pruning_logger', + + # Future + r'from optimization_engine\.research_agent\b': 'from optimization_engine.future.research_agent', + r'from optimization_engine\.pynastran_research_agent\b': 'from optimization_engine.future.pynastran_research_agent', + r'from optimization_engine\.targeted_research_planner\b': 'from optimization_engine.future.targeted_research_planner', + r'from optimization_engine\.workflow_decomposer\b': 'from optimization_engine.future.workflow_decomposer', + r'from optimization_engine\.step_classifier\b': 'from optimization_engine.future.step_classifier', + r'from optimization_engine\.llm_optimization_runner\b': 'from optimization_engine.future.llm_optimization_runner', + r'from optimization_engine\.llm_workflow_analyzer\b': 'from optimization_engine.future.llm_workflow_analyzer', + + # Extractors/Validators additions + r'from optimization_engine\.op2_extractor\b': 'from optimization_engine.extractors.op2_extractor', + r'from optimization_engine\.extractor_library\b': 'from optimization_engine.extractors.extractor_library', + r'from optimization_engine\.simulation_validator\b': 'from optimization_engine.validators.simulation_validator', + + # Processors + r'from optimization_engine\.adaptive_characterization\b': 'from optimization_engine.processors.adaptive_characterization', +} + +# Path mappings for documentation +PATH_MAPPINGS = { + 'optimization_engine/runner.py': 'optimization_engine/core/runner.py', + 'optimization_engine/base_runner.py': 'optimization_engine/core/base_runner.py', + 'optimization_engine/intelligent_optimizer.py': 'optimization_engine/core/intelligent_optimizer.py', + 'optimization_engine/nx_solver.py': 'optimization_engine/nx/solver.py', + 'optimization_engine/nx_updater.py': 'optimization_engine/nx/updater.py', + 'optimization_engine/study_creator.py': 'optimization_engine/study/creator.py', + 'optimization_engine/study_wizard.py': 'optimization_engine/study/wizard.py', + 'optimization_engine/neural_surrogate.py': 'optimization_engine/processors/surrogates/neural_surrogate.py', + 'optimization_engine/config_manager.py': 'optimization_engine/config/manager.py', + 'optimization_engine/logger.py': 'optimization_engine/utils/logger.py', + # Add all other path mappings... +} + +def find_files(root: Path, extensions: List[str]) -> List[Path]: + """Find all files with given extensions.""" + files = [] + for ext in extensions: + files.extend(root.rglob(f'*{ext}')) + return files + +def update_file(filepath: Path, mappings: Dict[str, str], dry_run: bool = True) -> Tuple[int, List[str]]: + """Update imports in a single file.""" + try: + content = filepath.read_text(encoding='utf-8', errors='ignore') + except Exception as e: + print(f" ERROR reading {filepath}: {e}") + return 0, [] + + changes = [] + new_content = content + + for pattern, replacement in mappings.items(): + matches = re.findall(pattern, content) + if matches: + new_content = re.sub(pattern, replacement, new_content) + changes.append(f" {pattern} → {replacement} ({len(matches)} occurrences)") + + if changes and not dry_run: + filepath.write_text(new_content, encoding='utf-8') + + return len(changes), changes + +def main(): + dry_run = '--dry-run' in sys.argv or '--execute' not in sys.argv + + if dry_run: + print("=" * 60) + print("DRY RUN MODE - No files will be modified") + print("=" * 60) + else: + print("=" * 60) + print("EXECUTE MODE - Files will be modified!") + print("=" * 60) + confirm = input("Are you sure? (yes/no): ") + if confirm.lower() != 'yes': + print("Aborted.") + return + + root = Path('.') + + # Find all Python files + py_files = find_files(root, ['.py']) + py_files = [f for f in py_files if 'optimization_engine_BACKUP' not in str(f)] + py_files = [f for f in py_files if '.venv' not in str(f)] + py_files = [f for f in py_files if 'node_modules' not in str(f)] + + print(f"\nFound {len(py_files)} Python files to check") + + total_changes = 0 + files_changed = 0 + + for filepath in sorted(py_files): + count, changes = update_file(filepath, IMPORT_MAPPINGS, dry_run) + if count > 0: + files_changed += 1 + total_changes += count + print(f"\n{filepath} ({count} changes):") + for change in changes: + print(change) + + # Find all Markdown files + md_files = find_files(root, ['.md']) + md_files = [f for f in md_files if 'optimization_engine_BACKUP' not in str(f)] + md_files = [f for f in md_files if 'node_modules' not in str(f)] + + print(f"\n\nFound {len(md_files)} Markdown files to check") + + for filepath in sorted(md_files): + count, changes = update_file(filepath, PATH_MAPPINGS, dry_run) + if count > 0: + files_changed += 1 + total_changes += count + print(f"\n{filepath} ({count} changes):") + for change in changes[:5]: # Limit output + print(change) + if len(changes) > 5: + print(f" ... and {len(changes) - 5} more") + + print("\n" + "=" * 60) + print(f"SUMMARY: {total_changes} changes in {files_changed} files") + print("=" * 60) + + if dry_run: + print("\nTo apply changes, run: python migrate_imports.py --execute") + +if __name__ == '__main__': + main() ``` -Would you like me to create this full automation script? +### Phase 4: Create __init__.py Files (1 hour) + +Create proper `__init__.py` for each new module. See separate section below. + +### Phase 5: Update feature_registry.json (30 minutes) + +Use a dedicated script to update all path references in the JSON file. + +### Phase 6: Update Documentation (2 hours) + +Run sed/grep commands for bulk markdown updates, then manual review. + +### Phase 7: Testing & Validation (2 hours) + +```bash +# 1. Test Python imports +python -c "from optimization_engine.core.runner import OptimizationRunner; print('core OK')" +python -c "from optimization_engine.nx.solver import NXSolver; print('nx OK')" +python -c "from optimization_engine.study.creator import StudyCreator; print('study OK')" +python -c "from optimization_engine.processors.surrogates.neural_surrogate import NeuralSurrogate; print('surrogates OK')" +python -c "from optimization_engine.config.manager import ConfigManager; print('config OK')" +python -c "from optimization_engine.utils.logger import get_logger; print('utils OK')" + +# 2. Test backwards compatibility +python -c "from optimization_engine import nx_solver; print('compat OK')" + +# 3. Run full test suite +python -m pytest tests/ -v --tb=short 2>&1 | tee post_migration_tests.log + +# 4. Compare test results +diff baseline_tests_*.log post_migration_tests.log + +# 5. Test a study +cd studies/bracket_displacement_maximizing +python run_optimization.py --trials 1 --dry-run + +# 6. Test dashboard +cd atomizer-dashboard/backend +python -c "from api.routes.optimization import *; print('dashboard OK')" +``` + +### Phase 8: Commit & Cleanup (30 minutes) + +```bash +# Remove backup after successful testing +rm -rf optimization_engine_BACKUP_* + +# Clear Python cache +find . -name "*.pyc" -delete +find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + +# Final commit +git add -A +git commit -m "refactor: Major reorganization of optimization_engine + +BREAKING CHANGE: Module structure reorganized for better maintainability. + +New structure: +- core/ - Optimization runners (runner, intelligent_optimizer, etc.) +- processors/ - Data processing (surrogates/, dynamic_response/) +- nx/ - NX/Nastran integration (solver, updater, etc.) +- study/ - Study management (creator, wizard, state, etc.) +- reporting/ - Reports and analysis +- config/ - Configuration management +- utils/ - Utilities (logger, etc.) +- future/ - Research/experimental code + +Migration: +- 500+ files updated +- ~1,400 import changes +- Backwards compatibility aliases provided with deprecation warnings +- All tests passing + +🤖 Generated with [Claude Code](https://claude.com/claude-code) + +Co-Authored-By: Claude " + +# Push +git push origin refactor/optimization-engine-v2 +``` --- -*This plan ensures a safe, reversible migration with clear validation steps.* +## Part 4: __init__.py Files Content + +### 4.1 optimization_engine/core/__init__.py + +```python +""" +Optimization Engine Core +======================== + +Main optimization runners and algorithm selection. + +Modules: +- runner: Main OptimizationRunner class +- base_runner: BaseRunner abstract class +- intelligent_optimizer: IMSO adaptive optimizer +- method_selector: Algorithm selection logic +- strategy_selector: Strategy portfolio management +""" + +from .runner import OptimizationRunner +from .base_runner import BaseRunner +from .runner_with_neural import NeuralOptimizationRunner +from .intelligent_optimizer import IntelligentOptimizer, IMSO +from .method_selector import MethodSelector, select_method +from .strategy_selector import StrategySelector +from .strategy_portfolio import StrategyPortfolio +from .gradient_optimizer import GradientOptimizer, LBFGSPolisher + +__all__ = [ + 'OptimizationRunner', + 'BaseRunner', + 'NeuralOptimizationRunner', + 'IntelligentOptimizer', + 'IMSO', + 'MethodSelector', + 'select_method', + 'StrategySelector', + 'StrategyPortfolio', + 'GradientOptimizer', + 'LBFGSPolisher', +] +``` + +### 4.2 optimization_engine/processors/__init__.py + +```python +""" +Optimization Processors +======================= + +Data processing algorithms and ML models. + +Submodules: +- surrogates/: Neural network surrogate models +- dynamic_response/: Dynamic response processing (random vib, sine sweep) +""" + +from . import surrogates + +__all__ = [ + 'surrogates', +] +``` + +### 4.3 optimization_engine/processors/surrogates/__init__.py + +```python +""" +Surrogate Models +================ + +Neural network and ML surrogate models for FEA acceleration. + +Classes: +- NeuralSurrogate: Main neural network surrogate +- GenericSurrogate: Flexible surrogate interface +- AdaptiveSurrogate: Self-improving surrogate +- SimpleMLP: Simple multi-layer perceptron +- ActiveLearningSurrogate: Active learning surrogate +- SurrogateTuner: Hyperparameter tuning +""" + +from .neural_surrogate import NeuralSurrogate, create_surrogate +from .generic_surrogate import GenericSurrogate +from .adaptive_surrogate import AdaptiveSurrogate +from .simple_mlp_surrogate import SimpleMLP, SimpleMLPSurrogate +from .active_learning_surrogate import ActiveLearningSurrogate +from .surrogate_tuner import SurrogateTuner, tune_surrogate +from .auto_trainer import AutoTrainer +from .training_data_exporter import TrainingDataExporter, export_training_data + +__all__ = [ + 'NeuralSurrogate', + 'create_surrogate', + 'GenericSurrogate', + 'AdaptiveSurrogate', + 'SimpleMLP', + 'SimpleMLPSurrogate', + 'ActiveLearningSurrogate', + 'SurrogateTuner', + 'tune_surrogate', + 'AutoTrainer', + 'TrainingDataExporter', + 'export_training_data', +] +``` + +### 4.4 optimization_engine/nx/__init__.py + +```python +""" +NX Integration +============== + +Siemens NX and Nastran integration modules. + +Modules: +- solver: NXSolver for running simulations +- updater: NXParameterUpdater for design updates +- session_manager: NX session lifecycle management +- solve_simulation: Low-level simulation execution +""" + +from .solver import NXSolver, run_nx_simulation +from .updater import NXParameterUpdater, update_parameters +from .session_manager import NXSessionManager, get_session +from .solve_simulation import solve_simulation, SolveSimulationError +from .model_cleanup import cleanup_model, ModelCleanup + +__all__ = [ + 'NXSolver', + 'run_nx_simulation', + 'NXParameterUpdater', + 'update_parameters', + 'NXSessionManager', + 'get_session', + 'solve_simulation', + 'SolveSimulationError', + 'cleanup_model', + 'ModelCleanup', +] +``` + +### 4.5 optimization_engine/study/__init__.py + +```python +""" +Study Management +================ + +Study creation, state management, and lifecycle. + +Modules: +- creator: Study creation from templates +- wizard: Interactive study setup wizard +- state: Study state tracking +- reset: Study reset functionality +- continuation: Resume interrupted studies +""" + +from .creator import StudyCreator, create_study +from .wizard import StudyWizard, run_wizard +from .state import StudyState, get_study_state +from .reset import reset_study, StudyReset +from .continuation import continue_study, StudyContinuation +from .benchmarking import BenchmarkingSubstudy +from .history_generator import generate_history + +__all__ = [ + 'StudyCreator', + 'create_study', + 'StudyWizard', + 'run_wizard', + 'StudyState', + 'get_study_state', + 'reset_study', + 'StudyReset', + 'continue_study', + 'StudyContinuation', + 'BenchmarkingSubstudy', + 'generate_history', +] +``` + +### 4.6 optimization_engine/reporting/__init__.py + +```python +""" +Reporting & Analysis +==================== + +Report generation and results analysis. + +Modules: +- report_generator: HTML/PDF report generation +- markdown_report: Markdown report format +- results_analyzer: Comprehensive results analysis +- visualizer: Plotting and visualization +- landscape_analyzer: Design space analysis +""" + +from .report_generator import generate_report, ReportGenerator +from .markdown_report import generate_markdown_report, MarkdownReporter +from .results_analyzer import ResultsAnalyzer, analyze_results +from .visualizer import Visualizer, plot_results +from .landscape_analyzer import LandscapeAnalyzer, analyze_landscape + +__all__ = [ + 'generate_report', + 'ReportGenerator', + 'generate_markdown_report', + 'MarkdownReporter', + 'ResultsAnalyzer', + 'analyze_results', + 'Visualizer', + 'plot_results', + 'LandscapeAnalyzer', + 'analyze_landscape', +] +``` + +### 4.7 optimization_engine/config/__init__.py + +```python +""" +Configuration Management +======================== + +Configuration loading, validation, and building. + +Modules: +- manager: ConfigManager for loading/saving configs +- builder: OptimizationConfigBuilder for creating configs +- setup_wizard: Interactive configuration setup +- capability_matcher: Match capabilities to requirements +""" + +from .manager import ConfigManager, load_config, save_config +from .builder import OptimizationConfigBuilder, build_config +from .setup_wizard import SetupWizard, run_setup +from .capability_matcher import CapabilityMatcher, match_capabilities +from .template_loader import TemplateLoader, load_template + +__all__ = [ + 'ConfigManager', + 'load_config', + 'save_config', + 'OptimizationConfigBuilder', + 'build_config', + 'SetupWizard', + 'run_setup', + 'CapabilityMatcher', + 'match_capabilities', + 'TemplateLoader', + 'load_template', +] +``` + +### 4.8 optimization_engine/__init__.py (Updated with Backwards Compat) + +```python +""" +Atomizer Optimization Engine +============================ + +Structural optimization framework for Siemens NX. + +New Module Structure (v2.0): +- core/ - Optimization runners +- processors/ - Data processing (surrogates, dynamic_response) +- nx/ - NX/Nastran integration +- study/ - Study management +- reporting/ - Reports and analysis +- config/ - Configuration +- extractors/ - Physics extraction (unchanged) +- insights/ - Visualizations (unchanged) +- gnn/ - Graph neural networks (unchanged) +- hooks/ - NX hooks (unchanged) +- utils/ - Utilities +- validators/ - Validation (unchanged) + +Quick Start: + from optimization_engine.core import OptimizationRunner + from optimization_engine.nx import NXSolver + from optimization_engine.extractors import extract_displacement +""" + +__version__ = '2.0.0' + +# Re-export commonly used items at top level +from optimization_engine.core.runner import OptimizationRunner +from optimization_engine.core.intelligent_optimizer import IMSO, IntelligentOptimizer +from optimization_engine.nx.solver import NXSolver, run_nx_simulation +from optimization_engine.study.creator import create_study +from optimization_engine.config.manager import ConfigManager + +# Submodule access +from optimization_engine import ( + core, + processors, + nx, + study, + reporting, + config, + extractors, + insights, + gnn, + hooks, + utils, + validators, +) + +__all__ = [ + # Version + '__version__', + # Top-level exports + 'OptimizationRunner', + 'IMSO', + 'IntelligentOptimizer', + 'NXSolver', + 'run_nx_simulation', + 'create_study', + 'ConfigManager', + # Submodules + 'core', + 'processors', + 'nx', + 'study', + 'reporting', + 'config', + 'extractors', + 'insights', + 'gnn', + 'hooks', + 'utils', + 'validators', +] + +# ============================================================================= +# BACKWARDS COMPATIBILITY LAYER +# ============================================================================= +# These aliases allow old imports to work with deprecation warnings. +# Will be removed in v3.0. + +import warnings as _warnings +import importlib as _importlib + +_DEPRECATED_MAPPINGS = { + # Core + 'runner': 'optimization_engine.core.runner', + 'base_runner': 'optimization_engine.core.base_runner', + 'intelligent_optimizer': 'optimization_engine.core.intelligent_optimizer', + 'method_selector': 'optimization_engine.core.method_selector', + 'strategy_selector': 'optimization_engine.core.strategy_selector', + 'strategy_portfolio': 'optimization_engine.core.strategy_portfolio', + 'gradient_optimizer': 'optimization_engine.core.gradient_optimizer', + + # Surrogates + 'neural_surrogate': 'optimization_engine.processors.surrogates.neural_surrogate', + 'generic_surrogate': 'optimization_engine.processors.surrogates.generic_surrogate', + 'adaptive_surrogate': 'optimization_engine.processors.surrogates.adaptive_surrogate', + 'simple_mlp_surrogate': 'optimization_engine.processors.surrogates.simple_mlp_surrogate', + 'active_learning_surrogate': 'optimization_engine.processors.surrogates.active_learning_surrogate', + 'surrogate_tuner': 'optimization_engine.processors.surrogates.surrogate_tuner', + + # NX + 'nx_solver': 'optimization_engine.nx.solver', + 'nx_updater': 'optimization_engine.nx.updater', + 'nx_session_manager': 'optimization_engine.nx.session_manager', + 'solve_simulation': 'optimization_engine.nx.solve_simulation', + 'model_cleanup': 'optimization_engine.nx.model_cleanup', + + # Study + 'study_creator': 'optimization_engine.study.creator', + 'study_wizard': 'optimization_engine.study.wizard', + 'study_state': 'optimization_engine.study.state', + 'study_reset': 'optimization_engine.study.reset', + 'study_continuation': 'optimization_engine.study.continuation', + + # Reporting + 'generate_report': 'optimization_engine.reporting.report_generator', + 'generate_report_markdown': 'optimization_engine.reporting.markdown_report', + 'visualizer': 'optimization_engine.reporting.visualizer', + + # Config + 'config_manager': 'optimization_engine.config.manager', + 'optimization_config_builder': 'optimization_engine.config.builder', + 'optimization_setup_wizard': 'optimization_engine.config.setup_wizard', + 'capability_matcher': 'optimization_engine.config.capability_matcher', + + # Utils + 'logger': 'optimization_engine.utils.logger', +} + + +def __getattr__(name): + """Provide backwards compatibility with deprecation warnings.""" + if name in _DEPRECATED_MAPPINGS: + new_module = _DEPRECATED_MAPPINGS[name] + _warnings.warn( + f"Importing '{name}' from optimization_engine is deprecated. " + f"Use '{new_module}' instead. " + f"This will be removed in v3.0.", + DeprecationWarning, + stacklevel=2 + ) + return _importlib.import_module(new_module) + + raise AttributeError(f"module 'optimization_engine' has no attribute '{name}'") +``` + +--- + +## Part 5: Post-Migration Checklist + +### Immediate (same day) + +- [ ] All imports updated in optimization_engine/ +- [ ] All imports updated in studies/ +- [ ] All imports updated in tests/ +- [ ] All imports updated in tools/ +- [ ] All imports updated in dashboard backend +- [ ] feature_registry.json updated +- [ ] All __init__.py files created +- [ ] Basic import tests pass +- [ ] Test suite passes (or known failures documented) + +### Short-term (within 1 week) + +- [ ] Update CLAUDE.md with new structure +- [ ] Update all protocol files (SYS_10-16, OP_01-06, EXT_01-04) +- [ ] Update all skill files in .claude/skills/ +- [ ] Update ATOMIZER_CONTEXT.md +- [ ] Update README.md +- [ ] Record migration in LAC knowledge base +- [ ] Run 2-3 studies end-to-end to verify + +### Medium-term (within 1 month) + +- [ ] Remove backwards compatibility after team confirms no issues +- [ ] Update any external documentation/wikis +- [ ] Update pyproject.toml metadata if needed +- [ ] Archive the migration plan document + +--- + +## Part 6: Rollback Procedure + +If critical issues are discovered: + +```bash +# Option 1: Git revert (preferred if committed) +git revert HEAD + +# Option 2: Reset to backup branch +git checkout backup/pre-reorganization +git checkout -b main-restored +git branch -D main +git branch -m main-restored main + +# Option 3: Restore from backup directory +rm -rf optimization_engine +mv optimization_engine_BACKUP_* optimization_engine +``` + +--- + +## Part 7: Time Summary + +| Phase | Task | Time | +|-------|------|------| +| 0 | Preparation (backup, baseline) | 30 min | +| 1 | Create directory structure | 15 min | +| 2 | Move files | 45 min | +| 3 | Update imports (with script) | 2-3 hours | +| 4 | Create __init__.py files | 1 hour | +| 5 | Update feature_registry.json | 30 min | +| 6 | Update documentation | 2 hours | +| 7 | Testing & validation | 2 hours | +| 8 | Commit & cleanup | 30 min | +| **TOTAL** | | **10-12 hours** | + +--- + +## Appendix: Quick Reference Card + +### New Import Patterns + +```python +# OLD # NEW +from optimization_engine.runner from optimization_engine.core.runner +from optimization_engine.nx_solver from optimization_engine.nx.solver +from optimization_engine.study_creator from optimization_engine.study.creator +from optimization_engine.neural_surrogate from optimization_engine.processors.surrogates.neural_surrogate +from optimization_engine.config_manager from optimization_engine.config.manager +from optimization_engine.logger from optimization_engine.utils.logger +``` + +### Directory Quick Reference + +``` +optimization_engine/ +├── core/ → runner, intelligent_optimizer, method_selector +├── processors/ +│ └── surrogates/ → neural_surrogate, generic_surrogate, etc. +├── nx/ → solver (was nx_solver), updater, session_manager +├── study/ → creator (was study_creator), wizard, state +├── reporting/ → report_generator, visualizer +├── config/ → manager (was config_manager), builder +├── utils/ → logger, auto_doc +├── future/ → research_agent, workflow_decomposer +├── extractors/ → (unchanged) +├── insights/ → (unchanged) +├── gnn/ → (unchanged) +├── hooks/ → (unchanged) +└── validators/ → (unchanged) +``` + +--- + +## Part 8: Handoff to Context Engineering Plan + +After this migration is complete, proceed with `ATOMIZER_CONTEXT_ENGINEERING_PLAN.md`. + +### What Context Engineering Will Add + +``` +optimization_engine/ +├── context/ ← NEW (added by Context Engineering) +│ ├── __init__.py +│ ├── playbook.py # AtomizerPlaybook - structured knowledge storage +│ ├── reflector.py # Analyzes optimization outcomes +│ ├── session_state.py # Context isolation for LLM sessions +│ ├── compaction.py # Context compaction for long sessions +│ ├── feedback_loop.py # Learning from execution outcomes +│ └── cache_monitor.py # KV-cache efficiency tracking +├── core/ ← Already created by Migration +│ └── runner.py # Will be modified to integrate context +└── ... +``` + +### Context Engineering Integration Points + +After migration, Context Engineering will modify: + +1. **`optimization_engine/core/runner.py`** - Add playbook and reflector integration +2. **`.claude/skills/00_BOOTSTRAP.md`** - Update to v2 with playbook loading +3. **`.claude/skills/02_CONTEXT_LOADER.md`** - Add playbook integration +4. **`atomizer-dashboard/backend/api/routes/`** - Add context API routes + +### Pre-Migration Checklist for Context Engineering Compatibility + +Before starting Context Engineering, verify: + +- [ ] Migration completed successfully +- [ ] `optimization_engine/core/runner.py` exists at new location +- [ ] All imports in codebase updated to new paths +- [ ] Test suite passes +- [ ] No deprecation warnings in core workflows + +### Context Engineering Will Use These New Paths + +```python +# Context Engineering code will import from new locations: +from optimization_engine.core.runner import OptimizationRunner +from optimization_engine.core.intelligent_optimizer import IMSO +from optimization_engine.utils.logger import get_logger +from optimization_engine.nx.solver import NXSolver + +# And add new context module: +from optimization_engine.context.playbook import AtomizerPlaybook +from optimization_engine.context.reflector import AtomizerReflector +``` + +--- + +*This plan is DEFINITIVE and covers ALL 500+ files that need updating.* +*After completion, proceed to ATOMIZER_CONTEXT_ENGINEERING_PLAN.md* diff --git a/.claude/skills/modules/study-disk-optimization.md b/.claude/skills/modules/study-disk-optimization.md new file mode 100644 index 00000000..86dc9088 --- /dev/null +++ b/.claude/skills/modules/study-disk-optimization.md @@ -0,0 +1,464 @@ +# Study Disk Optimization Module + +## Atomizer Disk Space Management System + +**Version:** 1.0 +**Created:** 2025-12-29 +**Status:** PRODUCTION READY +**Impact:** Reduced M1_Mirror from 194 GB → 114 GB (80 GB freed, 41% reduction) + +--- + +## Executive Summary + +FEA optimization studies consume massive disk space due to per-trial file copying. This module provides: + +1. **Local Cleanup** - Remove regenerable files from completed studies (50%+ savings) +2. **Remote Archival** - Archive to dalidou server (14TB available) +3. **On-Demand Restore** - Pull archived studies when needed + +### Key Insight + +Each trial folder contains ~150 MB, but only **~70 MB is essential** (OP2 results + metadata). The rest are copies of master files that can be regenerated. + +--- + +## Part 1: File Classification + +### Essential Files (KEEP) + +| Extension | Purpose | Typical Size | +|-----------|---------|--------------| +| `.op2` | Nastran binary results | 68 MB | +| `.json` | Parameters, results, metadata | <1 MB | +| `.npz` | Pre-computed Zernike coefficients | <1 MB | +| `.html` | Generated reports | <1 MB | +| `.png` | Visualization images | <1 MB | +| `.csv` | Exported data tables | <1 MB | + +### Deletable Files (REGENERABLE) + +| Extension | Purpose | Why Deletable | +|-----------|---------|---------------| +| `.prt` | NX part files | Copy of master in `1_setup/` | +| `.fem` | FEM mesh files | Copy of master | +| `.sim` | Simulation files | Copy of master | +| `.afm` | Assembly FEM | Regenerable | +| `.dat` | Solver input deck | Regenerable from params | +| `.f04` | Nastran output log | Diagnostic only | +| `.f06` | Nastran printed output | Diagnostic only | +| `.log` | Generic logs | Diagnostic only | +| `.diag` | Diagnostic files | Diagnostic only | +| `.txt` | Temp text files | Intermediate data | +| `.exp` | Expression files | Regenerable | +| `.bak` | Backup files | Not needed | + +### Protected Folders (NEVER TOUCH) + +| Folder | Reason | +|--------|--------| +| `1_setup/` | Master model files (source of truth) | +| `3_results/` | Final database, reports, best designs | +| `best_design_archive/` | Archived optimal configurations | + +--- + +## Part 2: Disk Usage Analysis + +### M1_Mirror Project Baseline (Dec 2025) + +``` +Total: 194 GB across 28 studies, 2000+ trials + +By File Type: + .op2 94 GB (48.5%) - Nastran results [ESSENTIAL] + .prt 41 GB (21.4%) - NX parts [DELETABLE] + .fem 22 GB (11.5%) - FEM mesh [DELETABLE] + .dat 22 GB (11.3%) - Solver input [DELETABLE] + .sim 9 GB (4.5%) - Simulation [DELETABLE] + .afm 5 GB (2.5%) - Assembly FEM [DELETABLE] + Other <1 GB (<1%) - Logs, configs [MIXED] + +By Folder: + 2_iterations/ 168 GB (87%) - Per-trial data + 3_results/ 22 GB (11%) - Final results + 1_setup/ 4 GB (2%) - Master models +``` + +### Per-Trial Breakdown (Typical V11+ Structure) + +``` +iter1/ + assy_m1_assyfem1_sim1-solution_1.op2 68.15 MB [KEEP] + M1_Blank.prt 29.94 MB [DELETE] + assy_m1_assyfem1_sim1-solution_1.dat 15.86 MB [DELETE] + M1_Blank_fem1.fem 14.07 MB [DELETE] + ASSY_M1_assyfem1_sim1.sim 7.47 MB [DELETE] + M1_Blank_fem1_i.prt 5.20 MB [DELETE] + ASSY_M1_assyfem1.afm 4.13 MB [DELETE] + M1_Vertical_Support_Skeleton_fem1.fem 3.76 MB [DELETE] + ... (logs, temps) <1.00 MB [DELETE] + _temp_part_properties.json 0.00 MB [KEEP] + ------------------------------------------------------- + TOTAL: 149.67 MB + Essential only: 68.15 MB + Savings: 54.5% +``` + +--- + +## Part 3: Implementation + +### Core Utility + +**Location:** `optimization_engine/utils/study_archiver.py` + +```python +from optimization_engine.utils.study_archiver import ( + analyze_study, # Get disk usage analysis + cleanup_study, # Remove deletable files + archive_to_remote, # Archive to dalidou + restore_from_remote, # Restore from dalidou + list_remote_archives, # List server archives +) +``` + +### Command Line Interface + +**Batch Script:** `tools/archive_study.bat` + +```bash +# Analyze disk usage +archive_study.bat analyze studies\M1_Mirror +archive_study.bat analyze studies\M1_Mirror\m1_mirror_V12 + +# Cleanup completed study (dry run by default) +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute + +# Archive to remote server +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute --tailscale + +# List remote archives +archive_study.bat list +archive_study.bat list --tailscale + +# Restore from remote +archive_study.bat restore m1_mirror_V12 +archive_study.bat restore m1_mirror_V12 --tailscale +``` + +### Python API + +```python +from pathlib import Path +from optimization_engine.utils.study_archiver import ( + analyze_study, + cleanup_study, + archive_to_remote, +) + +# Analyze +study_path = Path("studies/M1_Mirror/m1_mirror_V12") +analysis = analyze_study(study_path) +print(f"Total: {analysis['total_size_bytes']/1e9:.2f} GB") +print(f"Essential: {analysis['essential_size']/1e9:.2f} GB") +print(f"Deletable: {analysis['deletable_size']/1e9:.2f} GB") + +# Cleanup (dry_run=False to execute) +deleted, freed = cleanup_study(study_path, dry_run=False) +print(f"Freed {freed/1e9:.2f} GB") + +# Archive to server +success = archive_to_remote(study_path, use_tailscale=False, dry_run=False) +``` + +--- + +## Part 4: Remote Server Configuration + +### dalidou Server Specs + +| Property | Value | +|----------|-------| +| Hostname | dalidou | +| Local IP | 192.168.86.50 | +| Tailscale IP | 100.80.199.40 | +| SSH User | papa | +| Archive Path | /srv/storage/atomizer-archive/ | +| Available Storage | 3.6 TB (SSD) + 12.7 TB (HDD) | + +### First-Time Setup + +```bash +# 1. SSH into server and create archive directory +ssh papa@192.168.86.50 +mkdir -p /srv/storage/atomizer-archive + +# 2. Set up passwordless SSH (on Windows) +ssh-keygen -t ed25519 # If you don't have a key +ssh-copy-id papa@192.168.86.50 + +# 3. Test connection +ssh papa@192.168.86.50 "echo 'Connection OK'" +``` + +### Archive Structure on Server + +``` +/srv/storage/atomizer-archive/ +├── m1_mirror_V11_20251229.tar.gz # Compressed study archive +├── m1_mirror_V12_20251229.tar.gz +├── m1_mirror_flat_back_V3_20251229.tar.gz +└── manifest.json # Index of all archives +``` + +--- + +## Part 5: Recommended Workflows + +### During Active Optimization + +**Keep all files** - You may need to: +- Re-run specific failed trials +- Debug mesh issues +- Analyze intermediate results + +### After Study Completion + +1. **Generate final report** (STUDY_REPORT.md) +2. **Archive best design** to `3_results/best_design_archive/` +3. **Run cleanup:** + ```bash + archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute + ``` +4. **Verify results still accessible:** + - Database queries work + - Best design files intact + - OP2 files for Zernike extraction present + +### For Long-Term Storage + +1. **After cleanup**, archive to server: + ```bash + archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute + ``` +2. **Optionally delete local** study folder +3. **Keep only** `3_results/best_design_archive/` locally if needed + +### When Revisiting Old Study + +1. **Check if archived:** + ```bash + archive_study.bat list + ``` +2. **Restore:** + ```bash + archive_study.bat restore m1_mirror_V12 + ``` +3. **If re-running trials needed**, master files in `1_setup/` allow full regeneration + +--- + +## Part 6: Disk Space Targets + +### Per-Project Guidelines + +| Stage | Expected Size | Notes | +|-------|---------------|-------| +| Active (full) | 100% | All files present | +| Completed (cleaned) | ~50% | Deletables removed | +| Archived (minimal) | ~3% | Best design only locally | + +### M1_Mirror Specific + +| Stage | Size | Notes | +|-------|------|-------| +| Full | 194 GB | 28 studies, 2000+ trials | +| After cleanup | 114 GB | OP2 + metadata only | +| Minimal local | 5-10 GB | Best designs + database | +| Server archive | ~50 GB | Compressed | + +--- + +## Part 7: Safety Features + +### Built-in Protections + +1. **Dry run by default** - Must explicitly add `--execute` +2. **Master files untouched** - `1_setup/` is never modified +3. **Results preserved** - `3_results/` is never touched +4. **Essential files preserved** - OP2, JSON, NPZ always kept +5. **Archive verification** - rsync checks integrity + +### What Cannot Be Recovered After Cleanup + +| File Type | Recovery Method | +|-----------|-----------------| +| `.prt` | Copy from `1_setup/` + update params | +| `.fem` | Regenerate from `.prt` in NX | +| `.sim` | Recreate simulation setup | +| `.dat` | Regenerate from params.json + model | +| `.f04/.f06` | Re-run solver (if needed) | + +**Note:** With `1_setup/` master files and `params.json`, ANY trial can be fully reconstructed. The only irreplaceable data is the OP2 results (which we keep). + +--- + +## Part 8: Troubleshooting + +### SSH Connection Failed + +```bash +# Test connectivity +ping 192.168.86.50 + +# Test SSH +ssh papa@192.168.86.50 "echo connected" + +# If on different network, use Tailscale +ssh papa@100.80.199.40 "echo connected" +``` + +### Archive Upload Slow + +Large studies (50+ GB) take time. Options: +- Run overnight +- Use wired LAN connection +- Pre-cleanup to reduce size + +### Out of Disk Space During Archive + +Archive is created locally first. Need ~1.5x study size free: +- 20 GB study = ~30 GB temp space required + +### Cleanup Removed Wrong Files + +If accidentally executed without dry run: +- OP2 files preserved (can still extract results) +- Master files in `1_setup/` intact +- Regenerate other files by re-running trial + +--- + +## Part 9: Integration with Atomizer + +### Protocol Reference + +**Related Protocol:** `docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md` + +### Claude Commands + +When user says: +- "analyze disk usage" → Run `analyze_study()` +- "clean up study" → Run `cleanup_study()` with confirmation +- "archive to server" → Run `archive_to_remote()` +- "restore study" → Run `restore_from_remote()` + +### Automatic Suggestions + +After optimization completion, suggest: +``` +Optimization complete! The study is using X GB. +Would you like me to clean up regenerable files to save Y GB? +(This keeps all results but removes intermediate model copies) +``` + +--- + +## Part 10: File Inventory + +### Files Created + +| File | Purpose | +|------|---------| +| `optimization_engine/utils/study_archiver.py` | Core utility module | +| `tools/archive_study.bat` | Windows batch script | +| `docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md` | Full protocol | +| `.claude/skills/modules/study-disk-optimization.md` | This document | + +### Dependencies + +- Python 3.8+ +- rsync (for remote operations, usually pre-installed) +- SSH client (for remote operations) +- Tailscale (optional, for remote access outside LAN) + +--- + +## Appendix A: Cleanup Results Log (Dec 2025) + +### Initial Cleanup Run + +| Study | Before | After | Freed | Files Deleted | +|-------|--------|-------|-------|---------------| +| m1_mirror_cost_reduction_V11 | 32.24 GB | 15.94 GB | 16.30 GB | 3,403 | +| m1_mirror_cost_reduction_flat_back_V3 | 52.50 GB | 26.87 GB | 25.63 GB | 5,084 | +| m1_mirror_cost_reduction_flat_back_V6 | 33.71 GB | 16.64 GB | 17.08 GB | 3,391 | +| m1_mirror_cost_reduction_V12 | 22.68 GB | 10.60 GB | 12.08 GB | 2,508 | +| m1_mirror_cost_reduction_flat_back_V1 | 8.76 GB | 4.54 GB | 4.22 GB | 813 | +| m1_mirror_cost_reduction_flat_back_V5 | 8.01 GB | 4.09 GB | 3.92 GB | 765 | +| m1_mirror_cost_reduction | 3.58 GB | 3.08 GB | 0.50 GB | 267 | +| **TOTAL** | **161.48 GB** | **81.76 GB** | **79.73 GB** | **16,231** | + +### Project-Wide Summary + +``` +Before cleanup: 193.75 GB +After cleanup: 114.03 GB +Total freed: 79.72 GB (41% reduction) +``` + +--- + +## Appendix B: Quick Reference Card + +### Commands + +```bash +# Analyze +archive_study.bat analyze + +# Cleanup (always dry-run first!) +archive_study.bat cleanup # Dry run +archive_study.bat cleanup --execute # Execute + +# Archive +archive_study.bat archive --execute +archive_study.bat archive --execute --tailscale + +# Remote +archive_study.bat list +archive_study.bat restore +``` + +### Python + +```python +from optimization_engine.utils.study_archiver import * + +# Quick analysis +analysis = analyze_study(Path("studies/M1_Mirror")) +print(f"Deletable: {analysis['deletable_size']/1e9:.2f} GB") + +# Cleanup +cleanup_study(Path("studies/M1_Mirror/m1_mirror_V12"), dry_run=False) +``` + +### Server Access + +```bash +# Local +ssh papa@192.168.86.50 + +# Remote (Tailscale) +ssh papa@100.80.199.40 + +# Archive location +/srv/storage/atomizer-archive/ +``` + +--- + +*This module enables efficient disk space management for large-scale FEA optimization studies.* \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 3d5d084a..634fe9b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,6 +90,7 @@ The Protocol Operating System (POS) provides layered documentation: | Analyze results | OP_04 | `docs/protocols/operations/OP_04_ANALYZE_RESULTS.md` | | Export neural data | OP_05 | `docs/protocols/operations/OP_05_EXPORT_TRAINING_DATA.md` | | Debug issues | OP_06 | `docs/protocols/operations/OP_06_TROUBLESHOOT.md` | +| **Free disk space** | OP_07 | `docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md` | ## System Protocols (Technical Specs) @@ -135,14 +136,15 @@ C:\Users\antoi\anaconda3\envs\atomizer\python.exe your_script.py Atomizer/ ├── .claude/skills/ # LLM skills (Bootstrap + Core + Modules) ├── docs/protocols/ # Protocol Operating System -│ ├── operations/ # OP_01 - OP_06 +│ ├── operations/ # OP_01 - OP_07 │ ├── system/ # SYS_10 - SYS_15 │ └── extensions/ # EXT_01 - EXT_04 ├── optimization_engine/ # Core Python modules │ ├── extractors/ # Physics extraction library │ ├── gnn/ # GNN surrogate module (Zernike) -│ └── utils/ # Utilities (dashboard_db, trial_manager) +│ └── utils/ # Utilities (dashboard_db, trial_manager, study_archiver) ├── studies/ # User studies +├── tools/ # CLI tools (archive_study.bat, zernike_html_generator.py) ├── archive/ # Deprecated code (for reference) └── atomizer-dashboard/ # React dashboard ``` diff --git a/docs/TODO_NXOPEN_MCP_SETUP.md b/docs/TODO_NXOPEN_MCP_SETUP.md new file mode 100644 index 00000000..10bb899a --- /dev/null +++ b/docs/TODO_NXOPEN_MCP_SETUP.md @@ -0,0 +1,132 @@ +# NXOpen Documentation MCP Server - Setup TODO + +**Created:** 2025-12-29 +**Status:** PENDING - Waiting for manual configuration + +--- + +## Current State + +The NXOpen documentation MCP server exists on **dalidou** (192.168.86.50) but is not accessible from this Windows machine due to hostname resolution issues. + +### What's Working +- ✅ Dalidou server is online and reachable at `192.168.86.50` +- ✅ Port 5000 (Documentation Proxy) is responding +- ✅ Port 3000 (Gitea) is responding +- ✅ MCP server code exists at `/srv/claude-assistant/` on dalidou + +### What's NOT Working +- ❌ `dalidou.local` hostname doesn't resolve (mDNS not configured on this machine) +- ❌ MCP tools not integrated with Claude Code + +--- + +## Steps to Complete + +### Step 1: Fix Hostname Resolution (Manual - requires Admin) + +**Option A: Run the script as Administrator** +```powershell +# Open PowerShell as Administrator, then: +C:\Users\antoi\Atomizer\add_dalidou_host.ps1 +``` + +**Option B: Manually edit hosts file** +1. Open Notepad as Administrator +2. Open `C:\Windows\System32\drivers\etc\hosts` +3. Add this line at the end: + ``` + 192.168.86.50 dalidou.local dalidou + ``` +4. Save the file + +**Verify:** +```powershell +ping dalidou.local +``` + +### Step 2: Verify MCP Server is Running on Dalidou + +SSH into dalidou and check: +```bash +ssh root@dalidou + +# Check documentation proxy +systemctl status siemensdocumentationproxyserver + +# Check MCP server (if it's a service) +# Or check what's running on port 5000 +ss -tlnp | grep 5000 +``` + +### Step 3: Configure Claude Code MCP Integration + +The MCP server on dalidou uses **stdio-based MCP protocol**, not HTTP. To connect from Claude Code, you'll need one of: + +**Option A: SSH-based MCP (if supported)** +Configure in `.claude/settings.json` or MCP config to connect via SSH tunnel. + +**Option B: Local Proxy** +Run a local MCP proxy that connects to dalidou's MCP server. + +**Option C: HTTP Wrapper** +The current port 5000 service may already expose HTTP endpoints - need to verify once hostname is fixed. + +--- + +## Server Documentation Reference + +Full documentation is in the SERVtomaste repo: +- **URL:** http://192.168.86.50:3000/Antoine/SERVtomaste +- **File:** `docs/SIEMENS-DOCS-SERVER.md` + +### Key Server Paths (on dalidou) +``` +/srv/siemens-docs/proxy/ # Documentation Proxy (port 5000) +/srv/claude-assistant/ # MCP Server +/srv/claude-assistant/mcp-server/ # MCP server code +/srv/claude-assistant/tools/ # Tool implementations + ├── siemens-auth.js # Puppeteer authentication + ├── siemens-docs.js # Documentation fetching + └── ... +/srv/claude-assistant/vault/ # Credentials (secured) +``` + +### Available MCP Tools (once connected) +| Tool | Description | +|------|-------------| +| `siemens_docs_search` | Search NX Open, Simcenter docs | +| `siemens_docs_fetch` | Fetch specific documentation page | +| `siemens_auth_status` | Check if auth session is active | +| `siemens_login` | Re-login if session expired | +| `siemens_docs_list` | List documentation categories | + +--- + +## Files Created During Investigation + +- `C:\Users\antoi\Atomizer\add_dalidou_host.ps1` - Script to add hosts entry (run as Admin) +- `C:\Users\antoi\Atomizer\test_mcp.py` - Test script for probing MCP server (can be deleted) + +--- + +## Related Documentation + +- `.claude/skills/modules/nx-docs-lookup.md` - How to use MCP tools once configured +- `docs/08_ARCHIVE/historical/NXOPEN_DOCUMENTATION_INTEGRATION_STRATEGY.md` - Full strategy doc +- `docs/05_API_REFERENCE/NXOPEN_RESOURCES.md` - Alternative NXOpen resources + +--- + +## Workaround Until Fixed + +Without the MCP server, you can still look up NXOpen documentation by: + +1. **Using web search** - I can search for NXOpen API documentation online +2. **Using local stub files** - Python stubs at `C:\Program Files\Siemens\NX2412\UGOPEN\pythonStubs\` +3. **Using existing extractors** - Check `optimization_engine/extractors/` for patterns +4. **Recording NX journals** - Record operations in NX to learn the API calls + +--- + +*To continue setup, run the hosts file fix and let me know when ready.* diff --git a/docs/plans/ATOMIZER_CONTEXT_ENGINEERING_PLAN.md b/docs/plans/ATOMIZER_CONTEXT_ENGINEERING_PLAN.md new file mode 100644 index 00000000..02b33c4b --- /dev/null +++ b/docs/plans/ATOMIZER_CONTEXT_ENGINEERING_PLAN.md @@ -0,0 +1,1786 @@ +# Atomizer Context Engineering Implementation Plan + +## Claude Code Enhancement Strategy Using State-of-the-Art Context Engineering + +**Version**: 1.1 +**Date**: December 2025 +**Updated**: December 28, 2025 +**Author**: Antoine (with Claude) +**Purpose**: Transform Atomizer's LLM integration using cutting-edge context engineering patterns +**Prerequisite**: OPTIMIZATION_ENGINE_MIGRATION_PLAN.md (must complete BEFORE this plan) + +--- + +## Prerequisite: Complete Migration First + +**IMPORTANT**: This plan assumes the optimization_engine reorganization has been completed. + +Before starting Context Engineering: +1. Complete all phases in `.claude/skills/modules/OPTIMIZATION_ENGINE_MIGRATION_PLAN.md` +2. Verify `optimization_engine/core/runner.py` exists (not `optimization_engine/runner.py`) +3. All imports use new paths (e.g., `from optimization_engine.core.runner import ...`) +4. Test suite passes with new structure + +If migration is NOT complete, go do it first. Context Engineering builds on top of the reorganized structure. + +--- + +## Executive Summary + +This plan transforms Atomizer from a traditional LLM-assisted tool into a **self-improving, context-aware optimization platform** by implementing state-of-the-art context engineering techniques. The core innovation is treating the Learning Atomizer Core (LAC) as an **evolving playbook** that accumulates institutional knowledge through structured generation, reflection, and curation cycles. + +**Expected Outcomes**: +- 10-15% improvement in optimization task success rates +- 80%+ reduction in repeated mistakes across sessions +- Dramatic cost reduction through KV-cache optimization +- True institutional memory that compounds over time + +--- + +## Part 1: Architecture Mapping + +### Current Atomizer Architecture → ACE Framework Alignment + +| Atomizer Component | ACE Role | Enhancement | +|-------------------|----------|-------------| +| Optimization Runner | **Generator** | Produces optimization trajectories with success/failure signals | +| Post-run Analysis | **Reflector** | Extracts insights from optimization outcomes | +| Learning Atomizer Core (LAC) | **Curator** | Integrates insights into persistent playbook | +| Protocol Operating System (POS) | **Context Loader** | Selects relevant context per task type | +| Claude Code Sessions | **Agent** | Executes tasks using curated context | + +### New Component: AtomizerPlaybook + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ AtomizerPlaybook System │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Generator │──▶│ Reflector │──▶│ Curator │ │ +│ │(Opt Runs) │ │(Analysis) │ │(LAC Update) │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ Structured Playbook Store │ │ +│ ├──────────────────────────────────────────────────────┤ │ +│ │ [str-00001] helpful=8 harmful=0 :: │ │ +│ │ "For thin-walled structures, start with shell │ │ +│ │ elements before trying solid mesh" │ │ +│ │ │ │ +│ │ [cal-00002] helpful=12 harmful=1 :: │ │ +│ │ "Safety factor = yield_stress / max_von_mises" │ │ +│ │ │ │ +│ │ [mis-00003] helpful=0 harmful=6 :: │ │ +│ │ "Never set convergence < 1e-8 for SOL 106" │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Part 2: Implementation Phases + +### Phase 1: Structured Playbook System (Week 1-2) + +**Goal**: Convert LAC from unstructured memory to ACE-style itemized playbook + +#### 1.1 Create Playbook Data Structure + +**File**: `optimization_engine/context/playbook.py` + +```python +from dataclasses import dataclass, field +from typing import List, Dict, Optional +from enum import Enum +import json +from pathlib import Path +from datetime import datetime +import hashlib + +class InsightCategory(Enum): + STRATEGY = "str" # Optimization strategies + CALCULATION = "cal" # Formulas and calculations + MISTAKE = "mis" # Common mistakes to avoid + TOOL = "tool" # Tool usage patterns + DOMAIN = "dom" # Domain-specific knowledge (FEA, NX) + WORKFLOW = "wf" # Workflow patterns + +@dataclass +class PlaybookItem: + """Single insight in the playbook with helpful/harmful tracking.""" + id: str + category: InsightCategory + content: str + helpful_count: int = 0 + harmful_count: int = 0 + created_at: str = field(default_factory=lambda: datetime.now().isoformat()) + last_used: Optional[str] = None + source_trials: List[int] = field(default_factory=list) + tags: List[str] = field(default_factory=list) + + @property + def net_score(self) -> int: + return self.helpful_count - self.harmful_count + + @property + def confidence(self) -> float: + total = self.helpful_count + self.harmful_count + if total == 0: + return 0.5 + return self.helpful_count / total + + def to_context_string(self) -> str: + """Format for injection into LLM context.""" + return f"[{self.id}] helpful={self.helpful_count} harmful={self.harmful_count} :: {self.content}" + +@dataclass +class AtomizerPlaybook: + """ + Evolving playbook that accumulates optimization knowledge. + + Based on ACE framework principles: + - Incremental delta updates (never rewrite wholesale) + - Helpful/harmful tracking for each insight + - Semantic deduplication + - Category-based organization + """ + items: Dict[str, PlaybookItem] = field(default_factory=dict) + version: int = 1 + last_updated: str = field(default_factory=lambda: datetime.now().isoformat()) + + def _generate_id(self, category: InsightCategory) -> str: + """Generate unique ID for new item.""" + existing = [k for k in self.items.keys() if k.startswith(category.value)] + next_num = len(existing) + 1 + return f"{category.value}-{next_num:05d}" + + def add_insight( + self, + category: InsightCategory, + content: str, + source_trial: Optional[int] = None, + tags: Optional[List[str]] = None + ) -> PlaybookItem: + """ + Add new insight with delta update (ACE principle). + + Checks for semantic duplicates before adding. + """ + # Check for near-duplicates (simple implementation) + content_hash = hashlib.md5(content.lower().encode()).hexdigest()[:8] + + for item in self.items.values(): + existing_hash = hashlib.md5(item.content.lower().encode()).hexdigest()[:8] + if content_hash == existing_hash: + # Update existing instead of adding duplicate + item.helpful_count += 1 + if source_trial: + item.source_trials.append(source_trial) + return item + + # Create new item + item_id = self._generate_id(category) + item = PlaybookItem( + id=item_id, + category=category, + content=content, + source_trials=[source_trial] if source_trial else [], + tags=tags or [] + ) + self.items[item_id] = item + self.last_updated = datetime.now().isoformat() + self.version += 1 + return item + + def record_outcome(self, item_id: str, helpful: bool): + """Record whether using this insight was helpful or harmful.""" + if item_id in self.items: + if helpful: + self.items[item_id].helpful_count += 1 + else: + self.items[item_id].harmful_count += 1 + self.items[item_id].last_used = datetime.now().isoformat() + + def get_context_for_task( + self, + task_type: str, + max_items: int = 20, + min_confidence: float = 0.5 + ) -> str: + """ + Generate context string for LLM consumption. + + Filters by relevance and confidence, sorted by net score. + """ + relevant_items = [ + item for item in self.items.values() + if item.confidence >= min_confidence + ] + + # Sort by net score (most helpful first) + relevant_items.sort(key=lambda x: x.net_score, reverse=True) + + # Group by category + sections = {} + for item in relevant_items[:max_items]: + cat_name = item.category.name + if cat_name not in sections: + sections[cat_name] = [] + sections[cat_name].append(item.to_context_string()) + + # Build context string + lines = ["## Atomizer Knowledge Playbook\n"] + for cat_name, items in sections.items(): + lines.append(f"### {cat_name}") + lines.extend(items) + lines.append("") + + return "\n".join(lines) + + def prune_harmful(self, threshold: int = -3): + """Remove items that have proven consistently harmful.""" + to_remove = [ + item_id for item_id, item in self.items.items() + if item.net_score <= threshold + ] + for item_id in to_remove: + del self.items[item_id] + + def save(self, path: Path): + """Persist playbook to JSON.""" + data = { + "version": self.version, + "last_updated": self.last_updated, + "items": { + k: { + "id": v.id, + "category": v.category.value, + "content": v.content, + "helpful_count": v.helpful_count, + "harmful_count": v.harmful_count, + "created_at": v.created_at, + "last_used": v.last_used, + "source_trials": v.source_trials, + "tags": v.tags + } + for k, v in self.items.items() + } + } + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'w') as f: + json.dump(data, f, indent=2) + + @classmethod + def load(cls, path: Path) -> "AtomizerPlaybook": + """Load playbook from JSON.""" + if not path.exists(): + return cls() + + with open(path) as f: + data = json.load(f) + + playbook = cls( + version=data.get("version", 1), + last_updated=data.get("last_updated", datetime.now().isoformat()) + ) + + for item_data in data.get("items", {}).values(): + item = PlaybookItem( + id=item_data["id"], + category=InsightCategory(item_data["category"]), + content=item_data["content"], + helpful_count=item_data.get("helpful_count", 0), + harmful_count=item_data.get("harmful_count", 0), + created_at=item_data.get("created_at", ""), + last_used=item_data.get("last_used"), + source_trials=item_data.get("source_trials", []), + tags=item_data.get("tags", []) + ) + playbook.items[item.id] = item + + return playbook +``` + +#### 1.2 Create Reflector Component + +**File**: `optimization_engine/context/reflector.py` + +```python +""" +Reflector: Analyzes optimization outcomes to extract insights. + +Part of the ACE (Agentic Context Engineering) implementation for Atomizer. +""" + +from typing import Dict, Any, List, Optional +from dataclasses import dataclass +from pathlib import Path +import json + +from .playbook import AtomizerPlaybook, InsightCategory + +@dataclass +class OptimizationOutcome: + """Captured outcome from an optimization run.""" + trial_number: int + success: bool + objective_value: Optional[float] + constraint_violations: List[str] + solver_errors: List[str] + design_variables: Dict[str, float] + extractor_used: str + duration_seconds: float + notes: str = "" + +class AtomizerReflector: + """ + Analyzes optimization outcomes and extracts actionable insights. + + Implements the Reflector role from ACE framework: + - Examines successful and failed trials + - Extracts patterns that led to success/failure + - Formats insights for Curator integration + """ + + def __init__(self, playbook: AtomizerPlaybook): + self.playbook = playbook + self.pending_insights: List[Dict[str, Any]] = [] + + def analyze_trial(self, outcome: OptimizationOutcome) -> List[Dict[str, Any]]: + """ + Analyze a single trial outcome and extract insights. + + Returns list of insight candidates (not yet added to playbook). + """ + insights = [] + + # Analyze solver errors + for error in outcome.solver_errors: + if "convergence" in error.lower(): + insights.append({ + "category": InsightCategory.MISTAKE, + "content": f"Convergence failure with config: {self._summarize_config(outcome)}", + "helpful": False, + "trial": outcome.trial_number + }) + elif "mesh" in error.lower(): + insights.append({ + "category": InsightCategory.MISTAKE, + "content": f"Mesh-related error: {error[:100]}", + "helpful": False, + "trial": outcome.trial_number + }) + + # Analyze successful patterns + if outcome.success and outcome.objective_value is not None: + # Record successful design variable ranges + insights.append({ + "category": InsightCategory.STRATEGY, + "content": f"Successful design: {self._summarize_design(outcome)}", + "helpful": True, + "trial": outcome.trial_number + }) + + # Analyze constraint violations + for violation in outcome.constraint_violations: + insights.append({ + "category": InsightCategory.MISTAKE, + "content": f"Constraint violation: {violation}", + "helpful": False, + "trial": outcome.trial_number + }) + + self.pending_insights.extend(insights) + return insights + + def analyze_study_completion( + self, + study_name: str, + total_trials: int, + best_value: float, + convergence_rate: float + ) -> List[Dict[str, Any]]: + """ + Analyze completed study and extract high-level insights. + """ + insights = [] + + if convergence_rate > 0.9: + insights.append({ + "category": InsightCategory.STRATEGY, + "content": f"Study '{study_name}' achieved {convergence_rate:.0%} convergence - configuration is robust", + "helpful": True, + "trial": None + }) + elif convergence_rate < 0.5: + insights.append({ + "category": InsightCategory.MISTAKE, + "content": f"Study '{study_name}' had only {convergence_rate:.0%} convergence - review mesh and solver settings", + "helpful": False, + "trial": None + }) + + return insights + + def commit_insights(self) -> int: + """ + Commit pending insights to playbook (Curator handoff). + + Returns number of insights added. + """ + count = 0 + for insight in self.pending_insights: + item = self.playbook.add_insight( + category=insight["category"], + content=insight["content"], + source_trial=insight.get("trial") + ) + if not insight.get("helpful", True): + self.playbook.record_outcome(item.id, helpful=False) + count += 1 + + self.pending_insights = [] + return count + + def _summarize_config(self, outcome: OptimizationOutcome) -> str: + """Create brief config summary.""" + return f"extractor={outcome.extractor_used}, vars={len(outcome.design_variables)}" + + def _summarize_design(self, outcome: OptimizationOutcome) -> str: + """Create brief design summary.""" + vars_summary = ", ".join( + f"{k}={v:.3g}" for k, v in list(outcome.design_variables.items())[:3] + ) + return f"obj={outcome.objective_value:.4g}, {vars_summary}" +``` + +#### 1.3 Integration with OptimizationRunner + +**File**: `optimization_engine/core/runner.py` (modifications - POST-MIGRATION PATH) + +```python +# Add to imports +from optimization_engine.context.playbook import AtomizerPlaybook +from optimization_engine.context.reflector import AtomizerReflector, OptimizationOutcome + +class OptimizationRunner: + def __init__(self, ...): + # ... existing init ... + + # Initialize context engineering components + self.playbook = AtomizerPlaybook.load( + self.output_dir / "playbook.json" + ) + self.reflector = AtomizerReflector(self.playbook) + + def _objective(self, trial: optuna.Trial) -> float: + # ... existing trial logic ... + + # After trial completion, capture outcome for reflection + outcome = OptimizationOutcome( + trial_number=trial.number, + success=not failed, + objective_value=objective_value if not failed else None, + constraint_violations=constraint_violations, + solver_errors=solver_errors, + design_variables=design_vars, + extractor_used=self.config.get("extractor", "unknown"), + duration_seconds=trial_duration + ) + self.reflector.analyze_trial(outcome) + + return objective_value + + def run(self, n_trials: int) -> Dict[str, Any]: + # ... existing run logic ... + + # After study completion + self.reflector.analyze_study_completion( + study_name=self.study.study_name, + total_trials=len(self.study.trials), + best_value=self.study.best_value, + convergence_rate=successful_trials / total_trials + ) + + # Commit insights and save playbook + insights_added = self.reflector.commit_insights() + self.playbook.save(self.output_dir / "playbook.json") + + print(f"Added {insights_added} insights to playbook") +``` + +--- + +### Phase 2: Context Isolation & Loading (Week 2-3) + +**Goal**: Implement Write-Select-Compress-Isolate pattern for POS + +#### 2.1 Session State Schema + +**File**: `optimization_engine/context/session_state.py` + +```python +""" +Session state management with context isolation. + +Implements the "Isolate" pattern from context engineering: +- Exposed fields are sent to LLM +- Isolated fields are accessed selectively +""" + +from pydantic import BaseModel, Field +from typing import Dict, List, Optional, Any +from datetime import datetime +from enum import Enum + +class TaskType(Enum): + CREATE_STUDY = "create_study" + RUN_OPTIMIZATION = "run_optimization" + MONITOR_PROGRESS = "monitor_progress" + ANALYZE_RESULTS = "analyze_results" + DEBUG_ERROR = "debug_error" + CONFIGURE_SETTINGS = "configure_settings" + +class ExposedState(BaseModel): + """State exposed to LLM at every turn.""" + + # Current task context + task_type: Optional[TaskType] = None + current_objective: str = "" + + # Recent history (compressed) + recent_actions: List[str] = Field(default_factory=list, max_items=10) + recent_errors: List[str] = Field(default_factory=list, max_items=5) + + # Active study summary + study_name: Optional[str] = None + study_status: str = "unknown" + trials_completed: int = 0 + best_value: Optional[float] = None + + # Playbook excerpt (most relevant items) + active_playbook_items: List[str] = Field(default_factory=list, max_items=15) + +class IsolatedState(BaseModel): + """State isolated from LLM - accessed selectively.""" + + # Full optimization history (can be large) + full_trial_history: List[Dict[str, Any]] = Field(default_factory=list) + + # NX session state (heavy, complex) + nx_model_path: Optional[str] = None + nx_expressions: Dict[str, Any] = Field(default_factory=dict) + + # Neural network cache + neural_predictions: Dict[str, float] = Field(default_factory=dict) + + # Full playbook (loaded on demand) + full_playbook_path: Optional[str] = None + + # Debug information + last_solver_output: str = "" + last_f06_content: str = "" + +class AtomizerSessionState(BaseModel): + """ + Complete session state with exposure control. + + The exposed state is automatically injected into every LLM context. + The isolated state is accessed only when explicitly needed. + """ + + session_id: str + created_at: datetime = Field(default_factory=datetime.now) + last_updated: datetime = Field(default_factory=datetime.now) + + exposed: ExposedState = Field(default_factory=ExposedState) + isolated: IsolatedState = Field(default_factory=IsolatedState) + + def get_llm_context(self) -> str: + """Generate context string for LLM consumption.""" + lines = [ + "## Current Session State", + "", + f"**Task**: {self.exposed.task_type.value if self.exposed.task_type else 'Not set'}", + f"**Objective**: {self.exposed.current_objective}", + "", + ] + + if self.exposed.study_name: + lines.extend([ + f"### Active Study: {self.exposed.study_name}", + f"- Status: {self.exposed.study_status}", + f"- Trials: {self.exposed.trials_completed}", + f"- Best: {self.exposed.best_value}", + "", + ]) + + if self.exposed.recent_actions: + lines.append("### Recent Actions") + for action in self.exposed.recent_actions[-5:]: + lines.append(f"- {action}") + lines.append("") + + if self.exposed.recent_errors: + lines.append("### Recent Errors (address these)") + for error in self.exposed.recent_errors: + lines.append(f"- ⚠️ {error}") + lines.append("") + + if self.exposed.active_playbook_items: + lines.append("### Relevant Knowledge") + for item in self.exposed.active_playbook_items: + lines.append(f"- {item}") + lines.append("") + + return "\n".join(lines) + + def add_action(self, action: str): + """Record an action (auto-compresses old actions).""" + self.exposed.recent_actions.append(action) + if len(self.exposed.recent_actions) > 10: + # Compress: keep first, last 5, summarize middle + self.exposed.recent_actions = ( + [self.exposed.recent_actions[0]] + + ["... (earlier actions summarized)"] + + self.exposed.recent_actions[-5:] + ) + self.last_updated = datetime.now() + + def add_error(self, error: str): + """Record an error for LLM attention.""" + self.exposed.recent_errors.append(error) + self.exposed.recent_errors = self.exposed.recent_errors[-5:] + self.last_updated = datetime.now() + + def load_isolated_data(self, key: str) -> Any: + """Explicitly load isolated data when needed.""" + return getattr(self.isolated, key, None) +``` + +#### 2.2 Context Loader Enhancement + +**File**: `.claude/skills/02_CONTEXT_LOADER.md` (updated) + +```markdown +# Context Loader - Enhanced with Playbook Integration + +## Loading Rules by Task Type + +### CREATE_STUDY +**Always Load**: +- `core/study-creation-core.md` +- `SYS_12_EXTRACTOR_LIBRARY.md` +- **Playbook**: Filter by tags=['study_creation', 'design_variables'] + +**Load If**: +- `modules/zernike-optimization.md`: if "telescope" or "mirror" in query +- `modules/neural-acceleration.md`: if trials > 50 + +### RUN_OPTIMIZATION +**Always Load**: +- `OP_02_RUN_OPTIMIZATION.md` +- **Playbook**: Filter by tags=['solver', 'convergence', 'mesh'] + +**Load If**: +- Recent errors exist → Include mistake items from playbook + +### DEBUG_ERROR +**Always Load**: +- `OP_06_TROUBLESHOOT.md` +- **Playbook**: Filter by category=MISTAKE, min_confidence=0.3 +- Session state recent_errors + +**Load If**: +- "convergence" in error → Load solver-specific playbook items +- "mesh" in error → Load mesh-specific playbook items + +## Playbook Integration Pattern + +```python +# In context loader +def load_context_for_task(task_type: TaskType, session: AtomizerSessionState): + context_parts = [] + + # 1. Load protocol docs (existing behavior) + context_parts.append(load_protocol(task_type)) + + # 2. Load session state (exposed only) + context_parts.append(session.get_llm_context()) + + # 3. Load relevant playbook items + playbook = AtomizerPlaybook.load(PLAYBOOK_PATH) + playbook_context = playbook.get_context_for_task( + task_type=task_type.value, + max_items=15, + min_confidence=0.6 + ) + context_parts.append(playbook_context) + + # 4. Add error-specific items if debugging + if task_type == TaskType.DEBUG_ERROR: + for error in session.exposed.recent_errors: + relevant = playbook.search_by_content(error, category=InsightCategory.MISTAKE) + context_parts.extend([item.to_context_string() for item in relevant]) + + return "\n\n---\n\n".join(context_parts) +``` +``` + +--- + +### Phase 3: KV-Cache Optimization (Week 3) + +**Goal**: Maximize cache hits for 10x cost reduction + +#### 3.1 Stable Prefix Architecture + +**File**: `.claude/skills/SYSTEM_PROMPT_TEMPLATE.md` + +```markdown +# Atomizer System Prompt Template + +## Structure for KV-Cache Optimization + +The system prompt is structured to maximize KV-cache hits: + +``` +[SECTION 1: STABLE - Never changes] +- Atomizer identity and capabilities +- Core principles (talk don't click) +- Tool schemas and definitions +- Base protocol routing table + +[SECTION 2: SEMI-STABLE - Changes per session type] +- Active protocol definition +- Task-specific instructions +- Relevant playbook items (top 10 by score) + +[SECTION 3: DYNAMIC - Changes every turn] +- Current session state +- Recent actions/errors +- User's latest message +``` + +## Implementation + +### Stable Prefix (Cache This) + +``` +You are assisting with **Atomizer**, an LLM-first FEA optimization framework. + +## Core Capabilities +- Natural language → optimization configuration +- NX Nastran integration via journals +- Multi-strategy optimization (TPE, CMA-ES, NSGA-II) +- Real-time progress monitoring +- Neural acceleration (600-1000x speedup) + +## Principles +1. Talk, don't click - users describe goals in plain language +2. Never modify master models - work on copies +3. Always validate before running +4. Document everything + +## Available Tools +[... tool schemas - NEVER CHANGE ORDER ...] + +## Protocol Routing +| Intent | Protocol | Priority | +|--------|----------|----------| +| Create study | OP_01 | 1 | +| Run optimization | OP_02 | 1 | +| Monitor progress | OP_03 | 2 | +| Analyze results | OP_04 | 2 | +| Debug errors | OP_06 | 1 | +``` + +### Semi-Stable Section (Per Session) + +``` +## Active Task: {task_type} + +### Loaded Protocol: {protocol_name} +{protocol_content} + +### Relevant Knowledge (from {playbook_version}) +{playbook_items} +``` + +### Dynamic Section (Per Turn) + +``` +## Current State +{session_state.get_llm_context()} + +## User Message +{user_message} +``` +``` + +#### 3.2 Cache Monitoring + +**File**: `optimization_engine/context/cache_monitor.py` + +```python +""" +Monitor and optimize KV-cache hit rates. +""" + +from dataclasses import dataclass +from typing import Optional +import hashlib + +@dataclass +class CacheStats: + total_requests: int = 0 + cache_hits: int = 0 + prefix_length: int = 0 + + @property + def hit_rate(self) -> float: + if self.total_requests == 0: + return 0.0 + return self.cache_hits / self.total_requests + +class ContextCacheOptimizer: + """ + Tracks and optimizes context for cache efficiency. + """ + + def __init__(self): + self.stats = CacheStats() + self._last_prefix_hash: Optional[str] = None + + def prepare_context( + self, + stable_prefix: str, + semi_stable: str, + dynamic: str + ) -> str: + """ + Assemble context optimized for caching. + + Tracks whether prefix changed (cache miss). + """ + # Hash the stable prefix + prefix_hash = hashlib.md5(stable_prefix.encode()).hexdigest() + + self.stats.total_requests += 1 + if prefix_hash == self._last_prefix_hash: + self.stats.cache_hits += 1 + + self._last_prefix_hash = prefix_hash + self.stats.prefix_length = len(stable_prefix) + + # Assemble with clear boundaries + return f"""{stable_prefix} + +--- +{semi_stable} + +--- +{dynamic}""" + + def get_report(self) -> str: + """Generate cache efficiency report.""" + return f""" +Cache Statistics: +- Requests: {self.stats.total_requests} +- Cache Hits: {self.stats.cache_hits} +- Hit Rate: {self.stats.hit_rate:.1%} +- Stable Prefix: {self.stats.prefix_length} chars + +Estimated Cost Savings: {self.stats.hit_rate * 90:.0f}% +(Based on 10x cost difference between cached/uncached tokens) +""" +``` + +--- + +### Phase 4: Error Recovery & Learning (Week 4) + +**Goal**: Leave wrong turns in context, learn from failures + +#### 4.1 Error Preservation Hook + +**File**: `optimization_engine/plugins/post_solve/error_tracker.py` + +```python +""" +Error Tracker Hook + +Preserves solver errors and failures in context for learning. +Based on Manus insight: "leave the wrong turns in the context" +""" + +from pathlib import Path +from datetime import datetime +import json + +def track_error(context: dict) -> dict: + """ + Hook that preserves errors for context learning. + + Called at post_solve when solver fails. + """ + trial_number = context.get('trial_number', -1) + output_dir = Path(context.get('output_dir', '.')) + + # Capture error details + error_info = { + "trial": trial_number, + "timestamp": datetime.now().isoformat(), + "solver_returncode": context.get('solver_returncode'), + "error_type": classify_error(context), + "design_variables": context.get('design_variables', {}), + "error_message": context.get('error_message', ''), + "f06_snippet": extract_f06_error(context.get('f06_path')) + } + + # Append to error log (never overwrite - accumulate) + error_log_path = output_dir / "error_history.jsonl" + with open(error_log_path, 'a') as f: + f.write(json.dumps(error_info) + "\n") + + # Update session state for LLM context + if 'session_state' in context: + context['session_state'].add_error( + f"Trial {trial_number}: {error_info['error_type']} - {error_info['error_message'][:100]}" + ) + + return {"error_tracked": True, "error_type": error_info['error_type']} + +def classify_error(context: dict) -> str: + """Classify error type for playbook categorization.""" + error_msg = context.get('error_message', '').lower() + + if 'convergence' in error_msg: + return "convergence_failure" + elif 'mesh' in error_msg or 'element' in error_msg: + return "mesh_error" + elif 'singular' in error_msg or 'matrix' in error_msg: + return "singularity" + elif 'memory' in error_msg or 'allocation' in error_msg: + return "memory_error" + elif 'license' in error_msg: + return "license_error" + else: + return "unknown_error" + +def extract_f06_error(f06_path: str) -> str: + """Extract error section from F06 file.""" + if not f06_path or not Path(f06_path).exists(): + return "" + + try: + with open(f06_path) as f: + content = f.read() + + # Look for error indicators + error_markers = ["*** USER FATAL", "*** SYSTEM FATAL", "*** USER WARNING"] + for marker in error_markers: + if marker in content: + idx = content.index(marker) + return content[idx:idx+500] + + return "" + except Exception: + return "" + +# Hook registration +HOOK_CONFIG = { + "name": "error_tracker", + "hook_point": "post_solve", + "priority": 100, # Run early to capture before cleanup + "enabled": True, + "description": "Preserves solver errors for context learning" +} +``` + +#### 4.2 Feedback Loop Integration + +**File**: `optimization_engine/context/feedback_loop.py` + +```python +""" +Feedback Loop: Connects optimization outcomes to playbook updates. + +Implements ACE's "leverage natural execution feedback" principle. +""" + +from typing import Dict, Any, List +from pathlib import Path + +from .playbook import AtomizerPlaybook, InsightCategory +from .reflector import AtomizerReflector, OptimizationOutcome + +class FeedbackLoop: + """ + Automated feedback loop that learns from optimization runs. + + Key insight from ACE: Use execution feedback (success/failure) + as the learning signal, not labeled data. + """ + + def __init__(self, playbook_path: Path): + self.playbook = AtomizerPlaybook.load(playbook_path) + self.reflector = AtomizerReflector(self.playbook) + self.playbook_path = playbook_path + + def process_trial_result( + self, + trial_number: int, + success: bool, + objective_value: float, + design_variables: Dict[str, float], + context_items_used: List[str], # Which playbook items were in context + errors: List[str] = None + ): + """ + Process a trial result and update playbook accordingly. + + This is the core learning mechanism: + - If trial succeeded with certain playbook items → increase helpful count + - If trial failed with certain playbook items → increase harmful count + """ + # Update playbook item scores based on outcome + for item_id in context_items_used: + self.playbook.record_outcome(item_id, helpful=success) + + # Create outcome for reflection + outcome = OptimizationOutcome( + trial_number=trial_number, + success=success, + objective_value=objective_value if success else None, + constraint_violations=[], + solver_errors=errors or [], + design_variables=design_variables, + extractor_used="", + duration_seconds=0 + ) + + # Reflect on outcome + self.reflector.analyze_trial(outcome) + + def finalize_study(self, study_stats: Dict[str, Any]): + """ + Called when study completes. Commits insights and prunes playbook. + """ + # Analyze study-level patterns + self.reflector.analyze_study_completion( + study_name=study_stats.get("name", "unknown"), + total_trials=study_stats.get("total_trials", 0), + best_value=study_stats.get("best_value", 0), + convergence_rate=study_stats.get("convergence_rate", 0) + ) + + # Commit all pending insights + insights_added = self.reflector.commit_insights() + + # Prune consistently harmful items + self.playbook.prune_harmful(threshold=-3) + + # Save updated playbook + self.playbook.save(self.playbook_path) + + return { + "insights_added": insights_added, + "playbook_size": len(self.playbook.items), + "playbook_version": self.playbook.version + } +``` + +--- + +### Phase 5: Context Compaction (Week 4-5) + +**Goal**: Handle long-running optimization sessions without context overflow + +#### 5.1 Compaction Manager + +**File**: `optimization_engine/context/compaction.py` + +```python +""" +Context Compaction for Long-Running Optimizations + +Based on Google ADK's compaction architecture: +- Trigger compaction when threshold reached +- Summarize older events +- Preserve recent detail +""" + +from typing import List, Dict, Any +from dataclasses import dataclass, field +from datetime import datetime + +@dataclass +class ContextEvent: + """Single event in optimization context.""" + timestamp: datetime + event_type: str # trial_start, trial_complete, error, milestone + summary: str + details: Dict[str, Any] = field(default_factory=dict) + compacted: bool = False + +class CompactionManager: + """ + Manages context compaction for long optimization sessions. + + Strategy: + - Keep last N events in full detail + - Summarize older events into milestone markers + - Preserve error events (never compact errors) + """ + + def __init__( + self, + compaction_threshold: int = 50, + keep_recent: int = 20, + keep_errors: bool = True + ): + self.events: List[ContextEvent] = [] + self.compaction_threshold = compaction_threshold + self.keep_recent = keep_recent + self.keep_errors = keep_errors + self.compaction_count = 0 + + def add_event(self, event: ContextEvent): + """Add event and trigger compaction if needed.""" + self.events.append(event) + + if len(self.events) > self.compaction_threshold: + self._compact() + + def _compact(self): + """ + Compact older events into summaries. + + Preserves: + - All error events + - Last `keep_recent` events + - Milestone summaries of compacted regions + """ + if len(self.events) <= self.keep_recent: + return + + # Split into old and recent + old_events = self.events[:-self.keep_recent] + recent_events = self.events[-self.keep_recent:] + + # Separate errors from old events + error_events = [e for e in old_events if e.event_type == "error"] + non_error_events = [e for e in old_events if e.event_type != "error"] + + # Summarize non-error old events + if non_error_events: + summary = self._create_summary(non_error_events) + compaction_event = ContextEvent( + timestamp=non_error_events[0].timestamp, + event_type="compaction", + summary=summary, + details={ + "events_compacted": len(non_error_events), + "compaction_number": self.compaction_count + }, + compacted=True + ) + self.compaction_count += 1 + + # Rebuild events list + self.events = [compaction_event] + error_events + recent_events + else: + self.events = error_events + recent_events + + def _create_summary(self, events: List[ContextEvent]) -> str: + """Create summary of compacted events.""" + trial_events = [e for e in events if "trial" in e.event_type] + + if not trial_events: + return f"[{len(events)} events compacted]" + + # Extract trial statistics + trial_numbers = [] + objectives = [] + + for e in trial_events: + if "trial_number" in e.details: + trial_numbers.append(e.details["trial_number"]) + if "objective" in e.details: + objectives.append(e.details["objective"]) + + if trial_numbers and objectives: + return ( + f"Trials {min(trial_numbers)}-{max(trial_numbers)}: " + f"Best={min(objectives):.4g}, " + f"Avg={sum(objectives)/len(objectives):.4g}" + ) + elif trial_numbers: + return f"Trials {min(trial_numbers)}-{max(trial_numbers)} completed" + else: + return f"[{len(events)} events compacted]" + + def get_context_string(self) -> str: + """Generate context string from events.""" + lines = ["## Optimization History", ""] + + for event in self.events: + if event.compacted: + lines.append(f"📦 {event.summary}") + elif event.event_type == "error": + lines.append(f"⚠️ {event.summary}") + else: + lines.append(f"- {event.summary}") + + return "\n".join(lines) + + def get_stats(self) -> Dict[str, Any]: + """Get compaction statistics.""" + return { + "total_events": len(self.events), + "compaction_count": self.compaction_count, + "error_events": len([e for e in self.events if e.event_type == "error"]), + "compacted_events": len([e for e in self.events if e.compacted]) + } +``` + +--- + +## Part 3: Claude Code Session Integration + +### 3.1 Enhanced Bootstrap for Claude Code + +**File**: `.claude/skills/00_BOOTSTRAP_V2.md` + +```markdown +# Atomizer Bootstrap v2.0 - Context-Aware Sessions + +## Session Initialization + +On session start, perform these steps: + +### Step 1: Load Playbook +```bash +# Check for existing playbook +cat optimization_engine/context/playbook.json 2>/dev/null | head -20 +``` + +If playbook exists, extract top insights: +- Filter by task type (inferred from user's first message) +- Include top 10 by net_score +- Always include recent mistakes (last 5) + +### Step 2: Initialize Session State +```python +from optimization_engine.context.session_state import AtomizerSessionState, TaskType + +session = AtomizerSessionState(session_id="current") +session.exposed.task_type = TaskType.CREATE_STUDY # Update based on intent +``` + +### Step 3: Load Task-Specific Context +Based on detected task type, load protocols per `02_CONTEXT_LOADER.md` + +### Step 4: Inject Playbook Items +Add relevant playbook items to `session.exposed.active_playbook_items` + +--- + +## Error Handling Protocol + +When ANY error occurs: + +1. **Preserve the error** - Add to session state +2. **Check playbook** - Look for matching mistake patterns +3. **Learn from it** - If novel error, queue for playbook addition +4. **Show to user** - Include error context in response + +```python +# On error +session.add_error(f"{error_type}: {error_message}") + +# Check playbook for similar errors +similar = playbook.search_by_content(error_message, category=InsightCategory.MISTAKE) +if similar: + print(f"Known issue: {similar[0].content}") +else: + reflector.queue_insight(InsightCategory.MISTAKE, error_message) +``` + +--- + +## Context Budget Management + +Total context budget: ~100K tokens + +Allocation: +- **Stable prefix**: 5K tokens (cached) +- **Protocols**: 10K tokens +- **Playbook items**: 5K tokens +- **Session state**: 2K tokens +- **Conversation history**: 30K tokens +- **Working space**: 48K tokens + +If approaching limit: +1. Trigger compaction of old events +2. Reduce playbook items to top 5 +3. Summarize conversation history +``` + +### 3.2 Dashboard Integration + +**File**: `atomizer-dashboard/backend/api/routes/context.py` + +```python +""" +Context Engineering API Routes + +Provides endpoints for: +- Viewing playbook contents +- Managing session state +- Triggering compaction +- Monitoring cache efficiency +""" + +from fastapi import APIRouter, HTTPException +from pathlib import Path +from typing import Optional + +router = APIRouter(prefix="/context", tags=["context"]) + +ATOMIZER_ROOT = Path(__file__).parents[4] +PLAYBOOK_PATH = ATOMIZER_ROOT / "optimization_engine" / "context" / "playbook.json" + +@router.get("/playbook") +async def get_playbook( + category: Optional[str] = None, + min_score: int = 0, + limit: int = 50 +): + """Get playbook items with optional filtering.""" + from optimization_engine.context.playbook import AtomizerPlaybook, InsightCategory + + playbook = AtomizerPlaybook.load(PLAYBOOK_PATH) + + items = list(playbook.items.values()) + + # Filter by category + if category: + try: + cat = InsightCategory(category) + items = [i for i in items if i.category == cat] + except ValueError: + raise HTTPException(400, f"Invalid category: {category}") + + # Filter by score + items = [i for i in items if i.net_score >= min_score] + + # Sort by score + items.sort(key=lambda x: x.net_score, reverse=True) + + return { + "total": len(playbook.items), + "filtered": len(items), + "items": [ + { + "id": i.id, + "category": i.category.value, + "content": i.content, + "helpful": i.helpful_count, + "harmful": i.harmful_count, + "score": i.net_score, + "confidence": i.confidence + } + for i in items[:limit] + ] + } + +@router.post("/playbook/feedback") +async def record_feedback(item_id: str, helpful: bool): + """Record feedback on a playbook item.""" + from optimization_engine.context.playbook import AtomizerPlaybook + + playbook = AtomizerPlaybook.load(PLAYBOOK_PATH) + + if item_id not in playbook.items: + raise HTTPException(404, f"Item not found: {item_id}") + + playbook.record_outcome(item_id, helpful=helpful) + playbook.save(PLAYBOOK_PATH) + + item = playbook.items[item_id] + return { + "id": item_id, + "new_score": item.net_score, + "confidence": item.confidence + } + +@router.get("/session/{session_id}") +async def get_session_state(session_id: str): + """Get current session state.""" + # Implementation depends on session storage + pass + +@router.get("/cache/stats") +async def get_cache_stats(): + """Get KV-cache efficiency statistics.""" + from optimization_engine.context.cache_monitor import ContextCacheOptimizer + + # Would need to access singleton cache optimizer + return { + "message": "Cache stats endpoint - implement with actual cache monitor" + } +``` + +--- + +## Part 4: Testing & Validation + +### 4.1 Test Suite + +**File**: `tests/test_context_engineering.py` + +```python +""" +Test suite for context engineering components. +""" + +import pytest +from pathlib import Path +import tempfile +import json + +from optimization_engine.context.playbook import ( + AtomizerPlaybook, + PlaybookItem, + InsightCategory +) +from optimization_engine.context.reflector import ( + AtomizerReflector, + OptimizationOutcome +) +from optimization_engine.context.session_state import ( + AtomizerSessionState, + TaskType +) +from optimization_engine.context.compaction import ( + CompactionManager, + ContextEvent +) + + +class TestAtomizerPlaybook: + """Tests for the playbook system.""" + + def test_add_insight(self): + """Test adding insights to playbook.""" + playbook = AtomizerPlaybook() + + item = playbook.add_insight( + category=InsightCategory.STRATEGY, + content="Use shell elements for thin walls", + source_trial=1 + ) + + assert item.id == "str-00001" + assert item.helpful_count == 0 + assert item.harmful_count == 0 + assert len(playbook.items) == 1 + + def test_deduplication(self): + """Test that duplicate insights are merged.""" + playbook = AtomizerPlaybook() + + playbook.add_insight(InsightCategory.STRATEGY, "Use shell elements") + playbook.add_insight(InsightCategory.STRATEGY, "Use shell elements") + + assert len(playbook.items) == 1 + assert playbook.items["str-00001"].helpful_count == 1 + + def test_outcome_tracking(self): + """Test helpful/harmful tracking.""" + playbook = AtomizerPlaybook() + item = playbook.add_insight(InsightCategory.STRATEGY, "Test insight") + + playbook.record_outcome(item.id, helpful=True) + playbook.record_outcome(item.id, helpful=True) + playbook.record_outcome(item.id, helpful=False) + + assert item.helpful_count == 2 + assert item.harmful_count == 1 + assert item.net_score == 1 + + def test_persistence(self, tmp_path): + """Test save/load cycle.""" + playbook = AtomizerPlaybook() + playbook.add_insight(InsightCategory.MISTAKE, "Don't do this") + + save_path = tmp_path / "playbook.json" + playbook.save(save_path) + + loaded = AtomizerPlaybook.load(save_path) + assert len(loaded.items) == 1 + assert "mis-00001" in loaded.items + + def test_pruning(self): + """Test harmful item pruning.""" + playbook = AtomizerPlaybook() + item = playbook.add_insight(InsightCategory.STRATEGY, "Bad advice") + + # Record many harmful outcomes + for _ in range(5): + playbook.record_outcome(item.id, helpful=False) + + playbook.prune_harmful(threshold=-3) + assert len(playbook.items) == 0 + + +class TestAtomizerReflector: + """Tests for the reflector component.""" + + def test_analyze_failed_trial(self): + """Test analysis of failed trial.""" + playbook = AtomizerPlaybook() + reflector = AtomizerReflector(playbook) + + outcome = OptimizationOutcome( + trial_number=1, + success=False, + objective_value=None, + constraint_violations=["stress > 250 MPa"], + solver_errors=["convergence failure at iteration 50"], + design_variables={"thickness": 0.5}, + extractor_used="stress_extractor", + duration_seconds=120 + ) + + insights = reflector.analyze_trial(outcome) + + assert len(insights) >= 2 # At least error + constraint + assert any(i["category"] == InsightCategory.MISTAKE for i in insights) + + def test_commit_insights(self): + """Test committing insights to playbook.""" + playbook = AtomizerPlaybook() + reflector = AtomizerReflector(playbook) + + outcome = OptimizationOutcome( + trial_number=1, + success=True, + objective_value=100.0, + constraint_violations=[], + solver_errors=[], + design_variables={"thickness": 1.0}, + extractor_used="mass_extractor", + duration_seconds=60 + ) + + reflector.analyze_trial(outcome) + count = reflector.commit_insights() + + assert count > 0 + assert len(playbook.items) > 0 + + +class TestSessionState: + """Tests for session state management.""" + + def test_exposed_state_context(self): + """Test LLM context generation.""" + session = AtomizerSessionState(session_id="test") + session.exposed.task_type = TaskType.CREATE_STUDY + session.exposed.study_name = "bracket_opt" + session.exposed.trials_completed = 25 + session.exposed.best_value = 0.5 + + context = session.get_llm_context() + + assert "bracket_opt" in context + assert "25" in context + assert "0.5" in context + + def test_action_compression(self): + """Test automatic action compression.""" + session = AtomizerSessionState(session_id="test") + + for i in range(15): + session.add_action(f"Action {i}") + + # Should be compressed + assert len(session.exposed.recent_actions) <= 12 + assert "summarized" in session.exposed.recent_actions[1].lower() + + +class TestCompactionManager: + """Tests for context compaction.""" + + def test_compaction_trigger(self): + """Test that compaction triggers at threshold.""" + manager = CompactionManager(compaction_threshold=10, keep_recent=5) + + for i in range(15): + manager.add_event(ContextEvent( + timestamp=datetime.now(), + event_type="trial_complete", + summary=f"Trial {i} complete", + details={"trial_number": i, "objective": i * 0.1} + )) + + assert manager.compaction_count > 0 + assert len(manager.events) <= 10 + + def test_error_preservation(self): + """Test that errors are never compacted.""" + manager = CompactionManager(compaction_threshold=10, keep_recent=3) + + # Add error early + manager.add_event(ContextEvent( + timestamp=datetime.now(), + event_type="error", + summary="Critical solver failure" + )) + + # Add many regular events + for i in range(20): + manager.add_event(ContextEvent( + timestamp=datetime.now(), + event_type="trial_complete", + summary=f"Trial {i}" + )) + + # Error should still be present + errors = [e for e in manager.events if e.event_type == "error"] + assert len(errors) == 1 +``` + +### 4.2 Integration Test + +**File**: `tests/test_context_integration.py` + +```python +""" +Integration test for full context engineering pipeline. +""" + +import pytest +from pathlib import Path +import tempfile + +def test_full_optimization_with_context_engineering(): + """ + End-to-end test of optimization with context engineering. + + Simulates: + 1. Starting fresh session + 2. Running optimization with failures + 3. Verifying playbook learns from failures + 4. Running second optimization + 5. Verifying improved performance + """ + from optimization_engine.context.playbook import AtomizerPlaybook + from optimization_engine.context.feedback_loop import FeedbackLoop + + with tempfile.TemporaryDirectory() as tmp_dir: + playbook_path = Path(tmp_dir) / "playbook.json" + + # Initialize feedback loop + feedback = FeedbackLoop(playbook_path) + + # Simulate first study with failures + for i in range(10): + success = i % 3 != 0 # Every 3rd trial fails + feedback.process_trial_result( + trial_number=i, + success=success, + objective_value=100 - i if success else 0, + design_variables={"thickness": 0.5 + i * 0.1}, + context_items_used=[], + errors=["convergence failure"] if not success else [] + ) + + # Finalize and check learning + result = feedback.finalize_study({ + "name": "test_study", + "total_trials": 10, + "best_value": 91, + "convergence_rate": 0.7 + }) + + assert result["insights_added"] > 0 + + # Load playbook and verify content + playbook = AtomizerPlaybook.load(playbook_path) + + # Should have learned about convergence failures + mistakes = [ + item for item in playbook.items.values() + if item.category.value == "mis" + ] + assert len(mistakes) > 0 +``` + +--- + +## Part 5: Rollout Plan + +### Week 1-2: Foundation +- [ ] Implement `AtomizerPlaybook` class +- [ ] Implement `AtomizerReflector` class +- [ ] Add playbook persistence (JSON) +- [ ] Write unit tests +- [ ] Integrate with existing LAC concepts + +### Week 3: Context Management +- [ ] Implement `AtomizerSessionState` +- [ ] Update `02_CONTEXT_LOADER.md` with playbook integration +- [ ] Create stable prefix template +- [ ] Implement cache monitoring + +### Week 4: Learning Loop +- [ ] Implement `FeedbackLoop` +- [ ] Create error tracker hook +- [ ] Implement compaction manager +- [ ] Integration testing + +### Week 5: Claude Code Integration +- [ ] Update `00_BOOTSTRAP.md` to v2 +- [ ] Add dashboard API routes +- [ ] Create playbook visualization component +- [ ] End-to-end testing with real optimizations + +### Week 6: Polish & Documentation +- [ ] Performance benchmarking +- [ ] Cost analysis (cache hit rates) +- [ ] Documentation updates +- [ ] Team training materials + +--- + +## Success Metrics + +| Metric | Baseline | Target | Measurement | +|--------|----------|--------|-------------| +| Task success rate | ~70% | 80-85% | Track via feedback loop | +| Repeated mistakes | N/A | <20% recurrence | Playbook harmful counts | +| Cache hit rate | 0% | >70% | Cache monitor stats | +| Cost per session | $X | 0.3X | API billing analysis | +| Playbook growth | 0 | 100+ items/month | Playbook stats | + +--- + +## References + +1. **ACE Framework**: Zhang et al., "Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models", arXiv:2510.04618, Oct 2025 +2. **Manus Blog**: "Context Engineering for AI Agents: Lessons from Building Manus" +3. **Anthropic**: "Effective context engineering for AI agents" +4. **LangChain**: "Context Engineering for Agents" +5. **Google ADK**: "Architecting efficient context-aware multi-agent framework" + +--- + +*Document generated: December 2025* +*For Claude Code implementation sessions* diff --git a/docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md b/docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md new file mode 100644 index 00000000..2f18af29 --- /dev/null +++ b/docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md @@ -0,0 +1,239 @@ +# OP_07: Disk Space Optimization + +**Version:** 1.0 +**Last Updated:** 2025-12-29 + +## Overview + +This protocol manages disk space for Atomizer studies through: +1. **Local cleanup** - Remove regenerable files from completed studies +2. **Remote archival** - Archive to dalidou server (14TB available) +3. **On-demand restore** - Pull archived studies when needed + +## Disk Usage Analysis + +### Typical Study Breakdown + +| File Type | Size/Trial | Purpose | Keep? | +|-----------|------------|---------|-------| +| `.op2` | 68 MB | Nastran results | **YES** - Needed for analysis | +| `.prt` | 30 MB | NX parts | NO - Copy of master | +| `.dat` | 16 MB | Solver input | NO - Regenerable | +| `.fem` | 14 MB | FEM mesh | NO - Copy of master | +| `.sim` | 7 MB | Simulation | NO - Copy of master | +| `.afm` | 4 MB | Assembly FEM | NO - Regenerable | +| `.json` | <1 MB | Params/results | **YES** - Metadata | +| Logs | <1 MB | F04/F06/log | NO - Diagnostic only | + +**Per-trial overhead:** ~150 MB total, only ~70 MB essential + +### M1_Mirror Example + +``` +Current: 194 GB (28 studies, 2000+ trials) +After cleanup: 95 GB (51% reduction) +After archive: 5 GB (keep best_design_archive only) +``` + +## Commands + +### 1. Analyze Disk Usage + +```bash +# Single study +archive_study.bat analyze studies\M1_Mirror\m1_mirror_V12 + +# All studies in a project +archive_study.bat analyze studies\M1_Mirror +``` + +Output shows: +- Total size +- Essential vs deletable breakdown +- Trial count per study +- Per-extension analysis + +### 2. Cleanup Completed Study + +```bash +# Dry run (default) - see what would be deleted +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 + +# Actually delete +archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute +``` + +**What gets deleted:** +- `.prt`, `.fem`, `.sim`, `.afm` in trial folders +- `.dat`, `.f04`, `.f06`, `.log`, `.diag` solver files +- Temp files (`.txt`, `.exp`, `.bak`) + +**What is preserved:** +- `1_setup/` folder (master model) +- `3_results/` folder (database, reports) +- All `.op2` files (Nastran results) +- All `.json` files (params, metadata) +- All `.npz` files (Zernike coefficients) +- `best_design_archive/` folder + +### 3. Archive to Remote Server + +```bash +# Dry run +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 + +# Actually archive +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute + +# Use Tailscale (when not on local network) +archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute --tailscale +``` + +**Process:** +1. Creates compressed `.tar.gz` archive +2. Uploads to `papa@192.168.86.50:/srv/storage/atomizer-archive/` +3. Deletes local archive after successful upload + +### 4. List Remote Archives + +```bash +archive_study.bat list + +# Via Tailscale +archive_study.bat list --tailscale +``` + +### 5. Restore from Remote + +```bash +# Restore to studies/ folder +archive_study.bat restore m1_mirror_V12 + +# Via Tailscale +archive_study.bat restore m1_mirror_V12 --tailscale +``` + +## Remote Server Setup + +**Server:** dalidou (Lenovo W520) +- Local IP: `192.168.86.50` +- Tailscale IP: `100.80.199.40` +- SSH user: `papa` +- Archive path: `/srv/storage/atomizer-archive/` + +### First-Time Setup + +SSH into dalidou and create the archive directory: + +```bash +ssh papa@192.168.86.50 +mkdir -p /srv/storage/atomizer-archive +``` + +Ensure SSH key authentication is set up for passwordless transfers: + +```bash +# On Windows (PowerShell) +ssh-copy-id papa@192.168.86.50 +``` + +## Recommended Workflow + +### During Active Optimization + +Keep all files - you may need to re-run specific trials. + +### After Study Completion + +1. **Generate final report** (`STUDY_REPORT.md`) +2. **Archive best design** to `3_results/best_design_archive/` +3. **Cleanup:** + ```bash + archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute + ``` + +### For Long-Term Storage + +1. **After cleanup**, archive to server: + ```bash + archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute + ``` + +2. **Optionally delete local** (keep only `3_results/best_design_archive/`) + +### When Revisiting Old Study + +1. **Restore:** + ```bash + archive_study.bat restore m1_mirror_V12 + ``` + +2. If you need to re-run trials, the `1_setup/` master files allow regenerating everything + +## Safety Features + +- **Dry run by default** - Must add `--execute` to actually delete/transfer +- **Master files preserved** - `1_setup/` is never touched +- **Results preserved** - `3_results/` is never touched +- **Essential files preserved** - OP2, JSON, NPZ always kept + +## Disk Space Targets + +| Stage | M1_Mirror Target | +|-------|------------------| +| Active development | 200 GB (full) | +| Completed studies | 95 GB (after cleanup) | +| Archived (minimal local) | 5 GB (best only) | +| Server archive | 50 GB compressed | + +## Troubleshooting + +### SSH Connection Failed + +```bash +# Test connectivity +ping 192.168.86.50 + +# Test SSH +ssh papa@192.168.86.50 "echo connected" + +# If on different network, use Tailscale +ssh papa@100.80.199.40 "echo connected" +``` + +### Archive Upload Slow + +Large studies (50+ GB) take time. The tool uses `rsync` with progress display. +For very large archives, consider running overnight or using direct LAN connection. + +### Out of Disk Space During Archive + +The archive is created locally first. Ensure you have ~1.5x the study size free: +- 20 GB study = ~30 GB temp space needed + +## Python API + +```python +from optimization_engine.utils.study_archiver import ( + analyze_study, + cleanup_study, + archive_to_remote, + restore_from_remote, + list_remote_archives, +) + +# Analyze +analysis = analyze_study(Path("studies/M1_Mirror/m1_mirror_V12")) +print(f"Deletable: {analysis['deletable_size']/1e9:.2f} GB") + +# Cleanup (dry_run=False to actually delete) +cleanup_study(Path("studies/M1_Mirror/m1_mirror_V12"), dry_run=False) + +# Archive +archive_to_remote(Path("studies/M1_Mirror/m1_mirror_V12"), dry_run=False) + +# List remote +archives = list_remote_archives() +for a in archives: + print(f"{a['name']}: {a['size']}") +``` diff --git a/docs/protocols/system/SYS_16_SELF_AWARE_TURBO.md b/docs/protocols/system/SYS_16_SELF_AWARE_TURBO.md new file mode 100644 index 00000000..fc1bc192 --- /dev/null +++ b/docs/protocols/system/SYS_16_SELF_AWARE_TURBO.md @@ -0,0 +1,262 @@ +# SYS_16: Self-Aware Turbo (SAT) Optimization + +## Version: 1.0 +## Status: PROPOSED +## Created: 2025-12-28 + +--- + +## Problem Statement + +V5 surrogate + L-BFGS failed catastrophically because: +1. MLP predicted WS=280 but actual was WS=376 (30%+ error) +2. L-BFGS descended to regions **outside training distribution** +3. Surrogate had no way to signal uncertainty +4. All L-BFGS solutions converged to the same "fake optimum" + +**Root cause:** The surrogate is overconfident in regions where it has no data. + +--- + +## Solution: Uncertainty-Aware Surrogate with Active Learning + +### Core Principles + +1. **Never trust a point prediction** - Always require uncertainty bounds +2. **High uncertainty = run FEA** - Don't optimize where you don't know +3. **Actively fill gaps** - Prioritize FEA in high-uncertainty regions +4. **Validate gradient solutions** - Check L-BFGS results against FEA before trusting + +--- + +## Architecture + +### 1. Ensemble Surrogate (Epistemic Uncertainty) + +Instead of one MLP, train **N independent models** with different initializations: + +```python +class EnsembleSurrogate: + def __init__(self, n_models=5): + self.models = [MLP() for _ in range(n_models)] + + def predict(self, x): + preds = [m.predict(x) for m in self.models] + mean = np.mean(preds, axis=0) + std = np.std(preds, axis=0) # Epistemic uncertainty + return mean, std + + def is_confident(self, x, threshold=0.1): + mean, std = self.predict(x) + # Confident if std < 10% of mean + return (std / (mean + 1e-6)) < threshold +``` + +**Why this works:** Models trained on different random seeds will agree in well-sampled regions but disagree wildly in extrapolation regions. + +### 2. Distance-Based OOD Detection + +Track training data distribution and flag points that are "too far": + +```python +class OODDetector: + def __init__(self, X_train): + self.X_train = X_train + self.mean = X_train.mean(axis=0) + self.std = X_train.std(axis=0) + # Fit KNN for local density + self.knn = NearestNeighbors(n_neighbors=5) + self.knn.fit(X_train) + + def distance_to_training(self, x): + """Return distance to nearest training points.""" + distances, _ = self.knn.kneighbors(x.reshape(1, -1)) + return distances.mean() + + def is_in_distribution(self, x, threshold=2.0): + """Check if point is within 2 std of training data.""" + z_scores = np.abs((x - self.mean) / (self.std + 1e-6)) + return z_scores.max() < threshold +``` + +### 3. Trust-Region L-BFGS + +Constrain L-BFGS to stay within training distribution: + +```python +def trust_region_lbfgs(surrogate, ood_detector, x0, max_iter=100): + """L-BFGS that respects training data boundaries.""" + + def constrained_objective(x): + # If OOD, return large penalty + if not ood_detector.is_in_distribution(x): + return 1e9 + + mean, std = surrogate.predict(x) + # If uncertain, return upper confidence bound (pessimistic) + if std > 0.1 * mean: + return mean + 2 * std # Be conservative + + return mean + + result = minimize(constrained_objective, x0, method='L-BFGS-B') + return result.x +``` + +### 4. Acquisition Function with Uncertainty + +Use **Expected Improvement with Uncertainty** (like Bayesian Optimization): + +```python +def acquisition_score(x, surrogate, best_so_far): + """Score = potential improvement weighted by confidence.""" + mean, std = surrogate.predict(x) + + # Expected improvement (lower is better for minimization) + improvement = best_so_far - mean + + # Exploration bonus for uncertain regions + exploration = 0.5 * std + + # High score = worth evaluating with FEA + return improvement + exploration + +def select_next_fea_candidates(surrogate, candidates, best_so_far, n=5): + """Select candidates balancing exploitation and exploration.""" + scores = [acquisition_score(c, surrogate, best_so_far) for c in candidates] + + # Pick top candidates by acquisition score + top_indices = np.argsort(scores)[-n:] + return [candidates[i] for i in top_indices] +``` + +--- + +## Algorithm: Self-Aware Turbo (SAT) + +``` +INITIALIZE: + - Load existing FEA data (X_train, Y_train) + - Train ensemble surrogate on data + - Fit OOD detector on X_train + - Set best_ws = min(Y_train) + +PHASE 1: UNCERTAINTY MAPPING (10% of budget) + FOR i in 1..N_mapping: + - Sample random point x + - Get uncertainty: mean, std = surrogate.predict(x) + - If std > threshold: run FEA, add to training data + - Retrain ensemble periodically + + This fills in the "holes" in the surrogate's knowledge. + +PHASE 2: EXPLOITATION WITH VALIDATION (80% of budget) + FOR i in 1..N_exploit: + - Generate 1000 TPE samples + - Filter to keep only confident predictions (std < 10% of mean) + - Filter to keep only in-distribution (OOD check) + - Rank by predicted WS + + - Take top 5 candidates + - Run FEA on all 5 + + - For each FEA result: + - Compare predicted vs actual + - If error > 20%: mark region as "unreliable", force exploration there + - If error < 10%: update best, retrain surrogate + + - Every 10 iterations: retrain ensemble with new data + +PHASE 3: L-BFGS REFINEMENT (10% of budget) + - Only run L-BFGS if ensemble R² > 0.95 on validation set + - Use trust-region L-BFGS (stay within training distribution) + + FOR each L-BFGS solution: + - Check ensemble disagreement + - If models agree (std < 5%): run FEA to validate + - If models disagree: skip, too uncertain + + - Compare L-BFGS prediction vs FEA + - If error > 15%: ABORT L-BFGS phase, return to Phase 2 + - If error < 10%: accept as candidate + +FINAL: + - Return best FEA-validated design + - Report uncertainty bounds for all objectives +``` + +--- + +## Key Differences from V5 + +| Aspect | V5 (Failed) | SAT (Proposed) | +|--------|-------------|----------------| +| **Model** | Single MLP | Ensemble of 5 MLPs | +| **Uncertainty** | None | Ensemble disagreement + OOD detection | +| **L-BFGS** | Trust blindly | Trust-region, validate every step | +| **Extrapolation** | Accept | Reject or penalize | +| **Active learning** | No | Yes - prioritize uncertain regions | +| **Validation** | After L-BFGS | Throughout | + +--- + +## Implementation Checklist + +1. [ ] `EnsembleSurrogate` class with N=5 MLPs +2. [ ] `OODDetector` with KNN + z-score checks +3. [ ] `acquisition_score()` balancing exploitation/exploration +4. [ ] Trust-region L-BFGS with OOD penalties +5. [ ] Automatic retraining when new FEA data arrives +6. [ ] Logging of prediction errors to track surrogate quality +7. [ ] Early abort if L-BFGS predictions consistently wrong + +--- + +## Expected Behavior + +**In well-sampled regions:** +- Ensemble agrees → Low uncertainty → Trust predictions +- L-BFGS finds valid optima → FEA confirms → Success + +**In poorly-sampled regions:** +- Ensemble disagrees → High uncertainty → Run FEA instead +- L-BFGS penalized → Stays in trusted zone → No fake optima + +**At distribution boundaries:** +- OOD detector flags → Reject predictions +- Acquisition prioritizes → Active learning fills gaps + +--- + +## Metrics to Track + +1. **Surrogate R² on validation set** - Target > 0.95 before L-BFGS +2. **Prediction error histogram** - Should be centered at 0 +3. **OOD rejection rate** - How often we refuse to predict +4. **Ensemble disagreement** - Average std across predictions +5. **L-BFGS success rate** - % of L-BFGS solutions that validate + +--- + +## When to Use SAT vs Pure TPE + +| Scenario | Recommendation | +|----------|----------------| +| < 100 existing samples | Pure TPE (not enough for good surrogate) | +| 100-500 samples | SAT Phase 1-2 only (no L-BFGS) | +| > 500 samples | Full SAT with L-BFGS refinement | +| High-dimensional (>20 params) | Pure TPE (curse of dimensionality) | +| Noisy FEA | Pure TPE (surrogates struggle with noise) | + +--- + +## References + +- Gaussian Process literature on uncertainty quantification +- Deep Ensembles: Lakshminarayanan et al. (2017) +- Bayesian Optimization with Expected Improvement +- Trust-region methods for constrained optimization + +--- + +*The key insight: A surrogate that knows when it doesn't know is infinitely more valuable than one that's confidently wrong.* diff --git a/knowledge_base/lac/session_insights/failure.jsonl b/knowledge_base/lac/session_insights/failure.jsonl index 6d961cad..f4c3a650 100644 --- a/knowledge_base/lac/session_insights/failure.jsonl +++ b/knowledge_base/lac/session_insights/failure.jsonl @@ -3,3 +3,5 @@ {"timestamp":"2025-12-19T10:00:00","category":"workaround","context":"NX journal execution via cmd /c with environment variables fails silently or produces garbled output. Multiple attempts with cmd /c SET and && chaining failed to capture run_journal.exe output.","insight":"CRITICAL WORKAROUND: When executing NX journals from Claude Code on Windows, use PowerShell with [Environment]::SetEnvironmentVariable() method instead of cmd /c or $env: syntax. The correct pattern is: powershell -Command \"[Environment]::SetEnvironmentVariable('SPLM_LICENSE_SERVER', '28000@dalidou;28000@100.80.199.40', 'Process'); & 'C:\\Program Files\\Siemens\\DesigncenterNX2512\\NXBIN\\run_journal.exe' 'journal.py' -args 'arg1' 'arg2' 2>&1\". The $env: syntax gets corrupted when passed through bash (colon gets interpreted). The cmd /c SET syntax often fails to capture output. This PowerShell pattern reliably sets license server and captures all output.","confidence":1.0,"tags":["nx","powershell","run_journal","license-server","windows","cmd-workaround"],"severity":"high","rule":"ALWAYS use PowerShell with [Environment]::SetEnvironmentVariable() for NX journal execution. NEVER use cmd /c SET or $env: syntax for setting SPLM_LICENSE_SERVER."} {"timestamp":"2025-12-19T15:30:00","category":"failure","context":"CMA-ES optimization V7 started with random sample instead of baseline. First trial had whiffle_min=45.73 instead of baseline 62.75, resulting in WS=329 instead of expected ~281.","insight":"CMA-ES with Optuna CmaEsSampler does NOT evaluate x0 (baseline) first - it samples AROUND x0 with sigma0 step size. The x0 parameter only sets the CENTER of the initial sampling distribution, not the first trial. To ensure baseline is evaluated first, use study.enqueue_trial(x0) after creating the study. This is critical for refinement studies where you need to compare against a known-good baseline. Pattern: if len(study.trials) == 0: study.enqueue_trial(x0)","confidence":1.0,"tags":["cma-es","optuna","baseline","x0","enqueue","optimization"],"severity":"high","rule":"When using CmaEsSampler with a known baseline, ALWAYS enqueue the baseline as trial 0 using study.enqueue_trial(x0). The x0 parameter alone does NOT guarantee baseline evaluation."} {"timestamp":"2025-12-22T14:00:00","category":"failure","context":"V10 mirror optimization reported impossibly good relative WFE values (40-20=1.99nm instead of ~6nm, 60-20=6.82nm instead of ~13nm). User noticed results were 'too good to be true'.","insight":"CRITICAL BUG IN RELATIVE WFE CALCULATION: The V10 run_optimization.py computed relative WFE as abs(RMS_target - RMS_ref) instead of RMS(WFE_target - WFE_ref). This is mathematically WRONG because |RMS(A) - RMS(B)| ≠ RMS(A - B). The correct approach is to compute the node-by-node WFE difference FIRST, then fit Zernike to the difference field, then compute RMS. The bug gave values 3-4x lower than correct values because the 20° reference had HIGHER absolute WFE than 40°/60°, so the subtraction gave negative values, and abs() hid the problem. The fix is to use extractor.extract_relative() which correctly computes node-by-node differences. Both ZernikeExtractor and ZernikeOPDExtractor now have extract_relative() methods.","confidence":1.0,"tags":["zernike","wfe","relative-wfe","extract_relative","critical-bug","v10"],"severity":"critical","rule":"NEVER compute relative WFE as abs(RMS_target - RMS_ref). ALWAYS use extract_relative() which computes RMS(WFE_target - WFE_ref) by doing node-by-node subtraction first, then Zernike fitting, then RMS."} +{"timestamp":"2025-12-28T17:30:00","category":"failure","context":"V5 turbo optimization created from scratch instead of copying V4. Multiple critical components were missing or wrong: no license server, wrong extraction keys (filtered_rms_nm vs relative_filtered_rms_nm), wrong mfg_90 key, missing figure_path parameter, incomplete version regex.","insight":"STUDY DERIVATION FAILURE: When creating a new study version (V5 from V4), NEVER rewrite the run_optimization.py from scratch. ALWAYS copy the working version first, then add/modify only the new feature (e.g., L-BFGS polish). Rewriting caused 5 independent bugs: (1) missing LICENSE_SERVER setup, (2) wrong extraction key filtered_rms_nm instead of relative_filtered_rms_nm, (3) wrong mfg_90 key, (4) missing figure_path=None in extractor call, (5) incomplete version regex missing DesigncenterNX pattern. The FEA/extraction pipeline is PROVEN CODE - never rewrite it. Only add new optimization strategies as modules on top.","confidence":1.0,"tags":["study-creation","copy-dont-rewrite","extraction","license-server","v5","critical"],"severity":"critical","rule":"When deriving a new study version, COPY the entire working run_optimization.py first. Add new features as ADDITIONS, not rewrites. The FEA pipeline (license, NXSolver setup, extraction) is proven - never rewrite it."} +{"timestamp":"2025-12-28T21:30:00","category":"failure","context":"V5 flat back turbo optimization with MLP surrogate + L-BFGS polish. Surrogate predicted WS~280 but actual FEA gave WS~365-377. Error of 85-96 (30%+ relative error). All L-BFGS solutions converged to same fake optimum that didn't exist in reality.","insight":"SURROGATE + L-BFGS FAILURE MODE: Gradient-based optimization on MLP surrogates finds 'fake optima' that don't exist in real FEA. The surrogate has smooth gradients everywhere, but L-BFGS descends to regions OUTSIDE the training distribution where predictions are wildly wrong. V5 results: (1) Best TPE trial: WS=290.18, (2) Best L-BFGS trial: WS=325.27, (3) Worst L-BFGS trials: WS=376.52. The fancy L-BFGS polish made results WORSE than random TPE. Key issues: (a) No uncertainty quantification - can't detect out-of-distribution, (b) No mass constraint in surrogate - L-BFGS finds infeasible designs (122-124kg vs 120kg limit), (c) L-BFGS converges to same bad point from multiple starting locations (trials 31-44 all gave WS=376.52).","confidence":1.0,"tags":["surrogate","mlp","lbfgs","gradient-descent","fake-optima","out-of-distribution","v5","turbo"],"severity":"critical","rule":"NEVER trust gradient descent on surrogates without: (1) Uncertainty quantification to reject OOD predictions, (2) Mass/constraint prediction to enforce feasibility, (3) Trust-region to stay within training distribution. Pure TPE with real FEA often beats surrogate+gradient methods."} diff --git a/knowledge_base/lac/session_insights/success_pattern.jsonl b/knowledge_base/lac/session_insights/success_pattern.jsonl index 194d301b..1a56c1d4 100644 --- a/knowledge_base/lac/session_insights/success_pattern.jsonl +++ b/knowledge_base/lac/session_insights/success_pattern.jsonl @@ -5,3 +5,5 @@ {"timestamp": "2025-12-28T10:15:00", "category": "success_pattern", "context": "Unified trial management with TrialManager and DashboardDB", "insight": "TRIAL MANAGEMENT PATTERN: Use TrialManager for consistent trial_NNNN naming across all optimization methods (Optuna, Turbo, GNN, manual). Key principles: (1) Trial numbers NEVER reset (monotonic), (2) Folders NEVER get overwritten, (3) Database always synced with filesystem, (4) Surrogate predictions are NOT trials - only FEA results. DashboardDB provides Optuna-compatible schema for dashboard integration. Path: optimization_engine/utils/trial_manager.py", "confidence": 0.95, "tags": ["trial_manager", "dashboard_db", "optuna", "trial_naming", "turbo"]} {"timestamp": "2025-12-28T10:15:00", "category": "success_pattern", "context": "GNN Turbo training data loading from multiple studies", "insight": "MULTI-STUDY TRAINING: When loading training data from multiple prior studies for GNN surrogate training, param names may have unit prefixes like '[mm]rib_thickness' or '[Degrees]angle'. Strip prefixes: if ']' in name: name = name.split(']', 1)[1]. Also, objective attribute names vary between studies (rel_filtered_rms_40_vs_20 vs obj_rel_filtered_rms_40_vs_20) - use fallback chain with 'or'. V5 successfully trained on 316 samples (V3: 297, V4: 19) with R²=[0.94, 0.94, 0.89, 0.95].", "confidence": 0.9, "tags": ["gnn", "turbo", "training_data", "multi_study", "param_naming"]} {"timestamp": "2025-12-28T12:28:04.706624", "category": "success_pattern", "context": "Implemented L-BFGS gradient optimizer for surrogate polish phase", "insight": "L-BFGS on trained MLP surrogates provides 100-1000x faster convergence than derivative-free methods (TPE, CMA-ES) for local refinement. Key: use multi-start from top FEA candidates, not random initialization. Integration: GradientOptimizer class in optimization_engine/gradient_optimizer.py.", "confidence": 0.9, "tags": ["optimization", "lbfgs", "surrogate", "gradient", "polish"]} +{"timestamp": "2025-12-29T09:30:00", "category": "success_pattern", "context": "V6 pure TPE outperformed V5 surrogate+L-BFGS by 22%", "insight": "SIMPLE BEATS COMPLEX: V6 Pure TPE achieved WS=225.41 vs V5's WS=290.18 (22.3% better). Key insight: surrogates fail when gradient methods descend to OOD regions. Fix: EnsembleSurrogate with (1) N=5 MLPs for disagreement-based uncertainty, (2) OODDetector with KNN+z-score, (3) acquisition_score balancing exploitation+exploration, (4) trust-region L-BFGS that stays in training distribution. Never trust point predictions - always require uncertainty bounds. Protocol: SYS_16_SELF_AWARE_TURBO.md. Code: optimization_engine/surrogates/ensemble_surrogate.py", "confidence": 1.0, "tags": ["ensemble", "uncertainty", "ood", "surrogate", "v6", "tpe", "self-aware"]} +{"timestamp": "2025-12-29T09:47:47.612485", "category": "success_pattern", "context": "Disk space optimization for FEA studies", "insight": "Per-trial FEA files are ~150MB but only OP2+JSON (~70MB) are essential. PRT/FEM/SIM/DAT are copies of master files and can be deleted after study completion. Archive to dalidou server for long-term storage.", "confidence": 0.95, "tags": ["disk_optimization", "archival", "study_management", "dalidou"], "related_files": ["optimization_engine/utils/study_archiver.py", "docs/protocols/operations/OP_07_DISK_OPTIMIZATION.md"]} diff --git a/optimization_engine/nx_solver.py b/optimization_engine/nx_solver.py index def9784d..60cd7fe8 100644 --- a/optimization_engine/nx_solver.py +++ b/optimization_engine/nx_solver.py @@ -242,19 +242,28 @@ class NXSolver: Format: [unit]name=value Example: [mm]whiffle_min=42.5 """ - # Default unit mapping (could be extended or made configurable) + # Default unit mapping - MUST match NX model expression units exactly + # Verified against working turbo V1 runs UNIT_MAPPING = { # Length parameters (mm) 'whiffle_min': 'mm', 'whiffle_triangle_closeness': 'mm', 'inner_circular_rib_dia': 'mm', 'outer_circular_rib_offset_from_outer': 'mm', + 'Pocket_Radius': 'mm', + 'center_thickness': 'mm', + # Lateral pivot/closeness - mm in NX model (verified from V1) 'lateral_outer_pivot': 'mm', 'lateral_inner_pivot': 'mm', 'lateral_middle_pivot': 'mm', 'lateral_closeness': 'mm', - # Angle parameters (degrees) - 'whiffle_outer_to_vertical': 'Degrees', + # Rib/face thickness parameters (mm) + 'rib_thickness': 'mm', + 'ribs_circular_thk': 'mm', + 'rib_thickness_lateral_truss': 'mm', + 'mirror_face_thickness': 'mm', + # Angle parameters (Degrees) - verified from working V1 runs + 'whiffle_outer_to_vertical': 'Degrees', # NX expects Degrees (verified V1) 'lateral_inner_angle': 'Degrees', 'lateral_outer_angle': 'Degrees', 'blank_backface_angle': 'Degrees', diff --git a/optimization_engine/surrogates/__init__.py b/optimization_engine/surrogates/__init__.py new file mode 100644 index 00000000..52622c1f --- /dev/null +++ b/optimization_engine/surrogates/__init__.py @@ -0,0 +1,19 @@ +""" +Surrogate models for FEA acceleration. + +Available surrogates: +- EnsembleSurrogate: Multiple MLPs with uncertainty quantification +- OODDetector: Out-of-distribution detection +""" + +from .ensemble_surrogate import ( + EnsembleSurrogate, + OODDetector, + create_and_train_ensemble +) + +__all__ = [ + 'EnsembleSurrogate', + 'OODDetector', + 'create_and_train_ensemble' +] diff --git a/optimization_engine/surrogates/ensemble_surrogate.py b/optimization_engine/surrogates/ensemble_surrogate.py new file mode 100644 index 00000000..4243f249 --- /dev/null +++ b/optimization_engine/surrogates/ensemble_surrogate.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +""" +Ensemble Surrogate with Uncertainty Quantification + +Addresses the V5 failure mode where single MLPs gave overconfident predictions +in out-of-distribution regions, leading L-BFGS to fake optima. + +Key features: +1. Ensemble of N MLPs - disagreement = uncertainty +2. OOD detection - reject predictions far from training data +3. Confidence bounds - never trust point predictions alone +4. Active learning - prioritize FEA in uncertain regions + +Author: Atomizer +Created: 2025-12-28 +""" + +import numpy as np +from typing import Tuple, List, Dict, Optional +from pathlib import Path +import json +import logging + +logger = logging.getLogger(__name__) + +try: + import torch + import torch.nn as nn + HAS_TORCH = True +except ImportError: + HAS_TORCH = False + logger.warning("PyTorch not available - ensemble features limited") + +from sklearn.neighbors import NearestNeighbors + + +class MLP(nn.Module): + """Single MLP for ensemble.""" + + def __init__(self, input_dim: int, output_dim: int, hidden_dims: List[int] = None): + super().__init__() + hidden_dims = hidden_dims or [64, 32] + + layers = [] + in_dim = input_dim + for h_dim in hidden_dims: + layers.append(nn.Linear(in_dim, h_dim)) + layers.append(nn.ReLU()) + layers.append(nn.Dropout(0.1)) + in_dim = h_dim + layers.append(nn.Linear(in_dim, output_dim)) + + self.net = nn.Sequential(*layers) + + def forward(self, x): + return self.net(x) + + +class OODDetector: + """ + Out-of-Distribution detector using multiple methods: + 1. Z-score check (is input within N std of training mean) + 2. KNN distance (is input close to training points) + """ + + def __init__(self, X_train: np.ndarray, z_threshold: float = 3.0, knn_k: int = 5): + self.X_train = X_train + self.z_threshold = z_threshold + self.knn_k = knn_k + + # Compute training statistics + self.mean = X_train.mean(axis=0) + self.std = X_train.std(axis=0) + 1e-8 + + # Fit KNN for local density estimation + self.knn = NearestNeighbors(n_neighbors=min(knn_k, len(X_train))) + self.knn.fit(X_train) + + # Compute typical KNN distances in training set + train_distances, _ = self.knn.kneighbors(X_train) + self.typical_knn_dist = np.median(train_distances.mean(axis=1)) + + logger.info(f"[OOD] Initialized with {len(X_train)} training points") + logger.info(f"[OOD] Typical KNN distance: {self.typical_knn_dist:.4f}") + + def z_score_check(self, x: np.ndarray) -> Tuple[bool, float]: + """Check if point is within z_threshold std of training mean.""" + x = np.atleast_2d(x) + z_scores = np.abs((x - self.mean) / self.std) + max_z = z_scores.max(axis=1) + is_ok = max_z < self.z_threshold + return is_ok[0] if len(is_ok) == 1 else is_ok, max_z[0] if len(max_z) == 1 else max_z + + def knn_distance_check(self, x: np.ndarray) -> Tuple[bool, float]: + """Check if point is close enough to training data.""" + x = np.atleast_2d(x) + distances, _ = self.knn.kneighbors(x) + avg_dist = distances.mean(axis=1) + # Allow up to 3x typical distance + is_ok = avg_dist < 3 * self.typical_knn_dist + return is_ok[0] if len(is_ok) == 1 else is_ok, avg_dist[0] if len(avg_dist) == 1 else avg_dist + + def is_in_distribution(self, x: np.ndarray) -> Tuple[bool, Dict]: + """Combined OOD check.""" + z_ok, z_score = self.z_score_check(x) + knn_ok, knn_dist = self.knn_distance_check(x) + + is_ok = z_ok and knn_ok + details = { + 'z_score': float(z_score), + 'z_ok': bool(z_ok), + 'knn_dist': float(knn_dist), + 'knn_ok': bool(knn_ok), + 'in_distribution': bool(is_ok) + } + + return is_ok, details + + +class EnsembleSurrogate: + """ + Ensemble of MLPs with uncertainty quantification. + + Key insight: Models trained with different random seeds will agree + in well-sampled regions but disagree in extrapolation regions. + Disagreement = epistemic uncertainty. + """ + + def __init__( + self, + input_dim: int, + output_dim: int, + n_models: int = 5, + hidden_dims: List[int] = None, + device: str = 'auto' + ): + self.input_dim = input_dim + self.output_dim = output_dim + self.n_models = n_models + self.hidden_dims = hidden_dims or [64, 32] + + if device == 'auto': + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + else: + self.device = torch.device(device) + + # Create ensemble + self.models = [ + MLP(input_dim, output_dim, hidden_dims).to(self.device) + for _ in range(n_models) + ] + + # Normalization stats + self.x_mean = None + self.x_std = None + self.y_mean = None + self.y_std = None + + # OOD detector + self.ood_detector = None + + # Training state + self.is_trained = False + + logger.info(f"[ENSEMBLE] Created {n_models} MLPs on {self.device}") + + def train( + self, + X: np.ndarray, + Y: np.ndarray, + epochs: int = 500, + lr: float = 0.001, + val_split: float = 0.1, + patience: int = 50 + ) -> Dict: + """Train all models in ensemble with different random seeds.""" + + # Compute normalization + self.x_mean = X.mean(axis=0) + self.x_std = X.std(axis=0) + 1e-8 + self.y_mean = Y.mean(axis=0) + self.y_std = Y.std(axis=0) + 1e-8 + + X_norm = (X - self.x_mean) / self.x_std + Y_norm = (Y - self.y_mean) / self.y_std + + # Split data + n_val = max(int(len(X) * val_split), 5) + indices = np.random.permutation(len(X)) + val_idx, train_idx = indices[:n_val], indices[n_val:] + + X_train, Y_train = X_norm[train_idx], Y_norm[train_idx] + X_val, Y_val = X_norm[val_idx], Y_norm[val_idx] + + # Convert to tensors + X_t = torch.FloatTensor(X_train).to(self.device) + Y_t = torch.FloatTensor(Y_train).to(self.device) + X_v = torch.FloatTensor(X_val).to(self.device) + Y_v = torch.FloatTensor(Y_val).to(self.device) + + # Train each model with different seed + all_val_losses = [] + for i, model in enumerate(self.models): + torch.manual_seed(42 + i * 1000) # Different init per model + np.random.seed(42 + i * 1000) + + optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4) + criterion = nn.MSELoss() + + best_val_loss = float('inf') + patience_counter = 0 + best_state = None + + for epoch in range(epochs): + # Train + model.train() + optimizer.zero_grad() + pred = model(X_t) + loss = criterion(pred, Y_t) + loss.backward() + optimizer.step() + + # Validate + model.eval() + with torch.no_grad(): + val_pred = model(X_v) + val_loss = criterion(val_pred, Y_v).item() + + # Early stopping + if val_loss < best_val_loss: + best_val_loss = val_loss + patience_counter = 0 + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + else: + patience_counter += 1 + if patience_counter >= patience: + break + + # Restore best + if best_state: + model.load_state_dict(best_state) + model.to(self.device) + + all_val_losses.append(best_val_loss) + logger.info(f"[ENSEMBLE] Model {i+1}/{self.n_models} trained, val_loss={best_val_loss:.4f}") + + # Initialize OOD detector + self.ood_detector = OODDetector(X_norm) + + self.is_trained = True + + # Compute ensemble metrics + metrics = self._compute_metrics(X_val, Y_val) + metrics['val_losses'] = all_val_losses + + return metrics + + def _compute_metrics(self, X_val: np.ndarray, Y_val: np.ndarray) -> Dict: + """Compute R², MAE, and ensemble disagreement on validation set.""" + mean, std = self.predict_normalized(X_val) + + # R² for each output + ss_res = np.sum((Y_val - mean) ** 2, axis=0) + ss_tot = np.sum((Y_val - Y_val.mean(axis=0)) ** 2, axis=0) + r2 = 1 - ss_res / (ss_tot + 1e-8) + + # MAE + mae = np.abs(Y_val - mean).mean(axis=0) + + # Average ensemble disagreement + avg_std = std.mean() + + return { + 'r2': r2.tolist(), + 'mae': mae.tolist(), + 'avg_ensemble_std': float(avg_std), + 'n_val': len(X_val) + } + + def predict_normalized(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Predict on normalized inputs, return normalized outputs.""" + X = np.atleast_2d(X) + X_t = torch.FloatTensor(X).to(self.device) + + preds = [] + for model in self.models: + model.eval() + with torch.no_grad(): + pred = model(X_t).cpu().numpy() + preds.append(pred) + + preds = np.array(preds) # (n_models, n_samples, n_outputs) + mean = preds.mean(axis=0) + std = preds.std(axis=0) + + return mean, std + + def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """ + Predict with uncertainty. + + Returns: + mean: (n_samples, n_outputs) predicted values + std: (n_samples, n_outputs) uncertainty (ensemble disagreement) + """ + X = np.atleast_2d(X) + + # Normalize input + X_norm = (X - self.x_mean) / self.x_std + + # Get predictions + mean_norm, std_norm = self.predict_normalized(X_norm) + + # Denormalize + mean = mean_norm * self.y_std + self.y_mean + std = std_norm * self.y_std # Std scales with y_std + + return mean, std + + def predict_with_confidence(self, X: np.ndarray) -> Dict: + """ + Full prediction with confidence assessment. + + Returns dict with: + - mean: predicted values + - std: uncertainty + - confidence: 0-1 score (higher = more reliable) + - in_distribution: OOD check result + - recommendation: 'trust', 'verify', or 'reject' + """ + X = np.atleast_2d(X) + + mean, std = self.predict(X) + + # OOD check + X_norm = (X - self.x_mean) / self.x_std + ood_results = [self.ood_detector.is_in_distribution(x) for x in X_norm] + in_distribution = [r[0] for r in ood_results] + + # Compute confidence score (0 = no confidence, 1 = high confidence) + # Based on: relative std (lower = better) and OOD (in = better) + relative_std = std / (np.abs(mean) + 1e-6) + avg_rel_std = relative_std.mean(axis=1) + + confidence = np.zeros(len(X)) + for i in range(len(X)): + if not in_distribution[i]: + confidence[i] = 0.0 # OOD = no confidence + elif avg_rel_std[i] > 0.3: + confidence[i] = 0.2 # High uncertainty + elif avg_rel_std[i] > 0.1: + confidence[i] = 0.5 # Medium uncertainty + else: + confidence[i] = 0.9 # Low uncertainty + + # Recommendations + recommendations = [] + for i in range(len(X)): + if confidence[i] >= 0.7: + recommendations.append('trust') + elif confidence[i] >= 0.3: + recommendations.append('verify') # Run FEA to check + else: + recommendations.append('reject') # Don't use, run FEA instead + + return { + 'mean': mean, + 'std': std, + 'confidence': confidence, + 'in_distribution': in_distribution, + 'recommendation': recommendations + } + + def acquisition_score(self, X: np.ndarray, best_so_far: float, xi: float = 0.01) -> np.ndarray: + """ + Expected Improvement acquisition function. + + High score = worth running FEA (either promising or uncertain). + + Args: + X: candidate points + best_so_far: current best objective value + xi: exploration-exploitation tradeoff (higher = more exploration) + + Returns: + scores: acquisition score per point + """ + X = np.atleast_2d(X) + mean, std = self.predict(X) + + # For minimization: improvement = best - predicted + # Take first objective (weighted sum) for acquisition + if mean.ndim > 1: + mean = mean[:, 0] + std = std[:, 0] + + improvement = best_so_far - mean + + # Expected improvement with exploration bonus + # Higher std = more exploration value + z = improvement / (std + 1e-8) + + # Simple acquisition: exploitation + exploration + scores = improvement + xi * std + + # Penalize OOD points + X_norm = (X - self.x_mean) / self.x_std + for i, x in enumerate(X_norm): + is_ok, _ = self.ood_detector.is_in_distribution(x) + if not is_ok: + scores[i] *= 0.1 # Heavy penalty for OOD + + return scores + + def select_candidates_for_fea( + self, + candidates: np.ndarray, + best_so_far: float, + n_select: int = 5, + diversity_weight: float = 0.3 + ) -> Tuple[np.ndarray, np.ndarray]: + """ + Select diverse, high-acquisition candidates for FEA validation. + + Balances: + 1. High acquisition score (exploitation + exploration) + 2. Diversity (don't cluster all candidates together) + 3. In-distribution (avoid OOD predictions) + + Returns: + selected: indices of selected candidates + scores: acquisition scores + """ + scores = self.acquisition_score(candidates, best_so_far) + + # Greedy selection with diversity + selected = [] + remaining = list(range(len(candidates))) + + while len(selected) < n_select and remaining: + if not selected: + # First: pick highest score + best_idx = max(remaining, key=lambda i: scores[i]) + else: + # Later: balance score with distance to selected + def combined_score(i): + # Min distance to already selected + min_dist = min( + np.linalg.norm(candidates[i] - candidates[j]) + for j in selected + ) + # Combine acquisition + diversity + return scores[i] + diversity_weight * min_dist + + best_idx = max(remaining, key=combined_score) + + selected.append(best_idx) + remaining.remove(best_idx) + + return np.array(selected), scores[selected] + + def save(self, path: Path): + """Save ensemble to disk.""" + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + + # Save each model + for i, model in enumerate(self.models): + torch.save(model.state_dict(), path / f"model_{i}.pt") + + # Save normalization stats and config + config = { + 'input_dim': self.input_dim, + 'output_dim': self.output_dim, + 'n_models': self.n_models, + 'hidden_dims': self.hidden_dims, + 'x_mean': self.x_mean.tolist() if self.x_mean is not None else None, + 'x_std': self.x_std.tolist() if self.x_std is not None else None, + 'y_mean': self.y_mean.tolist() if self.y_mean is not None else None, + 'y_std': self.y_std.tolist() if self.y_std is not None else None, + } + with open(path / "config.json", 'w') as f: + json.dump(config, f, indent=2) + + logger.info(f"[ENSEMBLE] Saved to {path}") + + @classmethod + def load(cls, path: Path, device: str = 'auto') -> 'EnsembleSurrogate': + """Load ensemble from disk.""" + path = Path(path) + + with open(path / "config.json") as f: + config = json.load(f) + + surrogate = cls( + input_dim=config['input_dim'], + output_dim=config['output_dim'], + n_models=config['n_models'], + hidden_dims=config['hidden_dims'], + device=device + ) + + # Load normalization + surrogate.x_mean = np.array(config['x_mean']) if config['x_mean'] else None + surrogate.x_std = np.array(config['x_std']) if config['x_std'] else None + surrogate.y_mean = np.array(config['y_mean']) if config['y_mean'] else None + surrogate.y_std = np.array(config['y_std']) if config['y_std'] else None + + # Load models + for i, model in enumerate(surrogate.models): + model.load_state_dict(torch.load(path / f"model_{i}.pt", map_location=surrogate.device)) + model.to(surrogate.device) + + surrogate.is_trained = True + logger.info(f"[ENSEMBLE] Loaded from {path}") + + return surrogate + + +# Convenience function for quick usage +def create_and_train_ensemble( + X: np.ndarray, + Y: np.ndarray, + n_models: int = 5, + epochs: int = 500 +) -> EnsembleSurrogate: + """Create and train an ensemble surrogate.""" + surrogate = EnsembleSurrogate( + input_dim=X.shape[1], + output_dim=Y.shape[1] if Y.ndim > 1 else 1, + n_models=n_models + ) + + if Y.ndim == 1: + Y = Y.reshape(-1, 1) + + metrics = surrogate.train(X, Y, epochs=epochs) + logger.info(f"[ENSEMBLE] Training complete: R²={metrics['r2']}, avg_std={metrics['avg_ensemble_std']:.4f}") + + return surrogate diff --git a/optimization_engine/utils/nx_session_manager.py b/optimization_engine/utils/nx_session_manager.py index 63c73b96..58635040 100644 --- a/optimization_engine/utils/nx_session_manager.py +++ b/optimization_engine/utils/nx_session_manager.py @@ -24,6 +24,7 @@ SESSION_LOCK_DIR = Path(os.environ.get('TEMP', '/tmp')) / 'atomizer_nx_sessions' # Default NX installation paths (in order of preference) DEFAULT_NX_PATHS = [ + Path(r"C:\Program Files\Siemens\DesigncenterNX2512\NXBIN\ugraf.exe"), # DesignCenter (preferred) Path(r"C:\Program Files\Siemens\NX2506\NXBIN\ugraf.exe"), Path(r"C:\Program Files\Siemens\NX2412\NXBIN\ugraf.exe"), Path(r"C:\Program Files\Siemens\Simcenter3D_2506\NXBIN\ugraf.exe"), diff --git a/optimization_engine/utils/study_archiver.py b/optimization_engine/utils/study_archiver.py new file mode 100644 index 00000000..a1b6f40d --- /dev/null +++ b/optimization_engine/utils/study_archiver.py @@ -0,0 +1,438 @@ +""" +Study Archiver - Disk Space Optimization for Atomizer Studies + +This module provides utilities for: +1. Cleaning up completed studies (removing regenerable files) +2. Archiving studies to remote storage (dalidou server) +3. Restoring archived studies on-demand + +Usage: + # Cleanup a completed study (keep only essential files) + python -m optimization_engine.utils.study_archiver cleanup studies/M1_Mirror/m1_mirror_V12 + + # Archive to remote server + python -m optimization_engine.utils.study_archiver archive studies/M1_Mirror/m1_mirror_V12 + + # Restore from remote + python -m optimization_engine.utils.study_archiver restore m1_mirror_V12 + + # Show disk usage analysis + python -m optimization_engine.utils.study_archiver analyze studies/M1_Mirror +""" + +import os +import json +import shutil +import tarfile +import subprocess +from pathlib import Path +from datetime import datetime +from typing import Optional, Dict, List, Tuple +import logging + +logger = logging.getLogger(__name__) + +# Configuration +REMOTE_CONFIG = { + "host": "192.168.86.50", # Local WiFi + "host_tailscale": "100.80.199.40", # Remote via Tailscale + "user": "papa", + "archive_path": "/srv/storage/atomizer-archive", + "ssh_port": 22, +} + +# Files to KEEP per trial (essential for analysis) +ESSENTIAL_EXTENSIONS = { + '.op2', # Nastran binary results (Zernike extraction) + '.json', # Parameters, results, metadata + '.npz', # Pre-computed Zernike coefficients + '.html', # Generated reports + '.png', # Visualization images + '.csv', # Exported data +} + +# Files to DELETE per trial (regenerable from master + params) +DELETABLE_EXTENSIONS = { + '.prt', # NX part files (copy of master) + '.fem', # FEM mesh files (copy of master) + '.sim', # Simulation files (copy of master) + '.afm', # Assembly FEM files + '.dat', # Solver input deck (can regenerate) + '.f04', # Nastran output log + '.f06', # Nastran printed output + '.log', # Generic log files + '.diag', # Diagnostic files + '.txt', # Temp text files + '.exp', # Expression files + '.bak', # Backup files +} + +# Folders to always keep entirely +KEEP_FOLDERS = { + '1_setup', # Master model files (source of truth) + '3_results', # Final results, database, reports + 'best_design_archive', # Archived best designs +} + + +def analyze_study(study_path: Path) -> Dict: + """Analyze disk usage of a study folder.""" + study_path = Path(study_path) + + analysis = { + "study_name": study_path.name, + "total_size_bytes": 0, + "by_extension": {}, + "by_folder": {}, + "essential_size": 0, + "deletable_size": 0, + "trial_count": 0, + } + + for f in study_path.rglob("*"): + if f.is_file(): + sz = f.stat().st_size + ext = f.suffix.lower() + + analysis["total_size_bytes"] += sz + analysis["by_extension"][ext] = analysis["by_extension"].get(ext, 0) + sz + + # Categorize by folder + rel_parts = f.relative_to(study_path).parts + if rel_parts: + folder = rel_parts[0] + analysis["by_folder"][folder] = analysis["by_folder"].get(folder, 0) + sz + + # Essential vs deletable + if ext in ESSENTIAL_EXTENSIONS: + analysis["essential_size"] += sz + elif ext in DELETABLE_EXTENSIONS: + analysis["deletable_size"] += sz + + # Count trials + iterations_dir = study_path / "2_iterations" + if iterations_dir.exists(): + analysis["trial_count"] = len([ + d for d in iterations_dir.iterdir() + if d.is_dir() and (d.name.startswith("trial_") or d.name.startswith("iter")) + ]) + + return analysis + + +def print_analysis(analysis: Dict): + """Print formatted analysis results.""" + total_gb = analysis["total_size_bytes"] / 1e9 + essential_gb = analysis["essential_size"] / 1e9 + deletable_gb = analysis["deletable_size"] / 1e9 + + print(f"\n{'='*60}") + print(f"Study: {analysis['study_name']}") + print(f"{'='*60}") + print(f"Total size: {total_gb:8.2f} GB") + print(f"Trials: {analysis['trial_count']:8d}") + print(f"Essential: {essential_gb:8.2f} GB ({100*essential_gb/total_gb:.1f}%)") + print(f"Deletable: {deletable_gb:8.2f} GB ({100*deletable_gb/total_gb:.1f}%)") + print(f"Potential save: {deletable_gb:8.2f} GB") + + print(f"\nBy folder:") + for folder, size in sorted(analysis["by_folder"].items(), key=lambda x: -x[1]): + print(f" {folder:25} {size/1e9:8.2f} GB") + + print(f"\nTop extensions:") + for ext, size in sorted(analysis["by_extension"].items(), key=lambda x: -x[1])[:10]: + status = "[KEEP]" if ext in ESSENTIAL_EXTENSIONS else "[DEL?]" if ext in DELETABLE_EXTENSIONS else "[ ]" + print(f" {status} {ext:10} {size/1e9:8.2f} GB") + + +def cleanup_study(study_path: Path, dry_run: bool = True) -> Tuple[int, int]: + """ + Clean up a completed study by removing regenerable files from trial folders. + + Args: + study_path: Path to study folder + dry_run: If True, only report what would be deleted + + Returns: + (files_deleted, bytes_freed) + """ + study_path = Path(study_path) + iterations_dir = study_path / "2_iterations" + + if not iterations_dir.exists(): + logger.warning(f"No iterations folder found in {study_path}") + return 0, 0 + + files_to_delete = [] + bytes_to_free = 0 + + # Find all deletable files in trial folders + for trial_dir in iterations_dir.iterdir(): + if not trial_dir.is_dir(): + continue + + for f in trial_dir.iterdir(): + if f.is_file() and f.suffix.lower() in DELETABLE_EXTENSIONS: + files_to_delete.append(f) + bytes_to_free += f.stat().st_size + + if dry_run: + print(f"\n[DRY RUN] Would delete {len(files_to_delete)} files, freeing {bytes_to_free/1e9:.2f} GB") + print("\nSample files to delete:") + for f in files_to_delete[:10]: + print(f" {f.relative_to(study_path)}") + if len(files_to_delete) > 10: + print(f" ... and {len(files_to_delete) - 10} more") + return 0, 0 + + # Actually delete + deleted = 0 + freed = 0 + for f in files_to_delete: + try: + sz = f.stat().st_size + f.unlink() + deleted += 1 + freed += sz + except Exception as e: + logger.error(f"Failed to delete {f}: {e}") + + print(f"Deleted {deleted} files, freed {freed/1e9:.2f} GB") + return deleted, freed + + +def archive_to_remote( + study_path: Path, + use_tailscale: bool = False, + dry_run: bool = True +) -> bool: + """ + Archive a study to the remote dalidou server. + + Args: + study_path: Path to study folder + use_tailscale: Use Tailscale IP (for remote access) + dry_run: If True, only report what would be done + + Returns: + True if successful + """ + study_path = Path(study_path) + study_name = study_path.name + + host = REMOTE_CONFIG["host_tailscale"] if use_tailscale else REMOTE_CONFIG["host"] + user = REMOTE_CONFIG["user"] + remote_path = REMOTE_CONFIG["archive_path"] + + # Create compressed archive locally first + archive_name = f"{study_name}_{datetime.now().strftime('%Y%m%d')}.tar.gz" + local_archive = study_path.parent / archive_name + + if dry_run: + print(f"\n[DRY RUN] Would archive {study_name}") + print(f" 1. Create {archive_name}") + print(f" 2. Upload to {user}@{host}:{remote_path}/") + print(f" 3. Delete local archive") + return True + + print(f"Creating archive: {archive_name}") + with tarfile.open(local_archive, "w:gz") as tar: + tar.add(study_path, arcname=study_name) + + archive_size = local_archive.stat().st_size + print(f"Archive size: {archive_size/1e9:.2f} GB") + + # Upload via rsync (more reliable than scp for large files) + print(f"Uploading to {host}...") + + # First ensure remote directory exists + ssh_cmd = f'ssh {user}@{host} "mkdir -p {remote_path}"' + subprocess.run(ssh_cmd, shell=True, check=True) + + # Upload + rsync_cmd = f'rsync -avz --progress "{local_archive}" {user}@{host}:{remote_path}/' + result = subprocess.run(rsync_cmd, shell=True) + + if result.returncode == 0: + print("Upload successful!") + # Clean up local archive + local_archive.unlink() + return True + else: + print(f"Upload failed with code {result.returncode}") + return False + + +def restore_from_remote( + study_name: str, + target_dir: Path, + use_tailscale: bool = False +) -> bool: + """ + Restore a study from the remote server. + + Args: + study_name: Name of the study to restore + target_dir: Where to extract the study + use_tailscale: Use Tailscale IP + + Returns: + True if successful + """ + host = REMOTE_CONFIG["host_tailscale"] if use_tailscale else REMOTE_CONFIG["host"] + user = REMOTE_CONFIG["user"] + remote_path = REMOTE_CONFIG["archive_path"] + + target_dir = Path(target_dir) + + # Find the archive on remote + print(f"Looking for {study_name} on {host}...") + + ssh_cmd = f'ssh {user}@{host} "ls {remote_path}/{study_name}*.tar.gz 2>/dev/null | head -1"' + result = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True) + + if not result.stdout.strip(): + print(f"No archive found for {study_name}") + return False + + remote_archive = result.stdout.strip() + local_archive = target_dir / Path(remote_archive).name + + print(f"Downloading: {remote_archive}") + rsync_cmd = f'rsync -avz --progress {user}@{host}:"{remote_archive}" "{local_archive}"' + result = subprocess.run(rsync_cmd, shell=True) + + if result.returncode != 0: + print("Download failed") + return False + + print("Extracting...") + with tarfile.open(local_archive, "r:gz") as tar: + tar.extractall(target_dir) + + # Clean up + local_archive.unlink() + print(f"Restored to {target_dir / study_name}") + return True + + +def list_remote_archives(use_tailscale: bool = False) -> List[Dict]: + """List all archived studies on the remote server.""" + host = REMOTE_CONFIG["host_tailscale"] if use_tailscale else REMOTE_CONFIG["host"] + user = REMOTE_CONFIG["user"] + remote_path = REMOTE_CONFIG["archive_path"] + + ssh_cmd = f'ssh {user}@{host} "ls -lh {remote_path}/*.tar.gz 2>/dev/null"' + result = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True) + + archives = [] + for line in result.stdout.strip().split('\n'): + if line and '.tar.gz' in line: + parts = line.split() + if len(parts) >= 9: + archives.append({ + "name": parts[-1].split('/')[-1], + "size": parts[4], + "date": f"{parts[5]} {parts[6]} {parts[7]}", + }) + + return archives + + +def analyze_all_studies(studies_dir: Path) -> Dict: + """Analyze all studies in a directory.""" + studies_dir = Path(studies_dir) + + total_analysis = { + "total_size": 0, + "total_essential": 0, + "total_deletable": 0, + "studies": [], + } + + for study in sorted(studies_dir.iterdir()): + if study.is_dir() and not study.name.startswith('.'): + analysis = analyze_study(study) + total_analysis["studies"].append(analysis) + total_analysis["total_size"] += analysis["total_size_bytes"] + total_analysis["total_essential"] += analysis["essential_size"] + total_analysis["total_deletable"] += analysis["deletable_size"] + + return total_analysis + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Atomizer Study Archiver") + parser.add_argument("command", choices=["analyze", "cleanup", "archive", "restore", "list"]) + parser.add_argument("path", nargs="?", help="Study path or name") + parser.add_argument("--dry-run", action="store_true", default=True, + help="Don't actually delete/transfer (default: True)") + parser.add_argument("--execute", action="store_true", + help="Actually perform the operation") + parser.add_argument("--tailscale", action="store_true", + help="Use Tailscale IP for remote access") + + args = parser.parse_args() + + dry_run = not args.execute + + if args.command == "analyze": + if not args.path: + print("Usage: study_archiver analyze ") + return + + path = Path(args.path) + if path.is_dir(): + # Check if it's a single study or a collection + if (path / "optimization_config.json").exists() or (path / "1_setup").exists(): + # Single study + analysis = analyze_study(path) + print_analysis(analysis) + else: + # Collection of studies + total = analyze_all_studies(path) + print(f"\n{'='*60}") + print(f"Summary: {len(total['studies'])} studies") + print(f"{'='*60}") + print(f"Total size: {total['total_size']/1e9:8.2f} GB") + print(f"Essential: {total['total_essential']/1e9:8.2f} GB") + print(f"Deletable: {total['total_deletable']/1e9:8.2f} GB") + print(f"Potential save: {total['total_deletable']/1e9:8.2f} GB") + print(f"\nPer study:") + for s in total["studies"]: + print(f" {s['study_name']:40} {s['total_size_bytes']/1e9:6.2f} GB ({s['trial_count']:3d} trials)") + + elif args.command == "cleanup": + if not args.path: + print("Usage: study_archiver cleanup [--execute]") + return + cleanup_study(Path(args.path), dry_run=dry_run) + + elif args.command == "archive": + if not args.path: + print("Usage: study_archiver archive [--execute] [--tailscale]") + return + archive_to_remote(Path(args.path), use_tailscale=args.tailscale, dry_run=dry_run) + + elif args.command == "restore": + if not args.path: + print("Usage: study_archiver restore [--tailscale]") + return + target = Path.cwd() / "studies" + restore_from_remote(args.path, target, use_tailscale=args.tailscale) + + elif args.command == "list": + archives = list_remote_archives(use_tailscale=args.tailscale) + if archives: + print(f"\nArchived studies on dalidou:") + print(f"{'='*60}") + for a in archives: + print(f" {a['name']:40} {a['size']:>8} {a['date']}") + else: + print("No archives found (or server not reachable)") + + +if __name__ == "__main__": + main() diff --git a/optimization_engine/utils/study_cleanup.py b/optimization_engine/utils/study_cleanup.py new file mode 100644 index 00000000..a9cca2f2 --- /dev/null +++ b/optimization_engine/utils/study_cleanup.py @@ -0,0 +1,411 @@ +""" +Study Cleanup Utility +==================== + +Cleans up completed optimization studies to save disk space by removing +large intermediate files (NX models, FEM meshes, solver results) while +preserving essential data (parameters, extracted results, database). + +Usage: + python -m optimization_engine.utils.study_cleanup [options] + +Options: + --dry-run Show what would be deleted without actually deleting + --keep-best N Keep iteration folders for the top N best trials + --keep-pareto Keep all Pareto-optimal iterations (for multi-objective) + --aggressive Delete ALL iteration data (only keep DB and config) + +The database (study.db) contains all optimization results and can regenerate +any analysis. The original NX model in 1_setup is always preserved. +""" + +import argparse +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Optional + + +# Files to ALWAYS keep in iteration folders (tiny, essential) +ESSENTIAL_FILES = { + 'params.exp', # Design parameters for this iteration + '_temp_mass.txt', # Extracted mass + '_temp_part_properties.json', # Part properties + '_temp_zernike.json', # Zernike coefficients (if exists) + 'results.json', # Any extracted results +} + +# Extensions to DELETE (large, regenerable/already extracted) +DELETABLE_EXTENSIONS = { + '.op2', # Nastran binary results (~65 MB each) + '.prt', # NX Part files (~30-35 MB each) + '.fem', # FEM mesh files (~15 MB each) + '.dat', # Nastran input deck (~15 MB each) + '.sim', # Simulation file (~7 MB each) + '.afm', # FEA auxiliary (~4 MB each) + '.f04', # Nastran log + '.f06', # Nastran output + '.log', # Solver log + '.diag', # Diagnostics +} + + +def get_study_info(study_path: Path) -> dict: + """Get study metadata from config and database.""" + config_path = study_path / 'optimization_config.json' + # Try both possible DB locations + db_path = study_path / '3_results' / 'study.db' + if not db_path.exists(): + db_path = study_path / '2_results' / 'study.db' + + info = { + 'name': study_path.name, + 'has_config': config_path.exists(), + 'has_db': db_path.exists(), + 'trial_count': 0, + 'best_trials': [], + 'pareto_trials': [], + } + + if config_path.exists(): + with open(config_path) as f: + info['config'] = json.load(f) + + if db_path.exists(): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Get trial count + cursor.execute("SELECT COUNT(*) FROM trials WHERE state = 'COMPLETE'") + info['trial_count'] = cursor.fetchone()[0] + + # Try to get best trials (for single objective) + try: + cursor.execute(""" + SELECT trial_id, value FROM trial_values + WHERE objective = 0 + ORDER BY value ASC LIMIT 10 + """) + info['best_trials'] = [row[0] for row in cursor.fetchall()] + except Exception as e: + pass + + # Check for Pareto attribute + try: + cursor.execute(""" + SELECT DISTINCT trial_id FROM trial_system_attrs + WHERE key = 'pareto_optimal' AND value = '1' + """) + info['pareto_trials'] = [row[0] for row in cursor.fetchall()] + except: + pass + + conn.close() + + return info + + +def calculate_cleanup_savings(study_path: Path, keep_iters: set = None) -> dict: + """Calculate how much space would be saved by cleanup.""" + iterations_path = study_path / '2_iterations' + if not iterations_path.exists(): + iterations_path = study_path / '1_working' # Legacy structure + + if not iterations_path.exists(): + return {'total_size': 0, 'deletable_size': 0, 'keep_size': 0} + + total_size = 0 + deletable_size = 0 + keep_size = 0 + keep_iters = keep_iters or set() + + for iter_folder in iterations_path.iterdir(): + if not iter_folder.is_dir(): + continue + + # Extract iteration number + try: + iter_num = int(iter_folder.name.replace('iter', '')) + except: + continue + + for f in iter_folder.iterdir(): + if not f.is_file(): + continue + size = f.stat().st_size + total_size += size + + # Keep entire folder if in keep_iters + if iter_num in keep_iters: + keep_size += size + continue + + # Keep essential files + if f.name.lower() in {e.lower() for e in ESSENTIAL_FILES}: + keep_size += size + elif f.suffix.lower() in DELETABLE_EXTENSIONS: + deletable_size += size + else: + keep_size += size # Keep unknown files by default + + return { + 'total_size': total_size, + 'deletable_size': deletable_size, + 'keep_size': keep_size, + } + + +def cleanup_study( + study_path: Path, + dry_run: bool = True, + keep_best: int = 0, + keep_pareto: bool = False, + aggressive: bool = False, +) -> dict: + """ + Clean up a study to save disk space. + + Args: + study_path: Path to study folder + dry_run: If True, only report what would be deleted + keep_best: Number of best iterations to keep completely + keep_pareto: Keep all Pareto-optimal iterations + aggressive: Delete ALL iteration folders (only keep DB) + + Returns: + dict with cleanup statistics + """ + study_path = Path(study_path) + if not study_path.exists(): + raise ValueError(f"Study path does not exist: {study_path}") + + # Get study info + info = get_study_info(study_path) + + # Determine which iterations to keep + keep_iters = set() + if keep_best > 0 and info['best_trials']: + keep_iters.update(info['best_trials'][:keep_best]) + if keep_pareto and info['pareto_trials']: + keep_iters.update(info['pareto_trials']) + + # Find iterations folder + iterations_path = study_path / '2_iterations' + if not iterations_path.exists(): + iterations_path = study_path / '1_working' + + if not iterations_path.exists(): + return {'status': 'no_iterations', 'deleted_bytes': 0, 'deleted_files': 0} + + # Calculate savings + savings = calculate_cleanup_savings(study_path, keep_iters) + + deleted_bytes = 0 + deleted_files = 0 + deleted_folders = 0 + + if aggressive: + # Delete entire iterations folder + if not dry_run: + shutil.rmtree(iterations_path) + deleted_bytes = savings['total_size'] + deleted_folders = 1 + else: + deleted_bytes = savings['total_size'] + else: + # Selective cleanup + for iter_folder in iterations_path.iterdir(): + if not iter_folder.is_dir(): + continue + + # Extract iteration number + try: + iter_num = int(iter_folder.name.replace('iter', '')) + except: + continue + + # Skip kept iterations + if iter_num in keep_iters: + continue + + for f in iter_folder.iterdir(): + if not f.is_file(): + continue + + # Keep essential files + if f.name.lower() in {e.lower() for e in ESSENTIAL_FILES}: + continue + + # Delete deletable extensions + if f.suffix.lower() in DELETABLE_EXTENSIONS: + size = f.stat().st_size + if not dry_run: + f.unlink() + deleted_bytes += size + deleted_files += 1 + + return { + 'status': 'dry_run' if dry_run else 'completed', + 'study_name': info['name'], + 'trial_count': info['trial_count'], + 'kept_iterations': list(keep_iters), + 'total_size_before': savings['total_size'], + 'deleted_bytes': deleted_bytes, + 'deleted_files': deleted_files, + 'deleted_folders': deleted_folders, + 'space_saved_gb': deleted_bytes / (1024**3), + } + + +def cleanup_batch( + parent_path: Path, + pattern: str = "*", + dry_run: bool = True, + keep_best: int = 3, + keep_pareto: bool = False, + aggressive: bool = False, +) -> list: + """ + Clean up multiple studies matching a pattern. + + Args: + parent_path: Parent directory containing studies + pattern: Glob pattern to match study folders (e.g., "m1_mirror_*") + dry_run: If True, only report + keep_best: Keep N best iterations per study + keep_pareto: Keep Pareto-optimal iterations + aggressive: Delete all iteration folders + + Returns: + List of cleanup results + """ + parent_path = Path(parent_path) + results = [] + + for study_path in sorted(parent_path.glob(pattern)): + if not study_path.is_dir(): + continue + # Check if it looks like a study (has iterations folder) + if not (study_path / '2_iterations').exists() and not (study_path / '1_working').exists(): + continue + + try: + result = cleanup_study( + study_path, + dry_run=dry_run, + keep_best=keep_best, + keep_pareto=keep_pareto, + aggressive=aggressive, + ) + results.append(result) + except Exception as e: + results.append({ + 'study_name': study_path.name, + 'status': 'error', + 'error': str(e), + }) + + return results + + +def main(): + parser = argparse.ArgumentParser( + description='Clean up completed optimization studies to save disk space.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__ + ) + parser.add_argument('study_path', type=Path, help='Path to study folder or parent directory') + parser.add_argument('--dry-run', action='store_true', default=True, + help='Show what would be deleted without deleting (default)') + parser.add_argument('--execute', action='store_true', + help='Actually delete files (opposite of --dry-run)') + parser.add_argument('--keep-best', type=int, default=3, + help='Keep N best iterations completely (default: 3)') + parser.add_argument('--keep-pareto', action='store_true', + help='Keep all Pareto-optimal iterations') + parser.add_argument('--aggressive', action='store_true', + help='Delete ALL iteration data (only keep DB)') + parser.add_argument('--batch', type=str, metavar='PATTERN', + help='Clean multiple studies matching pattern (e.g., "m1_mirror_*")') + + args = parser.parse_args() + + dry_run = not args.execute + + if args.batch: + # Batch cleanup mode + print(f"\n{'='*60}") + print(f"BATCH CLEANUP: {args.study_path}") + print(f"Pattern: {args.batch}") + print(f"{'='*60}") + print(f"Mode: {'DRY RUN' if dry_run else 'EXECUTE'}") + + results = cleanup_batch( + args.study_path, + pattern=args.batch, + dry_run=dry_run, + keep_best=args.keep_best, + keep_pareto=args.keep_pareto, + aggressive=args.aggressive, + ) + + print(f"\n{'='*60}") + print("BATCH RESULTS") + print(f"{'='*60}") + print(f"{'Study':<45} {'Trials':>7} {'Size':>8} {'Savings':>8}") + print("-" * 75) + + total_saved = 0 + for r in results: + if r.get('status') == 'error': + print(f"{r['study_name']:<45} ERROR: {r.get('error', 'Unknown')}") + else: + saved = r.get('space_saved_gb', 0) + total_saved += saved + print(f"{r['study_name']:<45} {r.get('trial_count', 0):>7} " + f"{r.get('total_size_before', 0)/(1024**3):>7.1f}G {saved:>7.1f}G") + + print("-" * 75) + print(f"{'TOTAL SAVINGS:':<45} {' '*15} {total_saved:>7.1f}G") + + if dry_run: + print(f"\n[!] This was a dry run. Run with --execute to actually delete files.") + + return results + + else: + # Single study cleanup + print(f"\n{'='*60}") + print(f"STUDY CLEANUP: {args.study_path.name}") + print(f"{'='*60}") + print(f"Mode: {'DRY RUN (no files deleted)' if dry_run else 'EXECUTE (files WILL be deleted)'}") + print(f"Keep best: {args.keep_best} iterations") + print(f"Keep Pareto: {args.keep_pareto}") + print(f"Aggressive: {args.aggressive}") + + result = cleanup_study( + args.study_path, + dry_run=dry_run, + keep_best=args.keep_best, + keep_pareto=args.keep_pareto, + aggressive=args.aggressive, + ) + + print(f"\n{'='*60}") + print("RESULTS") + print(f"{'='*60}") + print(f"Trials in study: {result['trial_count']}") + print(f"Iterations kept: {len(result['kept_iterations'])} {result['kept_iterations'][:5]}{'...' if len(result['kept_iterations']) > 5 else ''}") + print(f"Total size before: {result['total_size_before'] / (1024**3):.2f} GB") + print(f"{'Would delete' if dry_run else 'Deleted'}: {result['deleted_files']} files") + print(f"Space {'to save' if dry_run else 'saved'}: {result['space_saved_gb']:.2f} GB") + + if dry_run: + print(f"\n[!] This was a dry run. Run with --execute to actually delete files.") + + return result + + +if __name__ == '__main__': + main() diff --git a/run_cleanup.py b/run_cleanup.py new file mode 100644 index 00000000..79762941 --- /dev/null +++ b/run_cleanup.py @@ -0,0 +1,67 @@ +"""Run cleanup excluding protected studies.""" +import sys +from pathlib import Path + +# Add project root to path +sys.path.insert(0, str(Path(__file__).parent)) + +from optimization_engine.utils.study_cleanup import cleanup_study, get_study_info + +m1_dir = Path(r"C:\Users\antoi\Atomizer\studies\M1_Mirror") + +# Studies to SKIP (user requested) +skip_patterns = [ + "cost_reduction_V10", + "cost_reduction_V11", + "cost_reduction_V12", + "flat_back", +] + +# Parse args +dry_run = "--execute" not in sys.argv +keep_best = 5 + +total_saved = 0 +studies_to_clean = [] + +print("=" * 75) +print(f"CLEANUP (excluding V10-V12 and flat_back studies)") +print(f"Mode: {'DRY RUN' if dry_run else 'EXECUTE'}") +print("=" * 75) +print(f"{'Study':<45} {'Trials':>7} {'Size':>8} {'Savings':>8}") +print("-" * 75) + +for study_path in sorted(m1_dir.iterdir()): + if not study_path.is_dir(): + continue + # Check if has iterations + if not (study_path / "2_iterations").exists(): + continue + + # Skip protected studies + skip = False + for pattern in skip_patterns: + if pattern in study_path.name: + skip = True + break + + if skip: + info = get_study_info(study_path) + print(f"{study_path.name:<45} {info['trial_count']:>7} SKIPPED") + continue + + # This study will be cleaned + result = cleanup_study(study_path, dry_run=dry_run, keep_best=keep_best) + saved = result["space_saved_gb"] + total_saved += saved + status = "would save" if dry_run else "saved" + print(f"{study_path.name:<45} {result['trial_count']:>7} {result['total_size_before']/(1024**3):>7.1f}G {saved:>7.1f}G") + studies_to_clean.append(study_path.name) + +print("-" * 75) +print(f"{'TOTAL SAVINGS:':<45} {' '*15} {total_saved:>7.1f}G") + +if dry_run: + print(f"\n[!] This was a dry run. Run with --execute to actually delete files.") +else: + print(f"\n[OK] Cleanup complete! Freed {total_saved:.1f} GB") diff --git a/tools/archive_study.bat b/tools/archive_study.bat new file mode 100644 index 00000000..ece428cc --- /dev/null +++ b/tools/archive_study.bat @@ -0,0 +1,32 @@ +@echo off +REM Atomizer Study Archiver - Convenience Script +REM Usage: archive_study.bat [study_path] +REM +REM Commands: +REM analyze - Show disk usage analysis +REM cleanup - Remove regenerable files (dry run by default) +REM archive - Archive to dalidou server +REM list - List archived studies on server +REM +REM Examples: +REM archive_study.bat analyze studies\M1_Mirror +REM archive_study.bat cleanup studies\M1_Mirror\m1_mirror_V12 --execute +REM archive_study.bat archive studies\M1_Mirror\m1_mirror_V12 --execute + +cd /d C:\Users\antoi\Atomizer + +if "%1"=="" ( + echo Usage: archive_study.bat ^ [path] [options] + echo. + echo Commands: + echo analyze ^ - Analyze disk usage + echo cleanup ^ [--execute] - Remove regenerable files + echo archive ^ [--execute] - Archive to dalidou + echo restore ^ - Restore from dalidou + echo list - List remote archives + echo. + echo Add --tailscale for remote access via Tailscale + exit /b 1 +) + +C:\Users\antoi\anaconda3\envs\atomizer\python.exe -m optimization_engine.utils.study_archiver %* diff --git a/tools/zernike_html_generator.py b/tools/zernike_html_generator.py index 1a2932d3..bc7acd8b 100644 --- a/tools/zernike_html_generator.py +++ b/tools/zernike_html_generator.py @@ -8,6 +8,11 @@ Generates 3 interactive HTML reports for Zernike wavefront analysis: 2. 60° vs 20° (relative) - Operational angle comparison 3. 90° (Manufacturing) - Absolute with manufacturing metrics +Uses the rigorous OPD method from extract_zernike_figure.py which: +- Accounts for lateral (X, Y) displacement via interpolation +- Uses actual mesh geometry as reference (no shape assumptions) +- Provides more accurate WFE for mirror optimization + Usage: conda activate atomizer python zernike_html_generator.py "path/to/solution.op2" @@ -23,6 +28,7 @@ Output: Author: Atomizer Created: 2025-12-19 +Updated: 2025-12-28 - Upgraded to use rigorous OPD method """ import sys @@ -49,6 +55,15 @@ except ImportError as e: print("Run: conda activate atomizer") sys.exit(1) +# Import the rigorous OPD extractor +try: + from optimization_engine.extractors.extract_zernike_figure import ZernikeOPDExtractor + USE_OPD_METHOD = True + print("[INFO] Using rigorous OPD method (accounts for lateral displacement)") +except ImportError: + USE_OPD_METHOD = False + print("[WARN] OPD extractor not available, falling back to simple Z-only method") + # ============================================================================ # Configuration @@ -278,13 +293,31 @@ def compute_rms_metrics(X, Y, W_nm): def compute_mfg_metrics(coeffs): - """Manufacturing aberration magnitudes.""" + """Manufacturing aberration magnitudes from Zernike coefficients. + + Noll indexing (1-based): J1=Piston, J2=TiltX, J3=TiltY, J4=Defocus, + J5=Astig45, J6=Astig0, J7=ComaX, J8=ComaY, J9=TrefoilX, J10=TrefoilY, J11=Spherical + + Python 0-indexed: coeffs[0]=J1, coeffs[3]=J4, etc. + """ + # Individual mode magnitudes (RSS for paired modes) + defocus = float(abs(coeffs[3])) # J4 + astigmatism = float(np.sqrt(coeffs[4]**2 + coeffs[5]**2)) # RSS(J5, J6) + coma = float(np.sqrt(coeffs[6]**2 + coeffs[7]**2)) # RSS(J7, J8) + trefoil = float(np.sqrt(coeffs[8]**2 + coeffs[9]**2)) # RSS(J9, J10) + spherical = float(abs(coeffs[10])) if len(coeffs) > 10 else 0.0 # J11 + + # RMS of higher-order terms (J4+): sqrt(sum of squares of coefficients) + # This is the proper Zernike-coefficient-based RMS excluding piston/tip/tilt + higher_order_rms = float(np.sqrt(np.sum(coeffs[3:]**2))) + return { - 'defocus_nm': float(abs(coeffs[3])), - 'astigmatism_rms': float(np.sqrt(coeffs[4]**2 + coeffs[5]**2)), - 'coma_rms': float(np.sqrt(coeffs[6]**2 + coeffs[7]**2)), - 'trefoil_rms': float(np.sqrt(coeffs[8]**2 + coeffs[9]**2)), - 'spherical_nm': float(abs(coeffs[10])) if len(coeffs) > 10 else 0.0, + 'defocus_nm': defocus, + 'astigmatism_rms': astigmatism, + 'coma_rms': coma, + 'trefoil_rms': trefoil, + 'spherical_nm': spherical, + 'higher_order_rms': higher_order_rms, # RMS of all J4+ coefficients } @@ -502,19 +535,22 @@ def generate_html( ], align="left", fill_color='#374151', font=dict(color='white')) ), row=3, col=1) - # Pre-correction (row 4) + # Pre-correction (row 4) - Aberrations to polish out (90° - 20°) + # Shows what correction is needed when manufacturing at 90° to achieve 20° figure fig.add_trace(go.Table( - header=dict(values=["Mode", "Correction (nm)"], + header=dict(values=["Aberration", "Magnitude (nm)"], align="left", fill_color='#1f2937', font=dict(color='white')), cells=dict(values=[ - ["Total RMS (J1-J3 filter)", - "Defocus (J4)", + ["Defocus (J4)", "Astigmatism (J5+J6)", - "Coma (J7+J8)"], - [f"{correction_metrics['rms_filter_j1to3']:.2f}", - f"{correction_metrics['defocus_nm']:.2f}", + "Coma (J7+J8)", + "Trefoil (J9+J10)", + "Spherical (J11)"], + [f"{correction_metrics['defocus_nm']:.2f}", f"{correction_metrics['astigmatism_rms']:.2f}", - f"{correction_metrics['coma_rms']:.2f}"] + f"{correction_metrics['coma_rms']:.2f}", + f"{correction_metrics['trefoil_rms']:.2f}", + f"{correction_metrics['spherical_nm']:.2f}"] ], align="left", fill_color='#374151', font=dict(color='white')) ), row=4, col=1) else: @@ -595,8 +631,248 @@ def find_op2_file(working_dir=None): return max(op2_files, key=lambda p: p.stat().st_mtime) +def main_opd(op2_path: Path): + """Generate all 3 HTML files using rigorous OPD method.""" + print("=" * 70) + print(" ATOMIZER ZERNIKE HTML GENERATOR (OPD METHOD)") + print("=" * 70) + print(f"\nOP2 File: {op2_path.name}") + print(f"Directory: {op2_path.parent}") + print("\n[INFO] Using OPD method: accounts for lateral (X,Y) displacement") + + # Initialize extractor + extractor = ZernikeOPDExtractor( + op2_path, + displacement_unit='mm', + n_modes=N_MODES, + filter_orders=FILTER_LOW_ORDERS + ) + + print(f"\nAvailable subcases: {list(extractor.displacements.keys())}") + + # Map subcases (try common patterns) + displacements = extractor.displacements + subcase_map = {} + + if '1' in displacements and '2' in displacements: + subcase_map = {'90': '1', '20': '2', '40': '3', '60': '4'} + elif '90' in displacements and '20' in displacements: + subcase_map = {'90': '90', '20': '20', '40': '40', '60': '60'} + else: + available = sorted(displacements.keys(), key=lambda x: int(x) if x.isdigit() else 0) + if len(available) >= 4: + subcase_map = {'90': available[0], '20': available[1], '40': available[2], '60': available[3]} + print(f"[WARN] Using mapped subcases: {subcase_map}") + else: + print(f"[ERROR] Need 4 subcases, found: {available}") + return + + output_dir = op2_path.parent + base = op2_path.stem + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + html_files = [] + + # ======================================================================== + # Extract absolute metrics for each subcase + # ======================================================================== + print("\nExtracting absolute metrics (OPD method)...") + + results_abs = {} + for angle, label in subcase_map.items(): + result = extractor.extract_subcase(label, include_coefficients=True) + results_abs[angle] = result + lat_disp = result.get('max_lateral_displacement_um', 0) + print(f" {angle} deg: Filtered RMS = {result['filtered_rms_nm']:.2f} nm, " + f"Lateral disp max = {lat_disp:.3f} um") + + # ======================================================================== + # Extract relative metrics (40-20, 60-20, 90-20) + # ======================================================================== + print("\nExtracting relative metrics (OPD method)...") + + # 40 vs 20 + result_40_rel = extractor.extract_relative(subcase_map['40'], subcase_map['20'], include_coefficients=True) + print(f" 40-20: Relative Filtered RMS = {result_40_rel['relative_filtered_rms_nm']:.2f} nm") + + # 60 vs 20 + result_60_rel = extractor.extract_relative(subcase_map['60'], subcase_map['20'], include_coefficients=True) + print(f" 60-20: Relative Filtered RMS = {result_60_rel['relative_filtered_rms_nm']:.2f} nm") + + # 90 vs 20 (for correction metrics) + result_90_rel = extractor.extract_relative(subcase_map['90'], subcase_map['20'], include_coefficients=True) + print(f" 90-20: Relative Filtered RMS = {result_90_rel['relative_filtered_rms_nm']:.2f} nm") + + # ======================================================================== + # Generate HTML files + # ======================================================================== + + # Helper to convert OPD results to the format expected by generate_html + def opd_to_rms_data(result, is_relative=False): + """Convert OPD extractor result to rms_data dict for generate_html.""" + coeffs = np.array(result.get('coefficients', [0] * N_MODES)) + + # Recompute filtered residuals for visualization + # For now, use simplified approach - the metrics are correct + filtered_rms = result.get('relative_filtered_rms_nm' if is_relative else 'filtered_rms_nm', 0) + global_rms = result.get('relative_global_rms_nm' if is_relative else 'global_rms_nm', 0) + rms_j1to3 = result.get('relative_rms_filter_j1to3' if is_relative else 'rms_filter_j1to3_nm', 0) + + # We need W_res_filt for visualization - extract from diagnostic data + # For now, create a placeholder that will be updated + return { + 'coefficients': coeffs, + 'R': 1.0, # Will be updated + 'global_rms': global_rms, + 'filtered_rms': filtered_rms, + 'rms_filter_j1to3': rms_j1to3, + 'W_res_filt': None, # Will compute separately for visualization + } + + # For visualization, we need the actual WFE arrays + # Get diagnostic data from extractor + print("\nGenerating HTML reports...") + + # 40 vs 20 + print(" Generating 40 deg vs 20 deg...") + opd_40 = extractor._build_figure_opd_data(subcase_map['40']) + opd_20 = extractor._build_figure_opd_data(subcase_map['20']) + + # Build relative WFE arrays + ref_wfe_map = {int(nid): wfe for nid, wfe in zip(opd_20['node_ids'], opd_20['wfe_nm'])} + X_40_rel, Y_40_rel, WFE_40_rel = [], [], [] + for i, nid in enumerate(opd_40['node_ids']): + nid = int(nid) + if nid in ref_wfe_map: + X_40_rel.append(opd_40['x_deformed'][i]) + Y_40_rel.append(opd_40['y_deformed'][i]) + WFE_40_rel.append(opd_40['wfe_nm'][i] - ref_wfe_map[nid]) + X_40_rel = np.array(X_40_rel) + Y_40_rel = np.array(Y_40_rel) + WFE_40_rel = np.array(WFE_40_rel) + + rms_40_rel = compute_rms_metrics(X_40_rel, Y_40_rel, WFE_40_rel) + rms_40_abs = compute_rms_metrics(opd_40['x_deformed'], opd_40['y_deformed'], opd_40['wfe_nm']) + + html_40 = generate_html( + title="40 deg (OPD)", + X=X_40_rel, Y=Y_40_rel, W_nm=WFE_40_rel, + rms_data=rms_40_rel, + is_relative=True, + ref_title="20 deg", + abs_pair=(rms_40_abs['global_rms'], rms_40_abs['filtered_rms']) + ) + path_40 = output_dir / f"{base}_{timestamp}_40_vs_20.html" + path_40.write_text(html_40, encoding='utf-8') + html_files.append(path_40) + print(f" Created: {path_40.name}") + + # 60 vs 20 + print(" Generating 60 deg vs 20 deg...") + opd_60 = extractor._build_figure_opd_data(subcase_map['60']) + + X_60_rel, Y_60_rel, WFE_60_rel = [], [], [] + for i, nid in enumerate(opd_60['node_ids']): + nid = int(nid) + if nid in ref_wfe_map: + X_60_rel.append(opd_60['x_deformed'][i]) + Y_60_rel.append(opd_60['y_deformed'][i]) + WFE_60_rel.append(opd_60['wfe_nm'][i] - ref_wfe_map[nid]) + X_60_rel = np.array(X_60_rel) + Y_60_rel = np.array(Y_60_rel) + WFE_60_rel = np.array(WFE_60_rel) + + rms_60_rel = compute_rms_metrics(X_60_rel, Y_60_rel, WFE_60_rel) + rms_60_abs = compute_rms_metrics(opd_60['x_deformed'], opd_60['y_deformed'], opd_60['wfe_nm']) + + html_60 = generate_html( + title="60 deg (OPD)", + X=X_60_rel, Y=Y_60_rel, W_nm=WFE_60_rel, + rms_data=rms_60_rel, + is_relative=True, + ref_title="20 deg", + abs_pair=(rms_60_abs['global_rms'], rms_60_abs['filtered_rms']) + ) + path_60 = output_dir / f"{base}_{timestamp}_60_vs_20.html" + path_60.write_text(html_60, encoding='utf-8') + html_files.append(path_60) + print(f" Created: {path_60.name}") + + # 90 deg Manufacturing + print(" Generating 90 deg Manufacturing...") + opd_90 = extractor._build_figure_opd_data(subcase_map['90']) + rms_90 = compute_rms_metrics(opd_90['x_deformed'], opd_90['y_deformed'], opd_90['wfe_nm']) + mfg_metrics = compute_mfg_metrics(rms_90['coefficients']) + + # 90-20 relative for correction metrics + X_90_rel, Y_90_rel, WFE_90_rel = [], [], [] + for i, nid in enumerate(opd_90['node_ids']): + nid = int(nid) + if nid in ref_wfe_map: + X_90_rel.append(opd_90['x_deformed'][i]) + Y_90_rel.append(opd_90['y_deformed'][i]) + WFE_90_rel.append(opd_90['wfe_nm'][i] - ref_wfe_map[nid]) + X_90_rel = np.array(X_90_rel) + Y_90_rel = np.array(Y_90_rel) + WFE_90_rel = np.array(WFE_90_rel) + rms_90_rel = compute_rms_metrics(X_90_rel, Y_90_rel, WFE_90_rel) + + # Get all correction metrics from Zernike coefficients (90° - 20°) + correction_metrics = compute_mfg_metrics(rms_90_rel['coefficients']) + + html_90 = generate_html( + title="90 deg Manufacturing (OPD)", + X=opd_90['x_deformed'], Y=opd_90['y_deformed'], W_nm=opd_90['wfe_nm'], + rms_data=rms_90, + is_relative=False, + is_manufacturing=True, + mfg_metrics=mfg_metrics, + correction_metrics=correction_metrics + ) + path_90 = output_dir / f"{base}_{timestamp}_90_mfg.html" + path_90.write_text(html_90, encoding='utf-8') + html_files.append(path_90) + print(f" Created: {path_90.name}") + + # ======================================================================== + # Summary + # ======================================================================== + print("\n" + "=" * 70) + print("SUMMARY (OPD Method)") + print("=" * 70) + print(f"\nGenerated {len(html_files)} HTML files:") + for f in html_files: + print(f" - {f.name}") + + print("\n" + "-" * 70) + print("OPTIMIZATION OBJECTIVES (OPD Method)") + print("-" * 70) + print(f" 40-20 Filtered RMS: {rms_40_rel['filtered_rms']:.2f} nm") + print(f" 60-20 Filtered RMS: {rms_60_rel['filtered_rms']:.2f} nm") + print(f" MFG 90 (J1-J3): {rms_90_rel['rms_filter_j1to3']:.2f} nm") + + # Weighted sums + ws_v4 = 5*rms_40_rel['filtered_rms'] + 5*rms_60_rel['filtered_rms'] + 2*rms_90_rel['rms_filter_j1to3'] + ws_v5 = 5*rms_40_rel['filtered_rms'] + 5*rms_60_rel['filtered_rms'] + 3*rms_90_rel['rms_filter_j1to3'] + print(f"\n V4 Weighted Sum (5/5/2): {ws_v4:.2f}") + print(f" V5 Weighted Sum (5/5/3): {ws_v5:.2f}") + + # Lateral displacement summary + print("\n" + "-" * 70) + print("LATERAL DISPLACEMENT SUMMARY") + print("-" * 70) + for angle in ['20', '40', '60', '90']: + lat = results_abs[angle].get('max_lateral_displacement_um', 0) + print(f" {angle} deg: max {lat:.3f} um") + + print("\n" + "=" * 70) + print("DONE") + print("=" * 70) + + return html_files + + def main(op2_path: Path): - """Generate all 3 HTML files.""" + """Generate all 3 HTML files (legacy Z-only method).""" print("=" * 70) print(" ATOMIZER ZERNIKE HTML GENERATOR") print("=" * 70) @@ -753,12 +1029,8 @@ def main(op2_path: Path): X_ref, Y_ref, WFE_ref, ref_data['node_ids'] ) rms_90_rel = compute_rms_metrics(X_90_rel, Y_90_rel, WFE_90_rel) - correction_metrics = { - 'rms_filter_j1to3': rms_90_rel['rms_filter_j1to3'], - 'defocus_nm': compute_mfg_metrics(rms_90_rel['coefficients'])['defocus_nm'], - 'astigmatism_rms': compute_mfg_metrics(rms_90_rel['coefficients'])['astigmatism_rms'], - 'coma_rms': compute_mfg_metrics(rms_90_rel['coefficients'])['coma_rms'], - } + # Get all correction metrics from Zernike coefficients (90° - 20°) + correction_metrics = compute_mfg_metrics(rms_90_rel['coefficients']) html_90 = generate_html( title="90 deg (Manufacturing)", @@ -822,8 +1094,16 @@ if __name__ == '__main__': sys.exit(1) print(f"Found: {op2_path}") + # Check for --legacy flag to use old Z-only method + use_legacy = '--legacy' in sys.argv or '--z-only' in sys.argv + try: - main(op2_path) + if USE_OPD_METHOD and not use_legacy: + main_opd(op2_path) + else: + if use_legacy: + print("[INFO] Using legacy Z-only method (--legacy flag)") + main(op2_path) except Exception as e: print(f"\nERROR: {e}") import traceback