feat: Add AtomizerField training data export and intelligent model discovery
Major additions: - Training data export system for AtomizerField neural network training - Bracket stiffness optimization study with 50+ training samples - Intelligent NX model discovery (auto-detect solutions, expressions, mesh) - Result extractors module for displacement, stress, frequency, mass - User-generated NX journals for advanced workflows - Archive structure for legacy scripts and test outputs - Protocol documentation and dashboard launcher 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
101
archive/scripts/analyze_v3_pareto.py
Normal file
101
archive/scripts/analyze_v3_pareto.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Analyze V3 Pareto front performance."""
|
||||
|
||||
import optuna
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Load study
|
||||
study = optuna.load_study(
|
||||
study_name='bracket_stiffness_optimization_V3',
|
||||
storage='sqlite:///studies/bracket_stiffness_optimization_V3/2_results/study.db'
|
||||
)
|
||||
|
||||
# Get Pareto front
|
||||
pareto = study.best_trials
|
||||
|
||||
print("=" * 80)
|
||||
print("BRACKET STIFFNESS OPTIMIZATION V3 - PERFORMANCE SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
print(f"\nPareto Front Size: {len(pareto)} solutions")
|
||||
print(f"Total Trials: {len(study.trials)} (100 requested)")
|
||||
print(f"Completed Trials: {len([t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE])}")
|
||||
print(f"Pruned Trials: {len([t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED])}")
|
||||
|
||||
# Objective ranges
|
||||
stiffnesses = [t.values[0] for t in pareto]
|
||||
masses = [t.values[1] for t in pareto]
|
||||
|
||||
print(f"\n--- OBJECTIVE RANGES (PARETO FRONT) ---")
|
||||
print(f"Stiffness Range: [{min(stiffnesses):.2f}, {max(stiffnesses):.2f}] N/mm")
|
||||
print(f" (inverted for maximization: stiffness = -compliance)")
|
||||
print(f"Mass Range: [{min(masses):.4f}, {max(masses):.4f}] kg")
|
||||
print(f" ({min(masses)*1000:.2f}g - {max(masses)*1000:.2f}g)")
|
||||
|
||||
# Efficiency
|
||||
efficiency = (len(pareto) / len([t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE])) * 100
|
||||
print(f"\nPareto Efficiency: {efficiency:.1f}% of completed trials are on Pareto front")
|
||||
|
||||
# Top 5 by stiffness
|
||||
print(f"\n--- TOP 5 PARETO SOLUTIONS (by STIFFNESS) ---")
|
||||
sorted_by_stiffness = sorted(pareto, key=lambda x: x.values[0])
|
||||
for i, trial in enumerate(sorted_by_stiffness[:5]):
|
||||
stiffness = -trial.values[0] # Invert back
|
||||
compliance = 1/stiffness if stiffness != 0 else float('inf')
|
||||
mass_g = trial.values[1] * 1000
|
||||
print(f"\n{i+1}. Trial #{trial.number}")
|
||||
print(f" Stiffness: {stiffness:.2f} N/mm (compliance: {compliance:.6f} mm/N)")
|
||||
print(f" Mass: {mass_g:.2f}g")
|
||||
print(f" Support Angle: {trial.params['support_angle']:.2f}°")
|
||||
print(f" Tip Thickness: {trial.params['tip_thickness']:.2f}mm")
|
||||
|
||||
# Top 5 by mass (lightest)
|
||||
print(f"\n--- TOP 5 PARETO SOLUTIONS (by LIGHTEST MASS) ---")
|
||||
sorted_by_mass = sorted(pareto, key=lambda x: x.values[1])
|
||||
for i, trial in enumerate(sorted_by_mass[:5]):
|
||||
stiffness = -trial.values[0]
|
||||
compliance = 1/stiffness if stiffness != 0 else float('inf')
|
||||
mass_g = trial.values[1] * 1000
|
||||
print(f"\n{i+1}. Trial #{trial.number}")
|
||||
print(f" Mass: {mass_g:.2f}g")
|
||||
print(f" Stiffness: {stiffness:.2f} N/mm (compliance: {compliance:.6f} mm/N)")
|
||||
print(f" Support Angle: {trial.params['support_angle']:.2f}°")
|
||||
print(f" Tip Thickness: {trial.params['tip_thickness']:.2f}mm")
|
||||
|
||||
# Load optimization summary
|
||||
summary_path = Path("studies/bracket_stiffness_optimization_V3/2_results/optimization_summary.json")
|
||||
with open(summary_path, 'r') as f:
|
||||
summary = json.load(f)
|
||||
|
||||
print(f"\n--- OPTIMIZATION PERFORMANCE ---")
|
||||
print(f"Total Time: {summary['elapsed_seconds']:.1f}s ({summary['elapsed_seconds']/60:.1f} minutes)")
|
||||
print(f"Average Time per Trial: {summary['elapsed_seconds']/summary['completed_trials']:.2f}s")
|
||||
print(f"Optimizer: {summary['optimizer']}")
|
||||
print(f"Final Strategy: NSGA-II (multi-objective)")
|
||||
|
||||
# Design space coverage
|
||||
all_angles = [t.params['support_angle'] for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
|
||||
all_thicknesses = [t.params['tip_thickness'] for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
|
||||
|
||||
print(f"\n--- DESIGN SPACE EXPLORATION ---")
|
||||
print(f"Support Angle Range: [{min(all_angles):.2f}°, {max(all_angles):.2f}°]")
|
||||
print(f"Tip Thickness Range: [{min(all_thicknesses):.2f}mm, {max(all_thicknesses):.2f}mm]")
|
||||
|
||||
# Pareto design space
|
||||
pareto_angles = [t.params['support_angle'] for t in pareto]
|
||||
pareto_thicknesses = [t.params['tip_thickness'] for t in pareto]
|
||||
|
||||
print(f"\n--- PARETO DESIGN SPACE (Optimal Regions) ---")
|
||||
print(f"Support Angle Range: [{min(pareto_angles):.2f}°, {max(pareto_angles):.2f}°]")
|
||||
print(f"Tip Thickness Range: [{min(pareto_thicknesses):.2f}mm, {max(pareto_thicknesses):.2f}mm]")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("CONCLUSION")
|
||||
print("=" * 80)
|
||||
print(f"✓ Successfully completed 100-trial multi-objective optimization")
|
||||
print(f"✓ Generated {len(pareto)} Pareto-optimal solutions ({efficiency:.1f}% efficiency)")
|
||||
print(f"✓ No crashes or Protocol 11 violations")
|
||||
print(f"✓ Stiffness improvements up to {-min(stiffnesses):.0f} N/mm")
|
||||
print(f"✓ Mass range: {min(masses)*1000:.0f}g - {max(masses)*1000:.0f}g")
|
||||
print("✓ All tracking files (trial_log.json, optimizer_state.json) written successfully")
|
||||
print("=" * 80)
|
||||
70
archive/scripts/create_circular_plate_study.py
Normal file
70
archive/scripts/create_circular_plate_study.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Create circular plate frequency tuning study with COMPLETE automation.
|
||||
|
||||
This demonstrates the proper Hybrid Mode workflow:
|
||||
1. Study structure creation
|
||||
2. Benchmarking
|
||||
3. Validation
|
||||
4. Auto-generated runner
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from optimization_engine.hybrid_study_creator import HybridStudyCreator
|
||||
|
||||
|
||||
def main():
|
||||
creator = HybridStudyCreator()
|
||||
|
||||
# Create workflow JSON first (in temp location)
|
||||
import json
|
||||
import tempfile
|
||||
workflow = {
|
||||
"study_name": "circular_plate_frequency_tuning",
|
||||
"optimization_request": "Tune the first natural frequency mode to exactly 115 Hz (within 0.1 Hz tolerance)",
|
||||
"design_variables": [
|
||||
{"parameter": "inner_diameter", "bounds": [50, 150]},
|
||||
{"parameter": "plate_thickness", "bounds": [2, 10]}
|
||||
],
|
||||
"objectives": [{
|
||||
"name": "frequency_error",
|
||||
"goal": "minimize",
|
||||
"extraction": {
|
||||
"action": "extract_first_natural_frequency",
|
||||
"params": {"mode_number": 1, "target_frequency": 115.0}
|
||||
}
|
||||
}],
|
||||
"constraints": [{
|
||||
"name": "frequency_tolerance",
|
||||
"type": "less_than",
|
||||
"threshold": 0.1
|
||||
}]
|
||||
}
|
||||
|
||||
# Write to temp file
|
||||
temp_workflow = Path(tempfile.gettempdir()) / "circular_plate_workflow.json"
|
||||
with open(temp_workflow, 'w') as f:
|
||||
json.dump(workflow, f, indent=2)
|
||||
|
||||
# Create study with complete automation
|
||||
study_dir = creator.create_from_workflow(
|
||||
workflow_json_path=temp_workflow,
|
||||
model_files={
|
||||
'prt': Path("examples/Models/Circular Plate/Circular_Plate.prt"),
|
||||
'sim': Path("examples/Models/Circular Plate/Circular_Plate_sim1.sim"),
|
||||
'fem': Path("examples/Models/Circular Plate/Circular_Plate_fem1.fem"),
|
||||
'fem_i': Path("examples/Models/Circular Plate/Circular_Plate_fem1_i.prt")
|
||||
},
|
||||
study_name="circular_plate_frequency_tuning"
|
||||
)
|
||||
|
||||
print(f"Study ready at: {study_dir}")
|
||||
print()
|
||||
print("Next step:")
|
||||
print(f" python {study_dir}/run_optimization.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
89
archive/scripts/create_circular_plate_study_v2.py
Normal file
89
archive/scripts/create_circular_plate_study_v2.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Create circular_plate_frequency_tuning_V2 study with ALL fixes applied.
|
||||
|
||||
Improvements:
|
||||
- Proper study naming
|
||||
- Reports go to 3_reports/ folder
|
||||
- Reports discuss actual goal (115 Hz target)
|
||||
- Fixed objective function
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import argparse
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from optimization_engine.hybrid_study_creator import HybridStudyCreator
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Create circular plate frequency tuning study')
|
||||
parser.add_argument('--study-name', default='circular_plate_frequency_tuning_V2',
|
||||
help='Name of the study folder')
|
||||
args = parser.parse_args()
|
||||
|
||||
study_name = args.study_name
|
||||
|
||||
creator = HybridStudyCreator()
|
||||
|
||||
# Create workflow JSON
|
||||
import json
|
||||
import tempfile
|
||||
workflow = {
|
||||
"study_name": study_name,
|
||||
"optimization_request": "Tune the first natural frequency mode to exactly 115 Hz (within 0.1 Hz tolerance)",
|
||||
"design_variables": [
|
||||
{"parameter": "inner_diameter", "bounds": [50, 150]},
|
||||
{"parameter": "plate_thickness", "bounds": [2, 10]}
|
||||
],
|
||||
"objectives": [{
|
||||
"name": "frequency_error",
|
||||
"goal": "minimize",
|
||||
"extraction": {
|
||||
"action": "extract_first_natural_frequency",
|
||||
"params": {"mode_number": 1, "target_frequency": 115.0}
|
||||
}
|
||||
}],
|
||||
"constraints": [{
|
||||
"name": "frequency_tolerance",
|
||||
"type": "less_than",
|
||||
"threshold": 0.1
|
||||
}]
|
||||
}
|
||||
|
||||
# Write to temp file
|
||||
temp_workflow = Path(tempfile.gettempdir()) / f"{study_name}_workflow.json"
|
||||
with open(temp_workflow, 'w') as f:
|
||||
json.dump(workflow, f, indent=2)
|
||||
|
||||
# Create study
|
||||
study_dir = creator.create_from_workflow(
|
||||
workflow_json_path=temp_workflow,
|
||||
model_files={
|
||||
'prt': Path("examples/Models/Circular Plate/Circular_Plate.prt"),
|
||||
'sim': Path("examples/Models/Circular Plate/Circular_Plate_sim1.sim"),
|
||||
'fem': Path("examples/Models/Circular Plate/Circular_Plate_fem1.fem"),
|
||||
'fem_i': Path("examples/Models/Circular Plate/Circular_Plate_fem1_i.prt")
|
||||
},
|
||||
study_name=study_name
|
||||
)
|
||||
|
||||
print()
|
||||
print("=" * 80)
|
||||
print(f"[OK] Study created: {study_name}")
|
||||
print("=" * 80)
|
||||
print()
|
||||
print(f"Location: {study_dir}")
|
||||
print()
|
||||
print("Structure:")
|
||||
print(" - 1_setup/: Model files and configuration")
|
||||
print(" - 2_results/: Optimization history and database")
|
||||
print(" - 3_reports/: Human-readable reports with graphs")
|
||||
print()
|
||||
print("To run optimization:")
|
||||
print(f" python {study_dir}/run_optimization.py")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
597
archive/scripts/create_intelligent_study.py
Normal file
597
archive/scripts/create_intelligent_study.py
Normal file
@@ -0,0 +1,597 @@
|
||||
"""
|
||||
Create a new circular plate frequency tuning study using Protocol 10.
|
||||
|
||||
This script creates a complete study configured for intelligent multi-strategy
|
||||
optimization (IMSO) to test the self-tuning framework.
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# Study configuration
|
||||
STUDY_NAME = "circular_plate_frequency_tuning_intelligent_optimizer"
|
||||
BASE_DIR = Path(__file__).parent
|
||||
STUDIES_DIR = BASE_DIR / "studies"
|
||||
STUDY_DIR = STUDIES_DIR / STUDY_NAME
|
||||
|
||||
# Source model files (copy from examples)
|
||||
SOURCE_MODEL_DIR = BASE_DIR / "examples" / "Models" / "Circular Plate"
|
||||
|
||||
def create_study_structure():
|
||||
"""Create complete study directory structure."""
|
||||
print(f"Creating study: {STUDY_NAME}")
|
||||
|
||||
# Create directories
|
||||
setup_dir = STUDY_DIR / "1_setup"
|
||||
model_dir = setup_dir / "model"
|
||||
results_dir = STUDY_DIR / "2_results"
|
||||
reports_dir = STUDY_DIR / "3_reports"
|
||||
|
||||
for directory in [setup_dir, model_dir, results_dir, reports_dir]:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
print(f" Created: {directory.relative_to(BASE_DIR)}")
|
||||
|
||||
return setup_dir, model_dir, results_dir, reports_dir
|
||||
|
||||
|
||||
def copy_model_files(model_dir):
|
||||
"""Copy model files from examples."""
|
||||
print("\nCopying model files...")
|
||||
|
||||
model_files = [
|
||||
"Circular_Plate.prt",
|
||||
"Circular_Plate_sim1.sim",
|
||||
"Circular_Plate_fem1.fem",
|
||||
"Circular_Plate_fem1_i.prt"
|
||||
]
|
||||
|
||||
for filename in model_files:
|
||||
source = SOURCE_MODEL_DIR / filename
|
||||
dest = model_dir / filename
|
||||
|
||||
if source.exists():
|
||||
shutil.copy2(source, dest)
|
||||
print(f" Copied: {filename}")
|
||||
else:
|
||||
print(f" WARNING: Source not found: {filename}")
|
||||
|
||||
return list(model_dir.glob("*"))
|
||||
|
||||
|
||||
def create_workflow_config(setup_dir):
|
||||
"""Create workflow configuration."""
|
||||
print("\nCreating workflow configuration...")
|
||||
|
||||
workflow = {
|
||||
"study_name": STUDY_NAME,
|
||||
"optimization_request": "Tune the first natural frequency mode to exactly 115 Hz using intelligent multi-strategy optimization",
|
||||
"design_variables": [
|
||||
{
|
||||
"parameter": "inner_diameter",
|
||||
"bounds": [50, 150],
|
||||
"units": "mm",
|
||||
"description": "Inner diameter of circular plate"
|
||||
},
|
||||
{
|
||||
"parameter": "plate_thickness",
|
||||
"bounds": [2, 10],
|
||||
"units": "mm",
|
||||
"description": "Thickness of circular plate"
|
||||
}
|
||||
],
|
||||
"objectives": [
|
||||
{
|
||||
"name": "frequency_error",
|
||||
"goal": "minimize",
|
||||
"extraction": {
|
||||
"action": "extract_first_natural_frequency",
|
||||
"params": {
|
||||
"mode_number": 1,
|
||||
"target_frequency": 115.0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"constraints": [
|
||||
{
|
||||
"name": "frequency_tolerance",
|
||||
"type": "less_than",
|
||||
"threshold": 0.1,
|
||||
"description": "Error from target must be less than 0.1 Hz"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
workflow_file = setup_dir / "workflow_config.json"
|
||||
with open(workflow_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(workflow, f, indent=2)
|
||||
|
||||
print(f" Saved: {workflow_file.relative_to(BASE_DIR)}")
|
||||
return workflow
|
||||
|
||||
|
||||
def create_optimization_config(setup_dir):
|
||||
"""Create Protocol 10 optimization configuration."""
|
||||
print("\nCreating Protocol 10 optimization configuration...")
|
||||
|
||||
config = {
|
||||
"_description": "Protocol 10: Intelligent Multi-Strategy Optimization - Circular Plate Test",
|
||||
"_version": "1.0",
|
||||
|
||||
"study_name": STUDY_NAME,
|
||||
"direction": "minimize",
|
||||
|
||||
"intelligent_optimization": {
|
||||
"_description": "Protocol 10 - Automatic landscape analysis and strategy selection",
|
||||
"enabled": True,
|
||||
|
||||
"characterization_trials": 15,
|
||||
"stagnation_window": 10,
|
||||
"min_improvement_threshold": 0.001,
|
||||
"min_analysis_trials": 10,
|
||||
"reanalysis_interval": 15,
|
||||
|
||||
"strategy_preferences": {
|
||||
"prefer_cmaes_for_smooth": True,
|
||||
"prefer_tpe_for_multimodal": True,
|
||||
"enable_hybrid_strategies": False
|
||||
}
|
||||
},
|
||||
|
||||
"sampler": {
|
||||
"_description": "Fallback sampler if Protocol 10 disabled",
|
||||
"type": "TPESampler",
|
||||
"params": {
|
||||
"n_startup_trials": 10,
|
||||
"n_ei_candidates": 24,
|
||||
"multivariate": True,
|
||||
"warn_independent_sampling": True
|
||||
}
|
||||
},
|
||||
|
||||
"pruner": {
|
||||
"type": "MedianPruner",
|
||||
"params": {
|
||||
"n_startup_trials": 5,
|
||||
"n_warmup_steps": 0
|
||||
}
|
||||
},
|
||||
|
||||
"adaptive_strategy": {
|
||||
"_description": "Protocol 8 - Adaptive exploitation based on surrogate confidence",
|
||||
"enabled": True,
|
||||
"min_confidence_for_exploitation": 0.65,
|
||||
"min_trials_for_confidence": 15,
|
||||
"target_confidence_metrics": {
|
||||
"convergence_weight": 0.4,
|
||||
"coverage_weight": 0.3,
|
||||
"stability_weight": 0.3
|
||||
}
|
||||
},
|
||||
|
||||
"trials": {
|
||||
"n_trials": 100,
|
||||
"timeout": None,
|
||||
"catch": []
|
||||
},
|
||||
|
||||
"reporting": {
|
||||
"auto_generate_plots": True,
|
||||
"include_optuna_visualizations": True,
|
||||
"include_confidence_report": True,
|
||||
"include_strategy_performance": True,
|
||||
"save_intelligence_report": True
|
||||
},
|
||||
|
||||
"verbosity": {
|
||||
"print_landscape_report": True,
|
||||
"print_strategy_recommendation": True,
|
||||
"print_phase_transitions": True,
|
||||
"print_confidence_updates": True,
|
||||
"log_to_file": True
|
||||
},
|
||||
|
||||
"optimization_notes": "Protocol 10 Test: Atomizer will automatically characterize the circular plate problem, select the best optimization algorithm (TPE, CMA-ES, or GP-BO), and adapt strategy if stagnation is detected. Expected: smooth_unimodal landscape → CMA-ES recommendation."
|
||||
}
|
||||
|
||||
config_file = setup_dir / "optimization_config.json"
|
||||
with open(config_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
print(f" Saved: {config_file.relative_to(BASE_DIR)}")
|
||||
return config
|
||||
|
||||
|
||||
def create_runner_script(study_dir, workflow, config):
|
||||
"""Create optimization runner using Protocol 10."""
|
||||
print("\nCreating Protocol 10 optimization runner...")
|
||||
|
||||
runner_code = '''"""
|
||||
Intelligent Multi-Strategy Optimization Runner
|
||||
Study: circular_plate_frequency_tuning_intelligent_optimizer
|
||||
|
||||
This runner uses Protocol 10 (IMSO) to automatically:
|
||||
1. Characterize the optimization landscape
|
||||
2. Select the best optimization algorithm
|
||||
3. Adapt strategy dynamically if needed
|
||||
|
||||
Generated: 2025-11-19
|
||||
Protocol: 10 (Intelligent Multi-Strategy Optimization)
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import optuna
|
||||
from pathlib import Path
|
||||
|
||||
# Add optimization engine to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from optimization_engine.intelligent_optimizer import IntelligentOptimizer
|
||||
from optimization_engine.nx_updater import NXParameterUpdater
|
||||
from optimization_engine.nx_solver import NXSolver
|
||||
from optimization_engine.extractors.frequency_extractor import extract_first_frequency
|
||||
from optimization_engine.generate_report_markdown import generate_markdown_report
|
||||
|
||||
|
||||
def main():
|
||||
"""Run Protocol 10 intelligent optimization."""
|
||||
|
||||
# Setup paths
|
||||
study_dir = Path(__file__).parent
|
||||
setup_dir = study_dir / "1_setup"
|
||||
model_dir = setup_dir / "model"
|
||||
results_dir = study_dir / "2_results"
|
||||
reports_dir = study_dir / "3_reports"
|
||||
|
||||
# Create directories
|
||||
results_dir.mkdir(exist_ok=True)
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Load configuration
|
||||
print("\\nLoading configuration...")
|
||||
with open(setup_dir / "workflow_config.json") as f:
|
||||
workflow = json.load(f)
|
||||
|
||||
with open(setup_dir / "optimization_config.json") as f:
|
||||
opt_config = json.load(f)
|
||||
|
||||
print(f"Study: {workflow['study_name']}")
|
||||
print(f"Protocol 10: {opt_config['intelligent_optimization']['enabled']}")
|
||||
|
||||
# Model files
|
||||
prt_file = model_dir / "Circular_Plate.prt"
|
||||
sim_file = model_dir / "Circular_Plate_sim1.sim"
|
||||
|
||||
# Initialize NX components
|
||||
updater = NXParameterUpdater(str(prt_file))
|
||||
solver = NXSolver()
|
||||
|
||||
# Incremental history tracking
|
||||
history_file = results_dir / "optimization_history_incremental.json"
|
||||
history = []
|
||||
|
||||
def objective(trial):
|
||||
"""Objective function for optimization."""
|
||||
|
||||
# Sample design variables
|
||||
inner_diameter = trial.suggest_float('inner_diameter', 50, 150)
|
||||
plate_thickness = trial.suggest_float('plate_thickness', 2, 10)
|
||||
|
||||
params = {
|
||||
'inner_diameter': inner_diameter,
|
||||
'plate_thickness': plate_thickness
|
||||
}
|
||||
|
||||
print(f"\\n Trial #{trial.number}")
|
||||
print(f" Inner Diameter: {inner_diameter:.4f} mm")
|
||||
print(f" Plate Thickness: {plate_thickness:.4f} mm")
|
||||
|
||||
# Update CAD model
|
||||
updater.update_expressions(params)
|
||||
|
||||
# Run simulation (use discovered solution name from benchmarking)
|
||||
result = solver.run_simulation(str(sim_file), solution_name="Solution_Normal_Modes")
|
||||
|
||||
if not result['success']:
|
||||
print(f" Simulation FAILED: {result.get('error', 'Unknown error')}")
|
||||
raise optuna.TrialPruned()
|
||||
|
||||
# Extract frequency
|
||||
op2_file = result['op2_file']
|
||||
frequency = extract_first_frequency(op2_file, mode_number=1)
|
||||
|
||||
# Calculate objective (error from target)
|
||||
target_frequency = 115.0
|
||||
objective_value = abs(frequency - target_frequency)
|
||||
|
||||
print(f" Frequency: {frequency:.4f} Hz")
|
||||
print(f" Target: {target_frequency:.4f} Hz")
|
||||
print(f" Error: {objective_value:.4f} Hz")
|
||||
|
||||
# Save to incremental history
|
||||
trial_data = {
|
||||
"trial_number": trial.number,
|
||||
"design_variables": params,
|
||||
"results": {"first_frequency": frequency},
|
||||
"objective": objective_value
|
||||
}
|
||||
|
||||
history.append(trial_data)
|
||||
|
||||
with open(history_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(history, f, indent=2)
|
||||
|
||||
return objective_value
|
||||
|
||||
# Create intelligent optimizer
|
||||
print("\\n" + "="*70)
|
||||
print(" PROTOCOL 10: INTELLIGENT MULTI-STRATEGY OPTIMIZATION")
|
||||
print("="*70)
|
||||
|
||||
optimizer = IntelligentOptimizer(
|
||||
study_name=workflow['study_name'],
|
||||
study_dir=results_dir,
|
||||
config=opt_config,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Extract design variable bounds
|
||||
design_vars = {
|
||||
var['parameter']: tuple(var['bounds'])
|
||||
for var in workflow['design_variables']
|
||||
}
|
||||
|
||||
# Run optimization
|
||||
results = optimizer.optimize(
|
||||
objective_function=objective,
|
||||
design_variables=design_vars,
|
||||
n_trials=opt_config['trials']['n_trials'],
|
||||
target_value=115.0,
|
||||
tolerance=0.1
|
||||
)
|
||||
|
||||
# Save intelligence report
|
||||
optimizer.save_intelligence_report()
|
||||
|
||||
# Generate markdown report
|
||||
print("\\nGenerating optimization report...")
|
||||
|
||||
# Load study for Optuna visualizations
|
||||
storage = f"sqlite:///{results_dir / 'study.db'}"
|
||||
study = optuna.load_study(
|
||||
study_name=workflow['study_name'],
|
||||
storage=storage
|
||||
)
|
||||
|
||||
report = generate_markdown_report(
|
||||
history_file=history_file,
|
||||
target_value=115.0,
|
||||
tolerance=0.1,
|
||||
reports_dir=reports_dir,
|
||||
study=study
|
||||
)
|
||||
|
||||
report_file = reports_dir / "OPTIMIZATION_REPORT.md"
|
||||
with open(report_file, 'w', encoding='utf-8') as f:
|
||||
f.write(report)
|
||||
|
||||
print(f"\\nReport saved: {report_file}")
|
||||
|
||||
# Print final summary
|
||||
print("\\n" + "="*70)
|
||||
print(" PROTOCOL 10 TEST COMPLETE")
|
||||
print("="*70)
|
||||
print(f"Best Frequency: {results['best_value'] + 115.0:.4f} Hz")
|
||||
print(f"Best Error: {results['best_value']:.4f} Hz")
|
||||
print(f"Best Parameters:")
|
||||
for param, value in results['best_params'].items():
|
||||
print(f" {param}: {value:.4f}")
|
||||
|
||||
if 'landscape_analysis' in results and results['landscape_analysis'].get('ready'):
|
||||
landscape = results['landscape_analysis']
|
||||
print(f"\\nLandscape Type: {landscape['landscape_type'].upper()}")
|
||||
print(f"Recommended Strategy: {results.get('final_strategy', 'N/A').upper()}")
|
||||
|
||||
print("="*70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
runner_file = study_dir / "run_optimization.py"
|
||||
with open(runner_file, 'w', encoding='utf-8') as f:
|
||||
f.write(runner_code)
|
||||
|
||||
print(f" Saved: {runner_file.relative_to(BASE_DIR)}")
|
||||
return runner_file
|
||||
|
||||
|
||||
def create_readme(study_dir):
|
||||
"""Create README for the study."""
|
||||
print("\nCreating README...")
|
||||
|
||||
readme = f"""# {STUDY_NAME}
|
||||
|
||||
**Protocol 10 Test Study** - Intelligent Multi-Strategy Optimization
|
||||
|
||||
## Overview
|
||||
|
||||
This study tests Atomizer's Protocol 10 (IMSO) framework on a circular plate frequency tuning problem.
|
||||
|
||||
**Goal**: Tune first natural frequency to exactly 115 Hz
|
||||
|
||||
**Protocol 10 Features Tested**:
|
||||
- Automatic landscape characterization (smoothness, multimodality, correlation)
|
||||
- Intelligent strategy selection (TPE, CMA-ES, or GP-BO)
|
||||
- Dynamic strategy switching based on stagnation detection
|
||||
- Comprehensive decision logging for transparency
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### Stage 1: Landscape Characterization (Trials 1-15)
|
||||
- Random exploration to gather data
|
||||
- Analyze problem characteristics
|
||||
- Expected classification: `smooth_unimodal` with strong parameter correlation
|
||||
|
||||
### Stage 2: Strategy Selection (Trial 15)
|
||||
- Expected recommendation: **CMA-ES** (92% confidence)
|
||||
- Reasoning: "Smooth unimodal with strong correlation - CMA-ES converges quickly"
|
||||
|
||||
### Stage 3: Adaptive Optimization (Trials 16-100)
|
||||
- Run with CMA-ES sampler
|
||||
- Monitor for stagnation
|
||||
- Switch strategies if needed (unlikely for this problem)
|
||||
|
||||
## Study Structure
|
||||
|
||||
```
|
||||
{STUDY_NAME}/
|
||||
├── 1_setup/
|
||||
│ ├── model/ # CAD and simulation files
|
||||
│ ├── workflow_config.json # Optimization goals
|
||||
│ └── optimization_config.json # Protocol 10 configuration
|
||||
├── 2_results/
|
||||
│ ├── study.db # Optuna database
|
||||
│ ├── optimization_history_incremental.json
|
||||
│ └── intelligent_optimizer/ # Protocol 10 tracking
|
||||
│ ├── strategy_transitions.json
|
||||
│ ├── strategy_performance.json
|
||||
│ └── intelligence_report.json
|
||||
└── 3_reports/
|
||||
└── OPTIMIZATION_REPORT.md # Final report with visualizations
|
||||
```
|
||||
|
||||
## Running the Optimization
|
||||
|
||||
```bash
|
||||
# Activate environment
|
||||
conda activate test_env
|
||||
|
||||
# Run optimization
|
||||
python run_optimization.py
|
||||
```
|
||||
|
||||
## What to Look For
|
||||
|
||||
### Console Output
|
||||
|
||||
**Landscape Analysis Report**:
|
||||
```
|
||||
======================================================================
|
||||
LANDSCAPE ANALYSIS REPORT
|
||||
======================================================================
|
||||
Type: SMOOTH_UNIMODAL
|
||||
Smoothness: 0.7X (smooth)
|
||||
Multimodal: NO (1 modes)
|
||||
Parameter Correlation: 0.6X (strong)
|
||||
```
|
||||
|
||||
**Strategy Recommendation**:
|
||||
```
|
||||
======================================================================
|
||||
STRATEGY RECOMMENDATION
|
||||
======================================================================
|
||||
Recommended: CMAES
|
||||
Confidence: 92.0%
|
||||
Reasoning: Smooth unimodal with strong correlation - CMA-ES converges quickly
|
||||
```
|
||||
|
||||
**Phase Transitions** (if any):
|
||||
```
|
||||
======================================================================
|
||||
STRATEGY TRANSITION
|
||||
======================================================================
|
||||
Trial #45
|
||||
TPE → CMAES
|
||||
Reason: Stagnation detected
|
||||
```
|
||||
|
||||
### Intelligence Report
|
||||
|
||||
Check `2_results/intelligent_optimizer/intelligence_report.json` for:
|
||||
- Complete landscape analysis
|
||||
- Strategy recommendation reasoning
|
||||
- All transition events
|
||||
- Performance breakdown by strategy
|
||||
|
||||
### Optimization Report
|
||||
|
||||
Check `3_reports/OPTIMIZATION_REPORT.md` for:
|
||||
- Best result (should be < 0.1 Hz error)
|
||||
- Convergence plots
|
||||
- Optuna visualizations
|
||||
- (Future: Protocol 10 analysis section)
|
||||
|
||||
## Expected Results
|
||||
|
||||
**Baseline (TPE only)**: ~160 trials to achieve 0.18 Hz error
|
||||
|
||||
**Protocol 10 (Intelligent)**: ~60-80 trials to achieve < 0.1 Hz error
|
||||
|
||||
**Improvement**: 40-50% faster convergence by selecting optimal algorithm
|
||||
|
||||
## Configuration
|
||||
|
||||
See [`optimization_config.json`](1_setup/optimization_config.json) for full Protocol 10 settings.
|
||||
|
||||
Key parameters:
|
||||
- `characterization_trials`: 15 (initial exploration)
|
||||
- `stagnation_window`: 10 (trials to check for stagnation)
|
||||
- `min_improvement_threshold`: 0.001 (0.1% minimum improvement)
|
||||
|
||||
## References
|
||||
|
||||
- [PROTOCOL.md](../../PROTOCOL.md) - Complete Protocol 10 documentation
|
||||
- [Protocol 10 Implementation Summary](../../docs/PROTOCOL_10_IMPLEMENTATION_SUMMARY.md)
|
||||
- [Example Configuration](../../examples/optimization_config_protocol10.json)
|
||||
|
||||
---
|
||||
|
||||
*Study created: 2025-11-19*
|
||||
*Protocol: 10 (Intelligent Multi-Strategy Optimization)*
|
||||
"""
|
||||
|
||||
readme_file = study_dir / "README.md"
|
||||
with open(readme_file, 'w', encoding='utf-8') as f:
|
||||
f.write(readme)
|
||||
|
||||
print(f" Saved: {readme_file.relative_to(BASE_DIR)}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Create complete Protocol 10 test study."""
|
||||
print("\n" + "="*70)
|
||||
print(" CREATING PROTOCOL 10 TEST STUDY")
|
||||
print("="*70)
|
||||
|
||||
# Create structure
|
||||
setup_dir, model_dir, results_dir, reports_dir = create_study_structure()
|
||||
|
||||
# Copy model files
|
||||
model_files = copy_model_files(model_dir)
|
||||
|
||||
# Create configurations
|
||||
workflow = create_workflow_config(setup_dir)
|
||||
config = create_optimization_config(setup_dir)
|
||||
|
||||
# Create runner
|
||||
runner_file = create_runner_script(STUDY_DIR, workflow, config)
|
||||
|
||||
# Create README
|
||||
create_readme(STUDY_DIR)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print(" STUDY CREATION COMPLETE")
|
||||
print("="*70)
|
||||
print(f"\nStudy directory: {STUDY_DIR.relative_to(BASE_DIR)}")
|
||||
print(f"\nTo run optimization:")
|
||||
print(f" cd {STUDY_DIR.relative_to(BASE_DIR)}")
|
||||
print(f" python run_optimization.py")
|
||||
print("\n" + "="*70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
89
archive/scripts/create_v2_study.py
Normal file
89
archive/scripts/create_v2_study.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Create circular_plate_frequency_tuning_V2 study with all fixes.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import json
|
||||
|
||||
# Study configuration
|
||||
study_name = "circular_plate_frequency_tuning_V2"
|
||||
study_dir = Path("studies") / study_name
|
||||
|
||||
# Create study structure
|
||||
print(f"Creating study: {study_name}")
|
||||
print("=" * 80)
|
||||
|
||||
# 1. Create directory structure
|
||||
(study_dir / "1_setup" / "model").mkdir(parents=True, exist_ok=True)
|
||||
(study_dir / "2_results").mkdir(parents=True, exist_ok=True)
|
||||
(study_dir / "3_reports").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 2. Copy model files
|
||||
source_dir = Path("examples/Models/Circular Plate")
|
||||
model_files = [
|
||||
"Circular_Plate.prt",
|
||||
"Circular_Plate_sim1.sim",
|
||||
"Circular_Plate_fem1.fem",
|
||||
"Circular_Plate_fem1_i.prt"
|
||||
]
|
||||
|
||||
print("\n[1/5] Copying model files...")
|
||||
for file in model_files:
|
||||
src = source_dir / file
|
||||
dst = study_dir / "1_setup" / "model" / file
|
||||
if src.exists():
|
||||
shutil.copy2(src, dst)
|
||||
print(f" ✓ {file}")
|
||||
|
||||
# 3. Create workflow config
|
||||
print("\n[2/5] Creating workflow configuration...")
|
||||
workflow = {
|
||||
"study_name": study_name,
|
||||
"optimization_request": "Tune the first natural frequency mode to exactly 115 Hz (within 0.1 Hz tolerance)",
|
||||
"design_variables": [
|
||||
{
|
||||
"parameter": "inner_diameter",
|
||||
"bounds": [50, 150]
|
||||
},
|
||||
{
|
||||
"parameter": "plate_thickness",
|
||||
"bounds": [2, 10]
|
||||
}
|
||||
],
|
||||
"objectives": [
|
||||
{
|
||||
"name": "frequency_error",
|
||||
"goal": "minimize",
|
||||
"extraction": {
|
||||
"action": "extract_first_natural_frequency",
|
||||
"params": {
|
||||
"mode_number": 1,
|
||||
"target_frequency": 115.0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"constraints": [
|
||||
{
|
||||
"name": "frequency_tolerance",
|
||||
"type": "less_than",
|
||||
"threshold": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
config_file = study_dir / "1_setup" / "workflow_config.json"
|
||||
with open(config_file, 'w') as f:
|
||||
json.dump(workflow, f, indent=2)
|
||||
print(f" ✓ Configuration saved")
|
||||
|
||||
print("\n[3/5] Study structure created")
|
||||
print(f" Location: {study_dir}")
|
||||
print(f" - 1_setup/model: Model files")
|
||||
print(f" - 2_results: Optimization results")
|
||||
print(f" - 3_reports: Human-readable reports")
|
||||
|
||||
print("\n[4/5] Next: Run intelligent setup to generate optimization runner")
|
||||
print(f" Command: python create_circular_plate_study.py --study-name {study_name}")
|
||||
|
||||
print("\nDone!")
|
||||
60
archive/scripts/extract_history_from_db.py
Normal file
60
archive/scripts/extract_history_from_db.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Extract optimization history from Optuna database and create incremental JSON file.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
import optuna
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python extract_history_from_db.py <path/to/study.db>")
|
||||
sys.exit(1)
|
||||
|
||||
db_file = Path(sys.argv[1])
|
||||
if not db_file.exists():
|
||||
print(f"ERROR: Database not found: {db_file}")
|
||||
sys.exit(1)
|
||||
|
||||
# Load Optuna study
|
||||
storage = f"sqlite:///{db_file}"
|
||||
study_name = db_file.parent.parent.name # Extract from path
|
||||
|
||||
try:
|
||||
study = optuna.load_study(study_name=study_name, storage=storage)
|
||||
except:
|
||||
# Try to get first study if name doesn't match
|
||||
studies = optuna.get_all_study_names(storage)
|
||||
if not studies:
|
||||
print("ERROR: No studies found in database")
|
||||
sys.exit(1)
|
||||
study = optuna.load_study(study_name=studies[0], storage=storage)
|
||||
|
||||
print(f"Study: {study.study_name}")
|
||||
print(f"Trials: {len(study.trials)}")
|
||||
|
||||
# Extract history
|
||||
history = []
|
||||
for trial in study.trials:
|
||||
if trial.state != optuna.trial.TrialState.COMPLETE:
|
||||
continue
|
||||
|
||||
record = {
|
||||
'trial_number': trial.number,
|
||||
'design_variables': trial.params,
|
||||
'results': trial.user_attrs, # May be empty if not stored
|
||||
'objective': trial.value
|
||||
}
|
||||
history.append(record)
|
||||
|
||||
# Write to JSON
|
||||
output_file = db_file.parent / 'optimization_history_incremental.json'
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(history, f, indent=2)
|
||||
|
||||
print(f"\nHistory exported to: {output_file}")
|
||||
print(f" {len(history)} completed trials")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
46
archive/scripts/monitor_e2e.py
Normal file
46
archive/scripts/monitor_e2e.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Monitor E2E test progress in real-time
|
||||
|
||||
Run this in a separate terminal to see live updates
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Load API key from .env
|
||||
env_vars = {}
|
||||
env_file = Path(".env")
|
||||
if env_file.exists():
|
||||
with open(env_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
env_vars[key.strip()] = value.strip()
|
||||
|
||||
if 'ANTHROPIC_API_KEY' not in env_vars:
|
||||
print("[FAIL] No API key found in .env")
|
||||
sys.exit(1)
|
||||
|
||||
print("[OK] API key loaded")
|
||||
print()
|
||||
print("=" * 80)
|
||||
print("RUNNING E2E TEST - LIVE OUTPUT")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
# Run test with live output (not in background)
|
||||
python_exe = "c:/Users/antoi/anaconda3/envs/test_env/python.exe"
|
||||
test_script = Path(__file__).parent / "tests" / "test_phase_3_2_e2e.py"
|
||||
|
||||
# Set environment and run
|
||||
import os
|
||||
os.environ.update(env_vars)
|
||||
|
||||
# Run with unbuffered output so we see it live
|
||||
result = subprocess.run(
|
||||
[python_exe, "-u", str(test_script)],
|
||||
env=os.environ
|
||||
)
|
||||
|
||||
sys.exit(result.returncode)
|
||||
97
archive/scripts/reorganize_study.py
Normal file
97
archive/scripts/reorganize_study.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
Reorganize simple_beam_optimization study to new structure.
|
||||
Handles locked files gracefully.
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
study_dir = Path("studies/simple_beam_optimization")
|
||||
|
||||
# Check current state
|
||||
print("Current directory structure:")
|
||||
print(f" 1_setup exists: {(study_dir / '1_setup').exists()}")
|
||||
print(f" 2_substudies exists: {(study_dir / '2_substudies').exists()}")
|
||||
print(f" 3_reports exists: {(study_dir / '3_reports').exists()}")
|
||||
print()
|
||||
|
||||
# Copy full_optimization_50trials if not already done
|
||||
src = study_dir / "substudies" / "full_optimization_50trials"
|
||||
dst = study_dir / "2_substudies" / "04_full_optimization_50trials"
|
||||
|
||||
if src.exists() and not dst.exists():
|
||||
print(f"Copying {src.name} to {dst.name}...")
|
||||
try:
|
||||
shutil.copytree(src, dst)
|
||||
print(f" SUCCESS: Copied to {dst}")
|
||||
except Exception as e:
|
||||
print(f" WARNING: {e}")
|
||||
print(f" Will attempt to continue...")
|
||||
|
||||
# Move OPTIMIZATION_RESULTS_50TRIALS.md
|
||||
old_results_file = study_dir / "OPTIMIZATION_RESULTS_50TRIALS.md"
|
||||
new_results_file = dst / "OPTIMIZATION_RESULTS.md"
|
||||
|
||||
if old_results_file.exists() and not new_results_file.exists():
|
||||
print(f"\nMoving {old_results_file.name}...")
|
||||
try:
|
||||
shutil.move(str(old_results_file), str(new_results_file))
|
||||
print(f" SUCCESS: Moved to {new_results_file}")
|
||||
except Exception as e:
|
||||
print(f" WARNING: {e}")
|
||||
|
||||
# Move COMPREHENSIVE_BENCHMARK_RESULTS.md
|
||||
old_bench_file = study_dir / "COMPREHENSIVE_BENCHMARK_RESULTS.md"
|
||||
new_bench_file = study_dir / "3_reports" / "COMPREHENSIVE_BENCHMARK_RESULTS.md"
|
||||
|
||||
if old_bench_file.exists() and not new_bench_file.exists():
|
||||
print(f"\nMoving {old_bench_file.name}...")
|
||||
try:
|
||||
shutil.move(str(old_bench_file), str(new_bench_file))
|
||||
print(f" SUCCESS: Moved to {new_bench_file}")
|
||||
except Exception as e:
|
||||
print(f" WARNING: {e}")
|
||||
|
||||
# Try to remove old substudies directory (may fail due to locked files - that's OK)
|
||||
old_substudies = study_dir / "substudies"
|
||||
if old_substudies.exists():
|
||||
print(f"\nAttempting to remove old 'substudies' directory...")
|
||||
try:
|
||||
# Try multiple times in case files get unlocked
|
||||
for attempt in range(3):
|
||||
try:
|
||||
shutil.rmtree(old_substudies)
|
||||
print(f" SUCCESS: Removed old 'substudies' directory")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
print(f" Attempt {attempt + 1} failed, retrying in 1 second...")
|
||||
time.sleep(1)
|
||||
else:
|
||||
print(f" INFO: Could not remove old 'substudies' directory (files may be locked)")
|
||||
print(f" You can manually delete it later: {old_substudies}")
|
||||
except Exception as e:
|
||||
print(f" WARNING: {e}")
|
||||
|
||||
# Remove old model directory if empty
|
||||
old_model = study_dir / "model"
|
||||
if old_model.exists() and not list(old_model.iterdir()):
|
||||
print(f"\nRemoving empty 'model' directory...")
|
||||
try:
|
||||
old_model.rmdir()
|
||||
print(f" SUCCESS: Removed empty 'model' directory")
|
||||
except Exception as e:
|
||||
print(f" WARNING: {e}")
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("Reorganization complete!")
|
||||
print("="*70)
|
||||
print("\nNew structure:")
|
||||
print(" 1_setup/ - Pre-optimization setup")
|
||||
print(" 2_substudies/ - Optimization runs (numbered)")
|
||||
print(" 3_reports/ - Study-level analysis")
|
||||
print()
|
||||
print("Next steps:")
|
||||
print(" 1. Update study_metadata.json")
|
||||
print(" 2. Create substudy README files")
|
||||
print(" 3. Delete old 'substudies' folder manually if it still exists")
|
||||
331
archive/scripts/run_calibration_loop.py
Normal file
331
archive/scripts/run_calibration_loop.py
Normal file
@@ -0,0 +1,331 @@
|
||||
"""
|
||||
Active Learning Calibration Loop
|
||||
|
||||
This script implements the iterative calibration workflow:
|
||||
1. Train initial NN on existing FEA data
|
||||
2. Run NN optimization to find promising designs
|
||||
3. Select high-uncertainty designs for FEA validation
|
||||
4. Run FEA on selected designs (simulated here, needs real FEA integration)
|
||||
5. Retrain NN with new data
|
||||
6. Repeat until confidence threshold reached
|
||||
|
||||
Usage:
|
||||
python run_calibration_loop.py --study uav_arm_optimization --iterations 5
|
||||
|
||||
Note: For actual FEA integration, replace the simulate_fea() function with real NX calls.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import json
|
||||
import numpy as np
|
||||
import optuna
|
||||
from optuna.samplers import NSGAIISampler
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Add project paths
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from optimization_engine.active_learning_surrogate import (
|
||||
ActiveLearningSurrogate,
|
||||
extract_training_data_from_study
|
||||
)
|
||||
|
||||
|
||||
def simulate_fea(design: dict, surrogate: ActiveLearningSurrogate) -> dict:
|
||||
"""
|
||||
PLACEHOLDER: Simulate FEA results.
|
||||
|
||||
In production, this would:
|
||||
1. Update NX model parameters
|
||||
2. Run FEA solve
|
||||
3. Extract results (mass, frequency, displacement, stress)
|
||||
|
||||
For now, we use the ensemble mean + noise to simulate "ground truth"
|
||||
with systematic differences to test calibration.
|
||||
"""
|
||||
# Get NN prediction
|
||||
pred = surrogate.predict(design)
|
||||
|
||||
# Add systematic bias + noise to simulate FEA
|
||||
# This simulates the case where NN is systematically off
|
||||
fea_mass = pred['mass'] * 0.95 + np.random.normal(0, 50) # NN overestimates mass by ~5%
|
||||
fea_freq = pred['frequency'] * 0.6 + np.random.normal(0, 2) # NN overestimates freq significantly
|
||||
|
||||
return {
|
||||
'mass': max(fea_mass, 1000), # Ensure positive
|
||||
'frequency': max(fea_freq, 1),
|
||||
'max_displacement': pred.get('max_displacement', 0),
|
||||
'max_stress': pred.get('max_stress', 0)
|
||||
}
|
||||
|
||||
|
||||
def run_nn_optimization(
|
||||
surrogate: ActiveLearningSurrogate,
|
||||
bounds: dict,
|
||||
n_trials: int = 500
|
||||
) -> list:
|
||||
"""Run NN-only optimization to generate candidate designs."""
|
||||
|
||||
study = optuna.create_study(
|
||||
directions=["minimize", "minimize"], # mass, -frequency
|
||||
sampler=NSGAIISampler()
|
||||
)
|
||||
|
||||
def objective(trial):
|
||||
params = {}
|
||||
for name, (low, high) in bounds.items():
|
||||
if name == 'hole_count':
|
||||
params[name] = trial.suggest_int(name, int(low), int(high))
|
||||
else:
|
||||
params[name] = trial.suggest_float(name, low, high)
|
||||
|
||||
pred = surrogate.predict(params)
|
||||
|
||||
# Store uncertainty in user_attrs
|
||||
trial.set_user_attr('uncertainty', pred['total_uncertainty'])
|
||||
trial.set_user_attr('params', params)
|
||||
|
||||
return pred['mass'], -pred['frequency']
|
||||
|
||||
study.optimize(objective, n_trials=n_trials, show_progress_bar=False)
|
||||
|
||||
# Extract Pareto front designs with their uncertainty
|
||||
pareto_designs = []
|
||||
for trial in study.best_trials:
|
||||
pareto_designs.append({
|
||||
'params': trial.user_attrs['params'],
|
||||
'uncertainty': trial.user_attrs['uncertainty'],
|
||||
'mass': trial.values[0],
|
||||
'frequency': -trial.values[1]
|
||||
})
|
||||
|
||||
return pareto_designs
|
||||
|
||||
|
||||
def plot_calibration_progress(history: list, save_path: str):
|
||||
"""Plot calibration progress over iterations."""
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
|
||||
|
||||
iterations = [h['iteration'] for h in history]
|
||||
|
||||
# 1. Confidence score
|
||||
ax = axes[0, 0]
|
||||
confidence = [h['confidence_score'] for h in history]
|
||||
ax.plot(iterations, confidence, 'b-o', linewidth=2)
|
||||
ax.axhline(y=0.7, color='g', linestyle='--', label='Target (0.7)')
|
||||
ax.set_xlabel('Iteration')
|
||||
ax.set_ylabel('Confidence Score')
|
||||
ax.set_title('Model Confidence Over Iterations')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 2. MAPE
|
||||
ax = axes[0, 1]
|
||||
mass_mape = [h['mass_mape'] for h in history]
|
||||
freq_mape = [h['freq_mape'] for h in history]
|
||||
ax.plot(iterations, mass_mape, 'b-o', label='Mass MAPE')
|
||||
ax.plot(iterations, freq_mape, 'r-s', label='Frequency MAPE')
|
||||
ax.axhline(y=10, color='g', linestyle='--', label='Target (10%)')
|
||||
ax.set_xlabel('Iteration')
|
||||
ax.set_ylabel('MAPE (%)')
|
||||
ax.set_title('Prediction Error Over Iterations')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 3. Training samples
|
||||
ax = axes[1, 0]
|
||||
n_samples = [h['n_training_samples'] for h in history]
|
||||
ax.plot(iterations, n_samples, 'g-o', linewidth=2)
|
||||
ax.set_xlabel('Iteration')
|
||||
ax.set_ylabel('Training Samples')
|
||||
ax.set_title('Training Data Growth')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 4. Average uncertainty of selected designs
|
||||
ax = axes[1, 1]
|
||||
avg_uncertainty = [h['avg_selected_uncertainty'] for h in history]
|
||||
ax.plot(iterations, avg_uncertainty, 'm-o', linewidth=2)
|
||||
ax.set_xlabel('Iteration')
|
||||
ax.set_ylabel('Average Uncertainty')
|
||||
ax.set_title('Uncertainty of Selected Designs')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
plt.suptitle('Active Learning Calibration Progress', fontsize=14)
|
||||
plt.tight_layout()
|
||||
plt.savefig(save_path, dpi=150)
|
||||
plt.close()
|
||||
print(f"Saved calibration progress plot: {save_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Run Active Learning Calibration Loop')
|
||||
parser.add_argument('--study', default='uav_arm_optimization', help='Study name')
|
||||
parser.add_argument('--iterations', type=int, default=5, help='Number of calibration iterations')
|
||||
parser.add_argument('--fea-per-iter', type=int, default=10, help='FEA evaluations per iteration')
|
||||
parser.add_argument('--confidence-target', type=float, default=0.7, help='Target confidence')
|
||||
parser.add_argument('--simulate', action='store_true', default=True, help='Simulate FEA (for testing)')
|
||||
args = parser.parse_args()
|
||||
|
||||
print("="*70)
|
||||
print("Active Learning Calibration Loop")
|
||||
print("="*70)
|
||||
print(f"Study: {args.study}")
|
||||
print(f"Max iterations: {args.iterations}")
|
||||
print(f"FEA per iteration: {args.fea_per_iter}")
|
||||
print(f"Confidence target: {args.confidence_target}")
|
||||
|
||||
# Find database
|
||||
db_path = project_root / f"studies/{args.study}/2_results/study.db"
|
||||
study_name = args.study
|
||||
|
||||
if not db_path.exists():
|
||||
db_path = project_root / "studies/uav_arm_atomizerfield_test/2_results/study.db"
|
||||
study_name = "uav_arm_atomizerfield_test"
|
||||
|
||||
if not db_path.exists():
|
||||
print(f"ERROR: Database not found: {db_path}")
|
||||
return
|
||||
|
||||
# Design bounds (from UAV arm study)
|
||||
bounds = {
|
||||
'beam_half_core_thickness': (1.0, 10.0),
|
||||
'beam_face_thickness': (0.5, 3.0),
|
||||
'holes_diameter': (0.5, 50.0),
|
||||
'hole_count': (6, 14)
|
||||
}
|
||||
|
||||
# Load initial training data
|
||||
print(f"\n[1] Loading initial training data from {db_path}")
|
||||
design_params, objectives, design_var_names = extract_training_data_from_study(
|
||||
str(db_path), study_name
|
||||
)
|
||||
print(f" Initial samples: {len(design_params)}")
|
||||
|
||||
# Calibration history
|
||||
calibration_history = []
|
||||
|
||||
# Track accumulated training data
|
||||
all_design_params = design_params.copy()
|
||||
all_objectives = objectives.copy()
|
||||
|
||||
for iteration in range(args.iterations):
|
||||
print(f"\n{'='*70}")
|
||||
print(f"ITERATION {iteration + 1}/{args.iterations}")
|
||||
print("="*70)
|
||||
|
||||
# Train ensemble surrogate
|
||||
print(f"\n[2.{iteration+1}] Training ensemble surrogate...")
|
||||
surrogate = ActiveLearningSurrogate(n_ensemble=5)
|
||||
surrogate.train(
|
||||
all_design_params, all_objectives, design_var_names,
|
||||
epochs=200
|
||||
)
|
||||
|
||||
# Run NN optimization to find candidate designs
|
||||
print(f"\n[3.{iteration+1}] Running NN optimization (500 trials)...")
|
||||
pareto_designs = run_nn_optimization(surrogate, bounds, n_trials=500)
|
||||
print(f" Found {len(pareto_designs)} Pareto designs")
|
||||
|
||||
# Select designs for FEA validation (highest uncertainty)
|
||||
print(f"\n[4.{iteration+1}] Selecting designs for FEA validation...")
|
||||
candidate_params = [d['params'] for d in pareto_designs]
|
||||
selected = surrogate.select_designs_for_validation(
|
||||
candidate_params,
|
||||
n_select=args.fea_per_iter,
|
||||
strategy='diverse' # Mix of high uncertainty + diversity
|
||||
)
|
||||
|
||||
print(f" Selected {len(selected)} designs:")
|
||||
avg_uncertainty = np.mean([s[2] for s in selected])
|
||||
for i, (idx, params, uncertainty) in enumerate(selected[:5]):
|
||||
print(f" {i+1}. Uncertainty={uncertainty:.3f}, params={params}")
|
||||
|
||||
# Run FEA (simulated)
|
||||
print(f"\n[5.{iteration+1}] Running FEA validation...")
|
||||
new_params = []
|
||||
new_objectives = []
|
||||
|
||||
for idx, params, uncertainty in selected:
|
||||
if args.simulate:
|
||||
fea_result = simulate_fea(params, surrogate)
|
||||
else:
|
||||
# TODO: Call actual FEA here
|
||||
# fea_result = run_actual_fea(params)
|
||||
raise NotImplementedError("Real FEA not implemented")
|
||||
|
||||
# Record for retraining
|
||||
param_array = [params.get(name, 0.0) for name in design_var_names]
|
||||
new_params.append(param_array)
|
||||
new_objectives.append([
|
||||
fea_result['mass'],
|
||||
fea_result['frequency'],
|
||||
fea_result.get('max_displacement', 0),
|
||||
fea_result.get('max_stress', 0)
|
||||
])
|
||||
|
||||
# Update validation tracking
|
||||
surrogate.update_with_validation([params], [fea_result])
|
||||
|
||||
# Add new data to training set
|
||||
all_design_params = np.vstack([all_design_params, np.array(new_params, dtype=np.float32)])
|
||||
all_objectives = np.vstack([all_objectives, np.array(new_objectives, dtype=np.float32)])
|
||||
|
||||
# Get confidence report
|
||||
report = surrogate.get_confidence_report()
|
||||
print(f"\n[6.{iteration+1}] Confidence Report:")
|
||||
print(f" Confidence Score: {report['confidence_score']:.3f}")
|
||||
print(f" Mass MAPE: {report['mass_mape']:.1f}%")
|
||||
print(f" Freq MAPE: {report['freq_mape']:.1f}%")
|
||||
print(f" Status: {report['status']}")
|
||||
print(f" Recommendation: {report['recommendation']}")
|
||||
|
||||
# Record history
|
||||
calibration_history.append({
|
||||
'iteration': iteration + 1,
|
||||
'n_training_samples': len(all_design_params),
|
||||
'confidence_score': report['confidence_score'],
|
||||
'mass_mape': report['mass_mape'],
|
||||
'freq_mape': report['freq_mape'],
|
||||
'avg_selected_uncertainty': avg_uncertainty,
|
||||
'status': report['status']
|
||||
})
|
||||
|
||||
# Check if we've reached target confidence
|
||||
if report['confidence_score'] >= args.confidence_target:
|
||||
print(f"\n*** TARGET CONFIDENCE REACHED ({report['confidence_score']:.3f} >= {args.confidence_target}) ***")
|
||||
break
|
||||
|
||||
# Save final model
|
||||
print("\n" + "="*70)
|
||||
print("CALIBRATION COMPLETE")
|
||||
print("="*70)
|
||||
|
||||
model_path = project_root / "calibrated_surrogate.pt"
|
||||
surrogate.save(str(model_path))
|
||||
print(f"Saved calibrated model to: {model_path}")
|
||||
|
||||
# Save calibration history
|
||||
history_path = project_root / "calibration_history.json"
|
||||
with open(history_path, 'w') as f:
|
||||
json.dump(calibration_history, f, indent=2)
|
||||
print(f"Saved calibration history to: {history_path}")
|
||||
|
||||
# Plot progress
|
||||
plot_calibration_progress(calibration_history, str(project_root / "calibration_progress.png"))
|
||||
|
||||
# Final summary
|
||||
final_report = surrogate.get_confidence_report()
|
||||
print(f"\nFinal Results:")
|
||||
print(f" Training samples: {len(all_design_params)}")
|
||||
print(f" Confidence score: {final_report['confidence_score']:.3f}")
|
||||
print(f" Mass MAPE: {final_report['mass_mape']:.1f}%")
|
||||
print(f" Freq MAPE: {final_report['freq_mape']:.1f}%")
|
||||
print(f" Ready for optimization: {surrogate.is_ready_for_optimization()}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
3
archive/scripts/run_e2e_test.bat
Normal file
3
archive/scripts/run_e2e_test.bat
Normal file
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
set ANTHROPIC_API_KEY=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
|
||||
"c:/Users/antoi/anaconda3/envs/test_env/python.exe" tests/test_phase_3_2_e2e.py
|
||||
43
archive/scripts/run_e2e_with_env.py
Normal file
43
archive/scripts/run_e2e_with_env.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Helper script to run E2E test with API key from .env file
|
||||
|
||||
This script loads the ANTHROPIC_API_KEY from .env and runs the E2E test.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
# Load .env file
|
||||
env_file = Path(__file__).parent / ".env"
|
||||
|
||||
if env_file.exists():
|
||||
print("Loading API key from .env file...")
|
||||
with open(env_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
os.environ[key.strip()] = value.strip()
|
||||
|
||||
if 'ANTHROPIC_API_KEY' in os.environ:
|
||||
print(f"[OK] API key loaded: {os.environ['ANTHROPIC_API_KEY'][:20]}...")
|
||||
else:
|
||||
print("[FAIL] No ANTHROPIC_API_KEY found in .env")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"[FAIL] .env file not found at {env_file}")
|
||||
print("\nPlease create a .env file with your API key:")
|
||||
print("ANTHROPIC_API_KEY=your-key-here")
|
||||
sys.exit(1)
|
||||
|
||||
# Run the E2E test
|
||||
print("\nRunning E2E test...")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
python_exe = "c:/Users/antoi/anaconda3/envs/test_env/python.exe"
|
||||
test_script = Path(__file__).parent / "tests" / "test_phase_3_2_e2e.py"
|
||||
|
||||
result = subprocess.run([python_exe, str(test_script)])
|
||||
sys.exit(result.returncode)
|
||||
163
archive/scripts/run_nn_optimization.py
Normal file
163
archive/scripts/run_nn_optimization.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Neural Network Only Optimization
|
||||
|
||||
This script runs multi-objective optimization using ONLY the neural network
|
||||
surrogate (no FEA). This demonstrates the speed improvement from NN predictions.
|
||||
|
||||
Objectives:
|
||||
- Minimize mass
|
||||
- Maximize frequency (minimize -frequency)
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import time
|
||||
import json
|
||||
import optuna
|
||||
from optuna.samplers import NSGAIISampler
|
||||
import numpy as np
|
||||
|
||||
# Add project paths
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
sys.path.insert(0, str(project_root / 'atomizer-field'))
|
||||
|
||||
from optimization_engine.simple_mlp_surrogate import SimpleSurrogate
|
||||
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print("Neural Network Only Optimization (Simple MLP)")
|
||||
print("="*60)
|
||||
|
||||
# Load surrogate
|
||||
print("\n[1] Loading neural surrogate...")
|
||||
model_path = project_root / "simple_mlp_surrogate.pt"
|
||||
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Model not found at {model_path}")
|
||||
print("Run 'python optimization_engine/simple_mlp_surrogate.py' first to train")
|
||||
return
|
||||
|
||||
surrogate = SimpleSurrogate.load(model_path)
|
||||
|
||||
if not surrogate:
|
||||
print("ERROR: Could not load neural surrogate")
|
||||
return
|
||||
|
||||
print(f" Design variables: {surrogate.design_var_names}")
|
||||
|
||||
# Define bounds (from UAV arm study)
|
||||
bounds = {
|
||||
'beam_half_core_thickness': (1.0, 5.0),
|
||||
'beam_face_thickness': (0.5, 3.0),
|
||||
'holes_diameter': (0.5, 5.0),
|
||||
'hole_count': (0.0, 6.0)
|
||||
}
|
||||
|
||||
print(f" Bounds: {bounds}")
|
||||
|
||||
# Create Optuna study
|
||||
print("\n[2] Creating Optuna study...")
|
||||
storage_path = project_root / "nn_only_optimization_study.db"
|
||||
|
||||
# Remove old study if exists
|
||||
if storage_path.exists():
|
||||
storage_path.unlink()
|
||||
|
||||
storage = optuna.storages.RDBStorage(f"sqlite:///{storage_path}")
|
||||
|
||||
study = optuna.create_study(
|
||||
study_name="nn_only_optimization",
|
||||
storage=storage,
|
||||
directions=["minimize", "minimize"], # mass, -frequency (minimize both)
|
||||
sampler=NSGAIISampler()
|
||||
)
|
||||
|
||||
# Track stats
|
||||
start_time = time.time()
|
||||
trial_times = []
|
||||
|
||||
def objective(trial: optuna.Trial):
|
||||
trial_start = time.time()
|
||||
|
||||
# Suggest parameters
|
||||
params = {}
|
||||
for name, (low, high) in bounds.items():
|
||||
if name == 'hole_count':
|
||||
params[name] = trial.suggest_int(name, int(low), int(high))
|
||||
else:
|
||||
params[name] = trial.suggest_float(name, low, high)
|
||||
|
||||
# Predict with NN
|
||||
results = surrogate.predict(params)
|
||||
|
||||
mass = results['mass']
|
||||
frequency = results['frequency']
|
||||
|
||||
trial_time = (time.time() - trial_start) * 1000
|
||||
trial_times.append(trial_time)
|
||||
|
||||
# Log progress every 100 trials
|
||||
if trial.number % 100 == 0:
|
||||
print(f" Trial {trial.number}: mass={mass:.1f}g, freq={frequency:.2f}Hz, time={trial_time:.1f}ms")
|
||||
|
||||
# Return objectives: minimize mass, minimize -frequency (= maximize frequency)
|
||||
return mass, -frequency
|
||||
|
||||
# Run optimization
|
||||
n_trials = 1000 # Much faster with NN!
|
||||
print(f"\n[3] Running {n_trials} trials...")
|
||||
|
||||
study.optimize(objective, n_trials=n_trials, show_progress_bar=True)
|
||||
|
||||
total_time = time.time() - start_time
|
||||
|
||||
# Results
|
||||
print("\n" + "="*60)
|
||||
print("RESULTS")
|
||||
print("="*60)
|
||||
|
||||
print(f"\nTotal time: {total_time:.1f}s for {n_trials} trials")
|
||||
print(f"Average time per trial: {np.mean(trial_times):.1f}ms")
|
||||
print(f"Trials per second: {n_trials/total_time:.1f}")
|
||||
|
||||
# Get Pareto front
|
||||
pareto_front = study.best_trials
|
||||
print(f"\nPareto front size: {len(pareto_front)} designs")
|
||||
|
||||
print("\nTop 5 Pareto-optimal designs:")
|
||||
for i, trial in enumerate(pareto_front[:5]):
|
||||
mass = trial.values[0]
|
||||
freq = -trial.values[1] # Convert back to positive
|
||||
print(f" {i+1}. Mass={mass:.1f}g, Freq={freq:.2f}Hz")
|
||||
print(f" Params: {trial.params}")
|
||||
|
||||
# Save results
|
||||
results_file = project_root / "nn_optimization_results.json"
|
||||
results = {
|
||||
'n_trials': n_trials,
|
||||
'total_time_s': total_time,
|
||||
'avg_trial_time_ms': np.mean(trial_times),
|
||||
'trials_per_second': n_trials/total_time,
|
||||
'pareto_front_size': len(pareto_front),
|
||||
'pareto_designs': [
|
||||
{
|
||||
'mass': t.values[0],
|
||||
'frequency': -t.values[1],
|
||||
'params': t.params
|
||||
}
|
||||
for t in pareto_front
|
||||
]
|
||||
}
|
||||
|
||||
with open(results_file, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
print(f"\nResults saved to: {results_file}")
|
||||
print(f"Study database: {storage_path}")
|
||||
print("\nView in Optuna dashboard:")
|
||||
print(f" optuna-dashboard sqlite:///{storage_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
470
archive/scripts/run_validated_nn_optimization.py
Normal file
470
archive/scripts/run_validated_nn_optimization.py
Normal file
@@ -0,0 +1,470 @@
|
||||
"""
|
||||
Production-Ready NN Optimization with Confidence Bounds
|
||||
|
||||
This script runs multi-objective optimization using the CV-validated neural network
|
||||
with proper extrapolation warnings and confidence-bounded results.
|
||||
|
||||
Key Features:
|
||||
1. Uses CV-validated model with known accuracy (1.8% mass, 1.1% freq MAPE)
|
||||
2. Warns when extrapolating outside training data range
|
||||
3. Reads optimization bounds from study's optimization_config.json
|
||||
4. Constrains optimization to prescribed bounds for reliable predictions
|
||||
5. Marks designs needing FEA validation
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import time
|
||||
import json
|
||||
import argparse
|
||||
import torch
|
||||
import numpy as np
|
||||
import optuna
|
||||
from optuna.samplers import NSGAIISampler
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Add project paths
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
def load_config_bounds(study_path: Path) -> dict:
|
||||
"""Load design variable bounds from optimization_config.json
|
||||
|
||||
Returns dict: {param_name: (min, max, is_int)}
|
||||
"""
|
||||
config_path = study_path / "1_setup" / "optimization_config.json"
|
||||
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Config not found: {config_path}")
|
||||
|
||||
with open(config_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
bounds = {}
|
||||
for var in config.get('design_variables', []):
|
||||
# Support both 'parameter' and 'name' keys
|
||||
name = var.get('parameter') or var.get('name')
|
||||
|
||||
# Support both "bounds": [min, max] and "min_value"/"max_value" formats
|
||||
if 'bounds' in var:
|
||||
min_val, max_val = var['bounds']
|
||||
else:
|
||||
min_val = var.get('min_value', var.get('min', 0))
|
||||
max_val = var.get('max_value', var.get('max', 1))
|
||||
|
||||
# Detect integer type based on name or explicit type
|
||||
is_int = (var.get('type') == 'integer' or
|
||||
'count' in name.lower() or
|
||||
(isinstance(min_val, int) and isinstance(max_val, int) and max_val - min_val < 20))
|
||||
|
||||
bounds[name] = (min_val, max_val, is_int)
|
||||
|
||||
return bounds
|
||||
|
||||
from optimization_engine.active_learning_surrogate import EnsembleMLP
|
||||
|
||||
|
||||
class ValidatedSurrogate:
|
||||
"""Surrogate with CV-validated accuracy and extrapolation detection."""
|
||||
|
||||
def __init__(self, model_path: str):
|
||||
state = torch.load(model_path, map_location='cpu')
|
||||
|
||||
self.model = EnsembleMLP(
|
||||
input_dim=len(state['design_var_names']),
|
||||
output_dim=4, # mass, freq, disp, stress
|
||||
hidden_dims=state['hidden_dims']
|
||||
)
|
||||
self.model.load_state_dict(state['model'])
|
||||
self.model.eval()
|
||||
|
||||
self.input_mean = np.array(state['input_mean'])
|
||||
self.input_std = np.array(state['input_std'])
|
||||
self.output_mean = np.array(state['output_mean'])
|
||||
self.output_std = np.array(state['output_std'])
|
||||
self.design_var_names = state['design_var_names']
|
||||
|
||||
# CV metrics
|
||||
self.cv_mass_mape = state['cv_mass_mape']
|
||||
self.cv_freq_mape = state['cv_freq_mape']
|
||||
self.cv_mass_std = state['cv_mass_std']
|
||||
self.cv_freq_std = state['cv_freq_std']
|
||||
self.n_training_samples = state['n_samples']
|
||||
|
||||
# Training bounds (for extrapolation detection)
|
||||
self.bounds_min = self.input_mean - 2 * self.input_std
|
||||
self.bounds_max = self.input_mean + 2 * self.input_std
|
||||
|
||||
def predict(self, params: dict) -> dict:
|
||||
"""Predict with extrapolation check."""
|
||||
x = np.array([[params.get(name, 0.0) for name in self.design_var_names]], dtype=np.float32)
|
||||
|
||||
# Check for extrapolation
|
||||
extrapolation_score = 0.0
|
||||
for i, name in enumerate(self.design_var_names):
|
||||
val = x[0, i]
|
||||
if val < self.bounds_min[i]:
|
||||
extrapolation_score += (self.bounds_min[i] - val) / (self.input_std[i] + 1e-8)
|
||||
elif val > self.bounds_max[i]:
|
||||
extrapolation_score += (val - self.bounds_max[i]) / (self.input_std[i] + 1e-8)
|
||||
|
||||
# Normalize input
|
||||
x_norm = (x - self.input_mean) / (self.input_std + 1e-8)
|
||||
x_t = torch.FloatTensor(x_norm)
|
||||
|
||||
# Predict
|
||||
with torch.no_grad():
|
||||
pred_norm = self.model(x_t).numpy()
|
||||
|
||||
pred = pred_norm * (self.output_std + 1e-8) + self.output_mean
|
||||
|
||||
# Calculate confidence-adjusted uncertainty
|
||||
base_uncertainty = self.cv_mass_mape / 100 # Base uncertainty from CV
|
||||
extrapolation_penalty = min(extrapolation_score * 0.1, 0.5) # Max 50% extra uncertainty
|
||||
total_uncertainty = base_uncertainty + extrapolation_penalty
|
||||
|
||||
return {
|
||||
'mass': float(pred[0, 0]),
|
||||
'frequency': float(pred[0, 1]),
|
||||
'max_displacement': float(pred[0, 2]),
|
||||
'max_stress': float(pred[0, 3]),
|
||||
'uncertainty': total_uncertainty,
|
||||
'extrapolating': extrapolation_score > 0.1,
|
||||
'extrapolation_score': extrapolation_score,
|
||||
'needs_fea_validation': extrapolation_score > 0.5
|
||||
}
|
||||
|
||||
|
||||
def run_optimization(surrogate: ValidatedSurrogate, bounds: dict, n_trials: int = 1000):
|
||||
"""Run multi-objective optimization.
|
||||
|
||||
Args:
|
||||
surrogate: ValidatedSurrogate model
|
||||
bounds: Dict from load_config_bounds {name: (min, max, is_int)}
|
||||
n_trials: Number of optimization trials
|
||||
"""
|
||||
print(f"\nOptimization bounds (from config):")
|
||||
for name, (low, high, is_int) in bounds.items():
|
||||
type_str = "int" if is_int else "float"
|
||||
print(f" {name}: [{low}, {high}] ({type_str})")
|
||||
|
||||
# Create study
|
||||
study = optuna.create_study(
|
||||
directions=["minimize", "minimize"], # mass, -frequency
|
||||
sampler=NSGAIISampler()
|
||||
)
|
||||
|
||||
# Track stats
|
||||
start_time = time.time()
|
||||
trial_times = []
|
||||
extrapolation_count = 0
|
||||
|
||||
def objective(trial: optuna.Trial):
|
||||
nonlocal extrapolation_count
|
||||
|
||||
params = {}
|
||||
for name, (low, high, is_int) in bounds.items():
|
||||
if is_int:
|
||||
params[name] = trial.suggest_int(name, int(low), int(high))
|
||||
else:
|
||||
params[name] = trial.suggest_float(name, float(low), float(high))
|
||||
|
||||
trial_start = time.time()
|
||||
result = surrogate.predict(params)
|
||||
trial_time = (time.time() - trial_start) * 1000
|
||||
trial_times.append(trial_time)
|
||||
|
||||
# Track extrapolation
|
||||
if result['extrapolating']:
|
||||
extrapolation_count += 1
|
||||
|
||||
# Store metadata
|
||||
trial.set_user_attr('uncertainty', result['uncertainty'])
|
||||
trial.set_user_attr('extrapolating', result['extrapolating'])
|
||||
trial.set_user_attr('needs_fea', result['needs_fea_validation'])
|
||||
trial.set_user_attr('params', params)
|
||||
|
||||
if trial.number % 200 == 0:
|
||||
print(f" Trial {trial.number}: mass={result['mass']:.1f}g, freq={result['frequency']:.2f}Hz, extrap={result['extrapolating']}")
|
||||
|
||||
return result['mass'], -result['frequency']
|
||||
|
||||
print(f"\nRunning {n_trials} trials...")
|
||||
study.optimize(objective, n_trials=n_trials, show_progress_bar=True)
|
||||
|
||||
total_time = time.time() - start_time
|
||||
|
||||
return study, {
|
||||
'total_time': total_time,
|
||||
'avg_trial_time_ms': np.mean(trial_times),
|
||||
'trials_per_second': n_trials / total_time,
|
||||
'extrapolation_count': extrapolation_count,
|
||||
'extrapolation_pct': extrapolation_count / n_trials * 100
|
||||
}
|
||||
|
||||
|
||||
def analyze_pareto_front(study, surrogate):
|
||||
"""Analyze Pareto front with confidence information."""
|
||||
pareto_trials = study.best_trials
|
||||
|
||||
results = {
|
||||
'total_pareto_designs': len(pareto_trials),
|
||||
'confident_designs': 0,
|
||||
'needs_fea_designs': 0,
|
||||
'designs': []
|
||||
}
|
||||
|
||||
for trial in pareto_trials:
|
||||
mass = trial.values[0]
|
||||
freq = -trial.values[1]
|
||||
uncertainty = trial.user_attrs.get('uncertainty', 0)
|
||||
needs_fea = trial.user_attrs.get('needs_fea', False)
|
||||
params = trial.user_attrs.get('params', trial.params)
|
||||
|
||||
if needs_fea:
|
||||
results['needs_fea_designs'] += 1
|
||||
else:
|
||||
results['confident_designs'] += 1
|
||||
|
||||
results['designs'].append({
|
||||
'mass': mass,
|
||||
'frequency': freq,
|
||||
'uncertainty': uncertainty,
|
||||
'needs_fea': needs_fea,
|
||||
'params': params
|
||||
})
|
||||
|
||||
# Sort by mass
|
||||
results['designs'].sort(key=lambda x: x['mass'])
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def plot_results(study, pareto_analysis, surrogate, save_path):
|
||||
"""Generate visualization of optimization results."""
|
||||
fig, axes = plt.subplots(2, 2, figsize=(14, 12))
|
||||
|
||||
# 1. Pareto Front with Confidence
|
||||
ax = axes[0, 0]
|
||||
pareto = pareto_analysis['designs']
|
||||
|
||||
confident = [d for d in pareto if not d['needs_fea']]
|
||||
needs_fea = [d for d in pareto if d['needs_fea']]
|
||||
|
||||
if confident:
|
||||
ax.scatter([d['mass'] for d in confident],
|
||||
[d['frequency'] for d in confident],
|
||||
c='green', s=50, alpha=0.7, label=f'Confident ({len(confident)})')
|
||||
|
||||
if needs_fea:
|
||||
ax.scatter([d['mass'] for d in needs_fea],
|
||||
[d['frequency'] for d in needs_fea],
|
||||
c='orange', s=50, alpha=0.7, marker='^', label=f'Needs FEA ({len(needs_fea)})')
|
||||
|
||||
ax.set_xlabel('Mass (g)')
|
||||
ax.set_ylabel('Frequency (Hz)')
|
||||
ax.set_title('Pareto Front with Confidence Assessment')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 2. Uncertainty vs Performance
|
||||
ax = axes[0, 1]
|
||||
all_trials = study.trials
|
||||
masses = [t.values[0] for t in all_trials if t.values]
|
||||
freqs = [-t.values[1] for t in all_trials if t.values]
|
||||
uncertainties = [t.user_attrs.get('uncertainty', 0) for t in all_trials if t.values]
|
||||
|
||||
sc = ax.scatter(masses, freqs, c=uncertainties, cmap='RdYlGn_r', s=10, alpha=0.5)
|
||||
plt.colorbar(sc, ax=ax, label='Uncertainty')
|
||||
ax.set_xlabel('Mass (g)')
|
||||
ax.set_ylabel('Frequency (Hz)')
|
||||
ax.set_title('All Trials Colored by Uncertainty')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 3. Top 10 Pareto Designs
|
||||
ax = axes[1, 0]
|
||||
top_10 = pareto_analysis['designs'][:10]
|
||||
|
||||
y_pos = np.arange(len(top_10))
|
||||
bars = ax.barh(y_pos, [d['mass'] for d in top_10],
|
||||
color=['green' if not d['needs_fea'] else 'orange' for d in top_10])
|
||||
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels([f"#{i+1}: {d['frequency']:.1f}Hz" for i, d in enumerate(top_10)])
|
||||
ax.set_xlabel('Mass (g)')
|
||||
ax.set_title('Top 10 Lowest Mass Designs')
|
||||
ax.grid(True, alpha=0.3, axis='x')
|
||||
|
||||
# Add confidence text
|
||||
for i, (bar, d) in enumerate(zip(bars, top_10)):
|
||||
status = "FEA!" if d['needs_fea'] else "OK"
|
||||
ax.text(bar.get_width() + 10, bar.get_y() + bar.get_height()/2,
|
||||
status, va='center', fontsize=9)
|
||||
|
||||
# 4. Summary Text
|
||||
ax = axes[1, 1]
|
||||
ax.axis('off')
|
||||
|
||||
summary_text = f"""
|
||||
OPTIMIZATION SUMMARY
|
||||
====================
|
||||
|
||||
CV-Validated Model Accuracy:
|
||||
Mass MAPE: {surrogate.cv_mass_mape:.1f}% +/- {surrogate.cv_mass_std:.1f}%
|
||||
Freq MAPE: {surrogate.cv_freq_mape:.1f}% +/- {surrogate.cv_freq_std:.1f}%
|
||||
Training samples: {surrogate.n_training_samples}
|
||||
|
||||
Pareto Front:
|
||||
Total designs: {pareto_analysis['total_pareto_designs']}
|
||||
High confidence: {pareto_analysis['confident_designs']}
|
||||
Needs FEA validation: {pareto_analysis['needs_fea_designs']}
|
||||
|
||||
Best Confident Design:
|
||||
"""
|
||||
# Find best confident design
|
||||
best_confident = [d for d in pareto_analysis['designs'] if not d['needs_fea']]
|
||||
if best_confident:
|
||||
best = best_confident[0]
|
||||
summary_text += f"""
|
||||
Mass: {best['mass']:.1f}g
|
||||
Frequency: {best['frequency']:.2f}Hz
|
||||
Parameters:
|
||||
"""
|
||||
for k, v in best['params'].items():
|
||||
if isinstance(v, float):
|
||||
summary_text += f" {k}: {v:.3f}\n"
|
||||
else:
|
||||
summary_text += f" {k}: {v}\n"
|
||||
else:
|
||||
summary_text += " No confident designs found - run more FEA!"
|
||||
|
||||
ax.text(0.05, 0.95, summary_text, transform=ax.transAxes,
|
||||
fontfamily='monospace', fontsize=10, verticalalignment='top')
|
||||
|
||||
plt.suptitle('NN-Based Multi-Objective Optimization Results', fontsize=14, fontweight='bold')
|
||||
plt.tight_layout()
|
||||
plt.savefig(save_path, dpi=150)
|
||||
plt.close()
|
||||
print(f"Saved plot: {save_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='NN Optimization with Confidence Bounds')
|
||||
parser.add_argument('--study', default='uav_arm_optimization',
|
||||
help='Study name (e.g., uav_arm_optimization)')
|
||||
parser.add_argument('--model', default=None,
|
||||
help='Path to CV-validated model (default: cv_validated_surrogate.pt)')
|
||||
parser.add_argument('--trials', type=int, default=2000,
|
||||
help='Number of optimization trials')
|
||||
args = parser.parse_args()
|
||||
|
||||
print("="*70)
|
||||
print("Production-Ready NN Optimization with Confidence Bounds")
|
||||
print("="*70)
|
||||
|
||||
# Load bounds from study config
|
||||
study_path = project_root / "studies" / args.study
|
||||
if not study_path.exists():
|
||||
print(f"ERROR: Study not found: {study_path}")
|
||||
return
|
||||
|
||||
print(f"\nLoading bounds from study: {args.study}")
|
||||
bounds = load_config_bounds(study_path)
|
||||
print(f" Loaded {len(bounds)} design variables from config")
|
||||
|
||||
# Load CV-validated model
|
||||
model_path = Path(args.model) if args.model else project_root / "cv_validated_surrogate.pt"
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: Run validate_surrogate_real_data.py first to create {model_path}")
|
||||
return
|
||||
|
||||
print(f"\nLoading CV-validated model: {model_path}")
|
||||
surrogate = ValidatedSurrogate(str(model_path))
|
||||
|
||||
print(f"\nModel Statistics:")
|
||||
print(f" Training samples: {surrogate.n_training_samples}")
|
||||
print(f" CV Mass MAPE: {surrogate.cv_mass_mape:.1f}% +/- {surrogate.cv_mass_std:.1f}%")
|
||||
print(f" CV Freq MAPE: {surrogate.cv_freq_mape:.1f}% +/- {surrogate.cv_freq_std:.1f}%")
|
||||
print(f" Design variables: {surrogate.design_var_names}")
|
||||
|
||||
# Verify model variables match config variables
|
||||
config_vars = set(bounds.keys())
|
||||
model_vars = set(surrogate.design_var_names)
|
||||
if config_vars != model_vars:
|
||||
print(f"\nWARNING: Variable mismatch!")
|
||||
print(f" Config has: {config_vars}")
|
||||
print(f" Model has: {model_vars}")
|
||||
print(f" Missing from model: {config_vars - model_vars}")
|
||||
print(f" Extra in model: {model_vars - config_vars}")
|
||||
|
||||
# Run optimization
|
||||
print("\n" + "="*70)
|
||||
print("Running Multi-Objective Optimization")
|
||||
print("="*70)
|
||||
|
||||
study, stats = run_optimization(surrogate, bounds, n_trials=args.trials)
|
||||
|
||||
print(f"\nOptimization Statistics:")
|
||||
print(f" Total time: {stats['total_time']:.1f}s")
|
||||
print(f" Avg trial time: {stats['avg_trial_time_ms']:.2f}ms")
|
||||
print(f" Trials per second: {stats['trials_per_second']:.1f}")
|
||||
print(f" Extrapolating trials: {stats['extrapolation_count']} ({stats['extrapolation_pct']:.1f}%)")
|
||||
|
||||
# Analyze Pareto front
|
||||
print("\n" + "="*70)
|
||||
print("Pareto Front Analysis")
|
||||
print("="*70)
|
||||
|
||||
pareto_analysis = analyze_pareto_front(study, surrogate)
|
||||
|
||||
print(f"\nPareto Front Summary:")
|
||||
print(f" Total Pareto-optimal designs: {pareto_analysis['total_pareto_designs']}")
|
||||
print(f" High confidence designs: {pareto_analysis['confident_designs']}")
|
||||
print(f" Needs FEA validation: {pareto_analysis['needs_fea_designs']}")
|
||||
|
||||
print(f"\nTop 5 Lowest Mass Designs:")
|
||||
for i, d in enumerate(pareto_analysis['designs'][:5]):
|
||||
status = "[NEEDS FEA]" if d['needs_fea'] else "[OK]"
|
||||
print(f" {i+1}. Mass={d['mass']:.1f}g, Freq={d['frequency']:.2f}Hz {status}")
|
||||
print(f" Params: {d['params']}")
|
||||
|
||||
# Generate plots
|
||||
plot_path = project_root / "validated_nn_optimization_results.png"
|
||||
plot_results(study, pareto_analysis, surrogate, str(plot_path))
|
||||
|
||||
# Save results
|
||||
results_path = project_root / "validated_nn_optimization_results.json"
|
||||
with open(results_path, 'w') as f:
|
||||
json.dump({
|
||||
'stats': stats,
|
||||
'pareto_summary': {
|
||||
'total': pareto_analysis['total_pareto_designs'],
|
||||
'confident': pareto_analysis['confident_designs'],
|
||||
'needs_fea': pareto_analysis['needs_fea_designs']
|
||||
},
|
||||
'top_designs': pareto_analysis['designs'][:20],
|
||||
'cv_metrics': {
|
||||
'mass_mape': surrogate.cv_mass_mape,
|
||||
'freq_mape': surrogate.cv_freq_mape
|
||||
}
|
||||
}, f, indent=2)
|
||||
print(f"\nSaved results: {results_path}")
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("NEXT STEPS")
|
||||
print("="*70)
|
||||
|
||||
if pareto_analysis['confident_designs'] > 0:
|
||||
print("1. Review the confident Pareto-optimal designs above")
|
||||
print("2. Consider validating top 3-5 designs with actual FEA")
|
||||
print("3. If FEA matches predictions, use for manufacturing")
|
||||
else:
|
||||
print("1. All Pareto designs are extrapolating - need more FEA data!")
|
||||
print("2. Run FEA on designs marked [NEEDS FEA]")
|
||||
print("3. Retrain the model with new data")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
432
archive/scripts/validate_surrogate_real_data.py
Normal file
432
archive/scripts/validate_surrogate_real_data.py
Normal file
@@ -0,0 +1,432 @@
|
||||
"""
|
||||
Real Data Cross-Validation for Surrogate Model
|
||||
|
||||
This script performs proper k-fold cross-validation using ONLY real FEA data
|
||||
to assess the true prediction accuracy of the neural network surrogate.
|
||||
|
||||
The key insight: We don't need simulated FEA - we already have real FEA data!
|
||||
We can use cross-validation to estimate out-of-sample performance.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
from sklearn.model_selection import KFold
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Add project paths
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from optimization_engine.active_learning_surrogate import (
|
||||
EnsembleMLP,
|
||||
extract_training_data_from_study
|
||||
)
|
||||
|
||||
|
||||
def k_fold_cross_validation(
|
||||
design_params: np.ndarray,
|
||||
objectives: np.ndarray,
|
||||
design_var_names: list,
|
||||
n_folds: int = 5,
|
||||
hidden_dims: list = [128, 64, 32], # Deeper network
|
||||
epochs: int = 300,
|
||||
lr: float = 0.001
|
||||
):
|
||||
"""
|
||||
Perform k-fold cross-validation to assess real prediction performance.
|
||||
|
||||
Returns detailed metrics for each fold.
|
||||
"""
|
||||
kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)
|
||||
|
||||
fold_results = []
|
||||
all_predictions = []
|
||||
all_actuals = []
|
||||
all_indices = []
|
||||
|
||||
for fold, (train_idx, test_idx) in enumerate(kf.split(design_params)):
|
||||
print(f"\n--- Fold {fold+1}/{n_folds} ---")
|
||||
|
||||
X_train, X_test = design_params[train_idx], design_params[test_idx]
|
||||
y_train, y_test = objectives[train_idx], objectives[test_idx]
|
||||
|
||||
# Normalization on training data
|
||||
X_mean = X_train.mean(axis=0)
|
||||
X_std = X_train.std(axis=0) + 1e-8
|
||||
y_mean = y_train.mean(axis=0)
|
||||
y_std = y_train.std(axis=0) + 1e-8
|
||||
|
||||
X_train_norm = (X_train - X_mean) / X_std
|
||||
X_test_norm = (X_test - X_mean) / X_std
|
||||
y_train_norm = (y_train - y_mean) / y_std
|
||||
|
||||
# Convert to tensors
|
||||
X_train_t = torch.FloatTensor(X_train_norm)
|
||||
y_train_t = torch.FloatTensor(y_train_norm)
|
||||
X_test_t = torch.FloatTensor(X_test_norm)
|
||||
|
||||
# Train model
|
||||
input_dim = X_train.shape[1]
|
||||
output_dim = y_train.shape[1]
|
||||
model = EnsembleMLP(input_dim, output_dim, hidden_dims)
|
||||
|
||||
optimizer = optim.Adam(model.parameters(), lr=lr)
|
||||
criterion = nn.MSELoss()
|
||||
|
||||
# Training with early stopping
|
||||
best_loss = float('inf')
|
||||
patience = 30
|
||||
patience_counter = 0
|
||||
best_state = None
|
||||
|
||||
for epoch in range(epochs):
|
||||
model.train()
|
||||
|
||||
# Mini-batch training
|
||||
batch_size = 32
|
||||
perm = torch.randperm(len(X_train_t))
|
||||
epoch_loss = 0
|
||||
n_batches = 0
|
||||
|
||||
for j in range(0, len(X_train_t), batch_size):
|
||||
batch_idx = perm[j:j+batch_size]
|
||||
X_batch = X_train_t[batch_idx]
|
||||
y_batch = y_train_t[batch_idx]
|
||||
|
||||
optimizer.zero_grad()
|
||||
pred = model(X_batch)
|
||||
loss = criterion(pred, y_batch)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
n_batches += 1
|
||||
|
||||
avg_loss = epoch_loss / n_batches
|
||||
if avg_loss < best_loss:
|
||||
best_loss = avg_loss
|
||||
best_state = model.state_dict().copy()
|
||||
patience_counter = 0
|
||||
else:
|
||||
patience_counter += 1
|
||||
if patience_counter >= patience:
|
||||
break
|
||||
|
||||
# Restore best model
|
||||
if best_state:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
# Evaluate on test set
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
pred_norm = model(X_test_t).numpy()
|
||||
pred = pred_norm * y_std + y_mean
|
||||
|
||||
# Calculate errors for each objective
|
||||
mass_pred = pred[:, 0]
|
||||
mass_actual = y_test[:, 0]
|
||||
freq_pred = pred[:, 1]
|
||||
freq_actual = y_test[:, 1]
|
||||
|
||||
mass_errors = np.abs(mass_pred - mass_actual) / (np.abs(mass_actual) + 1e-8)
|
||||
freq_errors = np.abs(freq_pred - freq_actual) / (np.abs(freq_actual) + 1e-8)
|
||||
|
||||
mass_mape = np.mean(mass_errors) * 100
|
||||
freq_mape = np.mean(freq_errors) * 100
|
||||
mass_rmse = np.sqrt(np.mean((mass_pred - mass_actual)**2))
|
||||
freq_rmse = np.sqrt(np.mean((freq_pred - freq_actual)**2))
|
||||
|
||||
fold_results.append({
|
||||
'fold': fold + 1,
|
||||
'n_train': len(train_idx),
|
||||
'n_test': len(test_idx),
|
||||
'mass_mape': mass_mape,
|
||||
'freq_mape': freq_mape,
|
||||
'mass_rmse': mass_rmse,
|
||||
'freq_rmse': freq_rmse,
|
||||
'epochs_trained': epoch + 1
|
||||
})
|
||||
|
||||
# Store for plotting
|
||||
all_predictions.extend(pred.tolist())
|
||||
all_actuals.extend(y_test.tolist())
|
||||
all_indices.extend(test_idx.tolist())
|
||||
|
||||
print(f" Mass MAPE: {mass_mape:.1f}%, RMSE: {mass_rmse:.1f}g")
|
||||
print(f" Freq MAPE: {freq_mape:.1f}%, RMSE: {freq_rmse:.1f}Hz")
|
||||
|
||||
return fold_results, np.array(all_predictions), np.array(all_actuals), all_indices
|
||||
|
||||
|
||||
def train_final_model_with_cv_uncertainty(
|
||||
design_params: np.ndarray,
|
||||
objectives: np.ndarray,
|
||||
design_var_names: list,
|
||||
cv_results: dict
|
||||
):
|
||||
"""
|
||||
Train final model on all data and attach CV-based uncertainty estimates.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("Training Final Model on All Data")
|
||||
print("="*60)
|
||||
|
||||
# Normalization
|
||||
X_mean = design_params.mean(axis=0)
|
||||
X_std = design_params.std(axis=0) + 1e-8
|
||||
y_mean = objectives.mean(axis=0)
|
||||
y_std = objectives.std(axis=0) + 1e-8
|
||||
|
||||
X_norm = (design_params - X_mean) / X_std
|
||||
y_norm = (objectives - y_mean) / y_std
|
||||
|
||||
X_t = torch.FloatTensor(X_norm)
|
||||
y_t = torch.FloatTensor(y_norm)
|
||||
|
||||
# Train on all data
|
||||
input_dim = design_params.shape[1]
|
||||
output_dim = objectives.shape[1]
|
||||
model = EnsembleMLP(input_dim, output_dim, [128, 64, 32])
|
||||
|
||||
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
||||
criterion = nn.MSELoss()
|
||||
|
||||
for epoch in range(500):
|
||||
model.train()
|
||||
optimizer.zero_grad()
|
||||
pred = model(X_t)
|
||||
loss = criterion(pred, y_t)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
# Save model with metadata
|
||||
model_state = {
|
||||
'model': model.state_dict(),
|
||||
'input_mean': X_mean.tolist(),
|
||||
'input_std': X_std.tolist(),
|
||||
'output_mean': y_mean.tolist(),
|
||||
'output_std': y_std.tolist(),
|
||||
'design_var_names': design_var_names,
|
||||
'cv_mass_mape': cv_results['mass_mape'],
|
||||
'cv_freq_mape': cv_results['freq_mape'],
|
||||
'cv_mass_std': cv_results['mass_std'],
|
||||
'cv_freq_std': cv_results['freq_std'],
|
||||
'n_samples': len(design_params),
|
||||
'hidden_dims': [128, 64, 32]
|
||||
}
|
||||
|
||||
save_path = project_root / "cv_validated_surrogate.pt"
|
||||
torch.save(model_state, save_path)
|
||||
print(f"Saved CV-validated model to: {save_path}")
|
||||
|
||||
return model, model_state
|
||||
|
||||
|
||||
def plot_cv_results(
|
||||
all_predictions: np.ndarray,
|
||||
all_actuals: np.ndarray,
|
||||
fold_results: list,
|
||||
save_path: str
|
||||
):
|
||||
"""Generate comprehensive validation plots."""
|
||||
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
|
||||
|
||||
# 1. Mass: Predicted vs Actual
|
||||
ax = axes[0, 0]
|
||||
ax.scatter(all_actuals[:, 0], all_predictions[:, 0], alpha=0.5, s=20)
|
||||
min_val = min(all_actuals[:, 0].min(), all_predictions[:, 0].min())
|
||||
max_val = max(all_actuals[:, 0].max(), all_predictions[:, 0].max())
|
||||
ax.plot([min_val, max_val], [min_val, max_val], 'r--', linewidth=2, label='Perfect')
|
||||
ax.set_xlabel('FEA Mass (g)')
|
||||
ax.set_ylabel('NN Predicted Mass (g)')
|
||||
ax.set_title('Mass: Predicted vs Actual')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 2. Frequency: Predicted vs Actual
|
||||
ax = axes[0, 1]
|
||||
ax.scatter(all_actuals[:, 1], all_predictions[:, 1], alpha=0.5, s=20)
|
||||
min_val = min(all_actuals[:, 1].min(), all_predictions[:, 1].min())
|
||||
max_val = max(all_actuals[:, 1].max(), all_predictions[:, 1].max())
|
||||
ax.plot([min_val, max_val], [min_val, max_val], 'r--', linewidth=2, label='Perfect')
|
||||
ax.set_xlabel('FEA Frequency (Hz)')
|
||||
ax.set_ylabel('NN Predicted Frequency (Hz)')
|
||||
ax.set_title('Frequency: Predicted vs Actual')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 3. Mass Errors by Fold
|
||||
ax = axes[0, 2]
|
||||
folds = [r['fold'] for r in fold_results]
|
||||
mass_mapes = [r['mass_mape'] for r in fold_results]
|
||||
bars = ax.bar(folds, mass_mapes, color='steelblue', alpha=0.7)
|
||||
ax.axhline(y=np.mean(mass_mapes), color='red', linestyle='--', label=f'Mean: {np.mean(mass_mapes):.1f}%')
|
||||
ax.axhline(y=10, color='green', linestyle='--', label='Target: 10%')
|
||||
ax.set_xlabel('Fold')
|
||||
ax.set_ylabel('Mass MAPE (%)')
|
||||
ax.set_title('Mass MAPE by Fold')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3, axis='y')
|
||||
|
||||
# 4. Frequency Errors by Fold
|
||||
ax = axes[1, 0]
|
||||
freq_mapes = [r['freq_mape'] for r in fold_results]
|
||||
bars = ax.bar(folds, freq_mapes, color='coral', alpha=0.7)
|
||||
ax.axhline(y=np.mean(freq_mapes), color='red', linestyle='--', label=f'Mean: {np.mean(freq_mapes):.1f}%')
|
||||
ax.axhline(y=10, color='green', linestyle='--', label='Target: 10%')
|
||||
ax.set_xlabel('Fold')
|
||||
ax.set_ylabel('Frequency MAPE (%)')
|
||||
ax.set_title('Frequency MAPE by Fold')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3, axis='y')
|
||||
|
||||
# 5. Error Distribution (Mass)
|
||||
ax = axes[1, 1]
|
||||
mass_errors = (all_predictions[:, 0] - all_actuals[:, 0]) / all_actuals[:, 0] * 100
|
||||
ax.hist(mass_errors, bins=30, color='steelblue', alpha=0.7, edgecolor='black')
|
||||
ax.axvline(x=0, color='red', linestyle='--', linewidth=2)
|
||||
ax.set_xlabel('Mass Error (%)')
|
||||
ax.set_ylabel('Count')
|
||||
ax.set_title(f'Mass Error Distribution (Mean={np.mean(mass_errors):.1f}%, Std={np.std(mass_errors):.1f}%)')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 6. Error Distribution (Frequency)
|
||||
ax = axes[1, 2]
|
||||
freq_errors = (all_predictions[:, 1] - all_actuals[:, 1]) / all_actuals[:, 1] * 100
|
||||
ax.hist(freq_errors, bins=30, color='coral', alpha=0.7, edgecolor='black')
|
||||
ax.axvline(x=0, color='red', linestyle='--', linewidth=2)
|
||||
ax.set_xlabel('Frequency Error (%)')
|
||||
ax.set_ylabel('Count')
|
||||
ax.set_title(f'Freq Error Distribution (Mean={np.mean(freq_errors):.1f}%, Std={np.std(freq_errors):.1f}%)')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
plt.suptitle('Cross-Validation Results on Real FEA Data', fontsize=14, fontweight='bold')
|
||||
plt.tight_layout()
|
||||
plt.savefig(save_path, dpi=150)
|
||||
plt.close()
|
||||
print(f"Saved validation plot: {save_path}")
|
||||
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("Cross-Validation Assessment on Real FEA Data")
|
||||
print("="*70)
|
||||
|
||||
# Find database
|
||||
db_path = project_root / "studies/uav_arm_optimization/2_results/study.db"
|
||||
study_name = "uav_arm_optimization"
|
||||
|
||||
if not db_path.exists():
|
||||
db_path = project_root / "studies/uav_arm_atomizerfield_test/2_results/study.db"
|
||||
study_name = "uav_arm_atomizerfield_test"
|
||||
|
||||
if not db_path.exists():
|
||||
print(f"ERROR: No database found")
|
||||
return
|
||||
|
||||
print(f"\nLoading data from: {db_path}")
|
||||
design_params, objectives, design_var_names = extract_training_data_from_study(
|
||||
str(db_path), study_name
|
||||
)
|
||||
|
||||
print(f"Total FEA samples: {len(design_params)}")
|
||||
print(f"Design variables: {design_var_names}")
|
||||
print(f"Objective ranges:")
|
||||
print(f" Mass: {objectives[:, 0].min():.1f} - {objectives[:, 0].max():.1f} g")
|
||||
print(f" Frequency: {objectives[:, 1].min():.1f} - {objectives[:, 1].max():.1f} Hz")
|
||||
|
||||
# Run k-fold cross-validation
|
||||
print("\n" + "="*70)
|
||||
print("Running 5-Fold Cross-Validation")
|
||||
print("="*70)
|
||||
|
||||
fold_results, all_predictions, all_actuals, all_indices = k_fold_cross_validation(
|
||||
design_params, objectives, design_var_names,
|
||||
n_folds=5,
|
||||
hidden_dims=[128, 64, 32], # Deeper network
|
||||
epochs=300
|
||||
)
|
||||
|
||||
# Summary statistics
|
||||
print("\n" + "="*70)
|
||||
print("CROSS-VALIDATION SUMMARY")
|
||||
print("="*70)
|
||||
|
||||
mass_mapes = [r['mass_mape'] for r in fold_results]
|
||||
freq_mapes = [r['freq_mape'] for r in fold_results]
|
||||
|
||||
cv_summary = {
|
||||
'mass_mape': np.mean(mass_mapes),
|
||||
'mass_std': np.std(mass_mapes),
|
||||
'freq_mape': np.mean(freq_mapes),
|
||||
'freq_std': np.std(freq_mapes)
|
||||
}
|
||||
|
||||
print(f"\nMass Prediction:")
|
||||
print(f" MAPE: {cv_summary['mass_mape']:.1f}% +/- {cv_summary['mass_std']:.1f}%")
|
||||
print(f" Status: {'[OK]' if cv_summary['mass_mape'] < 10 else '[NEEDS IMPROVEMENT]'}")
|
||||
|
||||
print(f"\nFrequency Prediction:")
|
||||
print(f" MAPE: {cv_summary['freq_mape']:.1f}% +/- {cv_summary['freq_std']:.1f}%")
|
||||
print(f" Status: {'[OK]' if cv_summary['freq_mape'] < 10 else '[NEEDS IMPROVEMENT]'}")
|
||||
|
||||
# Overall confidence
|
||||
mass_ok = cv_summary['mass_mape'] < 10
|
||||
freq_ok = cv_summary['freq_mape'] < 10
|
||||
|
||||
if mass_ok and freq_ok:
|
||||
confidence = "HIGH"
|
||||
recommendation = "NN surrogate is ready for optimization"
|
||||
elif mass_ok:
|
||||
confidence = "MEDIUM"
|
||||
recommendation = "Mass prediction is good, but frequency needs more data or better architecture"
|
||||
else:
|
||||
confidence = "LOW"
|
||||
recommendation = "Need more FEA data or improved NN architecture"
|
||||
|
||||
print(f"\nOverall Confidence: {confidence}")
|
||||
print(f"Recommendation: {recommendation}")
|
||||
|
||||
# Generate plots
|
||||
plot_path = project_root / "cv_validation_results.png"
|
||||
plot_cv_results(all_predictions, all_actuals, fold_results, str(plot_path))
|
||||
|
||||
# Train and save final model
|
||||
model, model_state = train_final_model_with_cv_uncertainty(
|
||||
design_params, objectives, design_var_names, cv_summary
|
||||
)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("FILES GENERATED")
|
||||
print("="*70)
|
||||
print(f" Validation plot: {plot_path}")
|
||||
print(f" CV-validated model: {project_root / 'cv_validated_surrogate.pt'}")
|
||||
|
||||
# Final assessment
|
||||
print("\n" + "="*70)
|
||||
print("NEXT STEPS")
|
||||
print("="*70)
|
||||
|
||||
if confidence == "HIGH":
|
||||
print("1. Use the NN surrogate for fast optimization")
|
||||
print("2. Periodically validate Pareto-optimal designs with FEA")
|
||||
elif confidence == "MEDIUM":
|
||||
print("1. Frequency prediction needs improvement")
|
||||
print("2. Options:")
|
||||
print(" a. Collect more FEA samples in underrepresented regions")
|
||||
print(" b. Try deeper/wider network architecture")
|
||||
print(" c. Add physics-informed features (e.g., I-beam moment of inertia)")
|
||||
print(" d. Use ensemble with uncertainty-weighted Pareto front")
|
||||
else:
|
||||
print("1. Current model not ready for optimization")
|
||||
print("2. Run more FEA trials to expand training data")
|
||||
print("3. Consider data augmentation or transfer learning")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
368
archive/scripts/visualize_nn_optimization.py
Normal file
368
archive/scripts/visualize_nn_optimization.py
Normal file
@@ -0,0 +1,368 @@
|
||||
"""
|
||||
Visualization and Validation of NN-Only Optimization Results
|
||||
|
||||
This script:
|
||||
1. Plots the Pareto front from NN optimization
|
||||
2. Compares NN predictions vs actual FEA data
|
||||
3. Shows prediction confidence and error analysis
|
||||
4. Validates selected NN-optimal designs with FEA data
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import json
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use('Agg') # Non-interactive backend for headless operation
|
||||
import matplotlib.pyplot as plt
|
||||
import optuna
|
||||
|
||||
# Add project paths
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from optimization_engine.simple_mlp_surrogate import SimpleSurrogate
|
||||
|
||||
def load_fea_data_from_database(db_path: str, study_name: str):
|
||||
"""Load actual FEA results from database for comparison."""
|
||||
storage = optuna.storages.RDBStorage(f"sqlite:///{db_path}")
|
||||
study = optuna.load_study(study_name=study_name, storage=storage)
|
||||
|
||||
completed_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
|
||||
|
||||
data = []
|
||||
for trial in completed_trials:
|
||||
if len(trial.values) < 2:
|
||||
continue
|
||||
mass = trial.values[0]
|
||||
# Handle both formats: some stores -freq (for minimization), some store +freq
|
||||
raw_freq = trial.values[1]
|
||||
# If frequency is stored as negative (minimization convention), flip it
|
||||
frequency = -raw_freq if raw_freq < 0 else raw_freq
|
||||
|
||||
# Skip invalid
|
||||
if np.isinf(mass) or np.isinf(frequency) or frequency <= 0:
|
||||
continue
|
||||
|
||||
data.append({
|
||||
'params': trial.params,
|
||||
'mass': mass,
|
||||
'frequency': frequency,
|
||||
'max_displacement': trial.user_attrs.get('max_displacement', 0),
|
||||
'max_stress': trial.user_attrs.get('max_stress', 0),
|
||||
})
|
||||
|
||||
return data
|
||||
|
||||
def plot_pareto_comparison(nn_results, fea_data, surrogate):
|
||||
"""Plot Pareto fronts: NN optimization vs FEA data."""
|
||||
fig, axes = plt.subplots(2, 2, figsize=(14, 12))
|
||||
|
||||
# Extract NN Pareto front
|
||||
nn_mass = [d['mass'] for d in nn_results['pareto_designs']]
|
||||
nn_freq = [d['frequency'] for d in nn_results['pareto_designs']]
|
||||
|
||||
# Extract FEA data
|
||||
fea_mass = [d['mass'] for d in fea_data]
|
||||
fea_freq = [d['frequency'] for d in fea_data]
|
||||
|
||||
# 1. Pareto Front Comparison
|
||||
ax = axes[0, 0]
|
||||
ax.scatter(fea_mass, fea_freq, alpha=0.5, label='FEA Trials', c='blue', s=30)
|
||||
ax.scatter(nn_mass, nn_freq, alpha=0.7, label='NN Pareto Front', c='red', s=20, marker='x')
|
||||
ax.set_xlabel('Mass (g)')
|
||||
ax.set_ylabel('Frequency (Hz)')
|
||||
ax.set_title('Pareto Front: NN Optimization vs FEA Data')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 2. NN Prediction Error on FEA Data
|
||||
ax = axes[0, 1]
|
||||
nn_pred_mass = []
|
||||
nn_pred_freq = []
|
||||
actual_mass = []
|
||||
actual_freq = []
|
||||
|
||||
for d in fea_data:
|
||||
pred = surrogate.predict(d['params'])
|
||||
nn_pred_mass.append(pred['mass'])
|
||||
nn_pred_freq.append(pred['frequency'])
|
||||
actual_mass.append(d['mass'])
|
||||
actual_freq.append(d['frequency'])
|
||||
|
||||
# Mass prediction error
|
||||
mass_errors = np.array(nn_pred_mass) - np.array(actual_mass)
|
||||
freq_errors = np.array(nn_pred_freq) - np.array(actual_freq)
|
||||
|
||||
scatter = ax.scatter(actual_mass, actual_freq, c=np.abs(mass_errors),
|
||||
cmap='RdYlGn_r', s=50, alpha=0.7)
|
||||
plt.colorbar(scatter, ax=ax, label='Mass Prediction Error (g)')
|
||||
ax.set_xlabel('Actual Mass (g)')
|
||||
ax.set_ylabel('Actual Frequency (Hz)')
|
||||
ax.set_title('FEA Points Colored by NN Mass Error')
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 3. Prediction vs Actual (Mass)
|
||||
ax = axes[1, 0]
|
||||
ax.scatter(actual_mass, nn_pred_mass, alpha=0.6, s=30)
|
||||
min_val, max_val = min(actual_mass), max(actual_mass)
|
||||
ax.plot([min_val, max_val], [min_val, max_val], 'r--', label='Perfect Prediction')
|
||||
ax.set_xlabel('Actual Mass (g)')
|
||||
ax.set_ylabel('NN Predicted Mass (g)')
|
||||
ax.set_title(f'Mass: NN vs FEA\nMAPE: {np.mean(np.abs(mass_errors)/np.array(actual_mass))*100:.1f}%')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# 4. Prediction vs Actual (Frequency)
|
||||
ax = axes[1, 1]
|
||||
ax.scatter(actual_freq, nn_pred_freq, alpha=0.6, s=30)
|
||||
min_val, max_val = min(actual_freq), max(actual_freq)
|
||||
ax.plot([min_val, max_val], [min_val, max_val], 'r--', label='Perfect Prediction')
|
||||
ax.set_xlabel('Actual Frequency (Hz)')
|
||||
ax.set_ylabel('NN Predicted Frequency (Hz)')
|
||||
ax.set_title(f'Frequency: NN vs FEA\nMAPE: {np.mean(np.abs(freq_errors)/np.array(actual_freq))*100:.1f}%')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
plt.tight_layout()
|
||||
plt.savefig(project_root / 'nn_optimization_analysis.png', dpi=150)
|
||||
print(f"Saved: nn_optimization_analysis.png")
|
||||
plt.close()
|
||||
|
||||
return mass_errors, freq_errors
|
||||
|
||||
def plot_design_space_coverage(nn_results, fea_data):
|
||||
"""Show how well NN explored the design space."""
|
||||
fig, axes = plt.subplots(2, 2, figsize=(14, 10))
|
||||
|
||||
param_names = ['beam_half_core_thickness', 'beam_face_thickness',
|
||||
'holes_diameter', 'hole_count']
|
||||
|
||||
for idx, (param1, param2) in enumerate([
|
||||
('beam_half_core_thickness', 'beam_face_thickness'),
|
||||
('holes_diameter', 'hole_count'),
|
||||
('beam_half_core_thickness', 'holes_diameter'),
|
||||
('beam_face_thickness', 'hole_count')
|
||||
]):
|
||||
ax = axes[idx // 2, idx % 2]
|
||||
|
||||
# FEA data points
|
||||
fea_p1 = [d['params'].get(param1, 0) for d in fea_data]
|
||||
fea_p2 = [d['params'].get(param2, 0) for d in fea_data]
|
||||
|
||||
# NN Pareto designs
|
||||
nn_p1 = [d['params'].get(param1, 0) for d in nn_results['pareto_designs']]
|
||||
nn_p2 = [d['params'].get(param2, 0) for d in nn_results['pareto_designs']]
|
||||
|
||||
ax.scatter(fea_p1, fea_p2, alpha=0.4, label='FEA Trials', c='blue', s=30)
|
||||
ax.scatter(nn_p1, nn_p2, alpha=0.7, label='NN Pareto', c='red', s=20, marker='x')
|
||||
|
||||
ax.set_xlabel(param1.replace('_', ' ').title())
|
||||
ax.set_ylabel(param2.replace('_', ' ').title())
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
plt.suptitle('Design Space Coverage: FEA vs NN Pareto Designs', fontsize=14)
|
||||
plt.tight_layout()
|
||||
plt.savefig(project_root / 'nn_design_space_coverage.png', dpi=150)
|
||||
print(f"Saved: nn_design_space_coverage.png")
|
||||
plt.close()
|
||||
|
||||
def plot_error_distribution(mass_errors, freq_errors):
|
||||
"""Plot error distributions to understand prediction confidence."""
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
|
||||
|
||||
# Mass error histogram
|
||||
ax = axes[0]
|
||||
ax.hist(mass_errors, bins=30, edgecolor='black', alpha=0.7)
|
||||
ax.axvline(0, color='r', linestyle='--', label='Zero Error')
|
||||
ax.axvline(np.mean(mass_errors), color='g', linestyle='-',
|
||||
label=f'Mean: {np.mean(mass_errors):.1f}g')
|
||||
ax.set_xlabel('Mass Prediction Error (g)')
|
||||
ax.set_ylabel('Count')
|
||||
ax.set_title(f'Mass Error Distribution\nStd: {np.std(mass_errors):.1f}g')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
# Frequency error histogram
|
||||
ax = axes[1]
|
||||
ax.hist(freq_errors, bins=30, edgecolor='black', alpha=0.7)
|
||||
ax.axvline(0, color='r', linestyle='--', label='Zero Error')
|
||||
ax.axvline(np.mean(freq_errors), color='g', linestyle='-',
|
||||
label=f'Mean: {np.mean(freq_errors):.1f}Hz')
|
||||
ax.set_xlabel('Frequency Prediction Error (Hz)')
|
||||
ax.set_ylabel('Count')
|
||||
ax.set_title(f'Frequency Error Distribution\nStd: {np.std(freq_errors):.1f}Hz')
|
||||
ax.legend()
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
plt.tight_layout()
|
||||
plt.savefig(project_root / 'nn_error_distribution.png', dpi=150)
|
||||
print(f"Saved: nn_error_distribution.png")
|
||||
plt.close()
|
||||
|
||||
def find_closest_fea_validation(nn_pareto, fea_data):
|
||||
"""Find FEA data points closest to NN Pareto designs for validation."""
|
||||
print("\n" + "="*70)
|
||||
print("VALIDATION: Comparing NN Pareto Designs to Nearest FEA Points")
|
||||
print("="*70)
|
||||
|
||||
# Get unique NN Pareto designs (remove duplicates)
|
||||
seen = set()
|
||||
unique_pareto = []
|
||||
for d in nn_pareto:
|
||||
key = (round(d['mass'], 1), round(d['frequency'], 1))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_pareto.append(d)
|
||||
|
||||
# Sort by mass
|
||||
unique_pareto.sort(key=lambda x: x['mass'])
|
||||
|
||||
# Sample 10 designs across the Pareto front
|
||||
indices = np.linspace(0, len(unique_pareto)-1, min(10, len(unique_pareto)), dtype=int)
|
||||
sampled_designs = [unique_pareto[i] for i in indices]
|
||||
|
||||
print(f"\nSampled {len(sampled_designs)} designs from NN Pareto front:")
|
||||
print("-"*70)
|
||||
|
||||
for i, nn_design in enumerate(sampled_designs):
|
||||
# Find closest FEA point (by parameter distance)
|
||||
min_dist = float('inf')
|
||||
closest_fea = None
|
||||
|
||||
for fea in fea_data:
|
||||
dist = sum((nn_design['params'].get(k, 0) - fea['params'].get(k, 0))**2
|
||||
for k in nn_design['params'])
|
||||
if dist < min_dist:
|
||||
min_dist = dist
|
||||
closest_fea = fea
|
||||
|
||||
if closest_fea:
|
||||
mass_err = nn_design['mass'] - closest_fea['mass']
|
||||
freq_err = nn_design['frequency'] - closest_fea['frequency']
|
||||
|
||||
print(f"\n{i+1}. NN Design: mass={nn_design['mass']:.1f}g, freq={nn_design['frequency']:.1f}Hz")
|
||||
print(f" Closest FEA: mass={closest_fea['mass']:.1f}g, freq={closest_fea['frequency']:.1f}Hz")
|
||||
print(f" Error: mass={mass_err:+.1f}g ({mass_err/closest_fea['mass']*100:+.1f}%), "
|
||||
f"freq={freq_err:+.1f}Hz ({freq_err/closest_fea['frequency']*100:+.1f}%)")
|
||||
print(f" Parameter Distance: {np.sqrt(min_dist):.2f}")
|
||||
|
||||
def print_optimization_summary(nn_results, fea_data, mass_errors, freq_errors):
|
||||
"""Print summary statistics."""
|
||||
print("\n" + "="*70)
|
||||
print("OPTIMIZATION SUMMARY")
|
||||
print("="*70)
|
||||
|
||||
print(f"\n1. NN Optimization Performance:")
|
||||
print(f" - Trials: {nn_results['n_trials']}")
|
||||
print(f" - Time: {nn_results['total_time_s']:.1f}s ({nn_results['trials_per_second']:.1f} trials/sec)")
|
||||
print(f" - Pareto Front Size: {nn_results['pareto_front_size']}")
|
||||
|
||||
print(f"\n2. NN Prediction Accuracy (on FEA data):")
|
||||
print(f" - Mass MAPE: {np.mean(np.abs(mass_errors)/np.array([d['mass'] for d in fea_data]))*100:.1f}%")
|
||||
print(f" - Mass Mean Error: {np.mean(mass_errors):.1f}g (Std: {np.std(mass_errors):.1f}g)")
|
||||
print(f" - Freq MAPE: {np.mean(np.abs(freq_errors)/np.array([d['frequency'] for d in fea_data]))*100:.1f}%")
|
||||
print(f" - Freq Mean Error: {np.mean(freq_errors):.1f}Hz (Std: {np.std(freq_errors):.1f}Hz)")
|
||||
|
||||
print(f"\n3. Design Space:")
|
||||
nn_mass = [d['mass'] for d in nn_results['pareto_designs']]
|
||||
nn_freq = [d['frequency'] for d in nn_results['pareto_designs']]
|
||||
fea_mass = [d['mass'] for d in fea_data]
|
||||
fea_freq = [d['frequency'] for d in fea_data]
|
||||
|
||||
print(f" - NN Pareto Mass Range: {min(nn_mass):.1f}g - {max(nn_mass):.1f}g")
|
||||
print(f" - NN Pareto Freq Range: {min(nn_freq):.1f}Hz - {max(nn_freq):.1f}Hz")
|
||||
print(f" - FEA Mass Range: {min(fea_mass):.1f}g - {max(fea_mass):.1f}g")
|
||||
print(f" - FEA Freq Range: {min(fea_freq):.1f}Hz - {max(fea_freq):.1f}Hz")
|
||||
|
||||
print(f"\n4. Confidence Assessment:")
|
||||
if np.std(mass_errors) < 100 and np.std(freq_errors) < 5:
|
||||
print(" [OK] LOW prediction variance - NN is fairly confident")
|
||||
else:
|
||||
print(" [!] HIGH prediction variance - consider more training data")
|
||||
|
||||
if abs(np.mean(mass_errors)) < 50:
|
||||
print(" [OK] LOW mass bias - predictions are well-centered")
|
||||
else:
|
||||
print(f" [!] Mass bias detected ({np.mean(mass_errors):+.1f}g) - systematic error")
|
||||
|
||||
if abs(np.mean(freq_errors)) < 2:
|
||||
print(" [OK] LOW frequency bias - predictions are well-centered")
|
||||
else:
|
||||
print(f" [!] Frequency bias detected ({np.mean(freq_errors):+.1f}Hz) - systematic error")
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("NN Optimization Visualization & Validation")
|
||||
print("="*70)
|
||||
|
||||
# Load NN optimization results
|
||||
results_file = project_root / "nn_optimization_results.json"
|
||||
if not results_file.exists():
|
||||
print(f"ERROR: {results_file} not found. Run NN optimization first.")
|
||||
return
|
||||
|
||||
with open(results_file) as f:
|
||||
nn_results = json.load(f)
|
||||
|
||||
print(f"\nLoaded NN results: {nn_results['n_trials']} trials, "
|
||||
f"{nn_results['pareto_front_size']} Pareto designs")
|
||||
|
||||
# Load surrogate model
|
||||
model_path = project_root / "simple_mlp_surrogate.pt"
|
||||
if not model_path.exists():
|
||||
print(f"ERROR: {model_path} not found.")
|
||||
return
|
||||
|
||||
surrogate = SimpleSurrogate.load(model_path)
|
||||
print(f"Loaded surrogate model with {len(surrogate.design_var_names)} design variables")
|
||||
|
||||
# Load FEA data from original study
|
||||
# Try each database path with its matching study name
|
||||
db_options = [
|
||||
(project_root / "studies/uav_arm_optimization/2_results/study.db", "uav_arm_optimization"),
|
||||
(project_root / "studies/uav_arm_atomizerfield_test/2_results/study.db", "uav_arm_atomizerfield_test"),
|
||||
]
|
||||
|
||||
db_path = None
|
||||
study_name = None
|
||||
for path, name in db_options:
|
||||
if path.exists():
|
||||
db_path = path
|
||||
study_name = name
|
||||
break
|
||||
|
||||
if db_path and study_name:
|
||||
fea_data = load_fea_data_from_database(str(db_path), study_name)
|
||||
print(f"Loaded {len(fea_data)} FEA data points from {study_name}")
|
||||
else:
|
||||
print("WARNING: No FEA database found. Using only NN results.")
|
||||
fea_data = []
|
||||
|
||||
if fea_data:
|
||||
# Generate all plots
|
||||
mass_errors, freq_errors = plot_pareto_comparison(nn_results, fea_data, surrogate)
|
||||
plot_design_space_coverage(nn_results, fea_data)
|
||||
plot_error_distribution(mass_errors, freq_errors)
|
||||
|
||||
# Print validation analysis
|
||||
find_closest_fea_validation(nn_results['pareto_designs'], fea_data)
|
||||
print_optimization_summary(nn_results, fea_data, mass_errors, freq_errors)
|
||||
else:
|
||||
# Just plot NN results
|
||||
print("\nPlotting NN Pareto front only (no FEA data for comparison)")
|
||||
nn_mass = [d['mass'] for d in nn_results['pareto_designs']]
|
||||
nn_freq = [d['frequency'] for d in nn_results['pareto_designs']]
|
||||
|
||||
plt.figure(figsize=(10, 6))
|
||||
plt.scatter(nn_mass, nn_freq, alpha=0.7, c='red', s=30)
|
||||
plt.xlabel('Mass (g)')
|
||||
plt.ylabel('Frequency (Hz)')
|
||||
plt.title('NN Optimization Pareto Front')
|
||||
plt.grid(True, alpha=0.3)
|
||||
plt.savefig(project_root / 'nn_pareto_front.png', dpi=150)
|
||||
plt.close()
|
||||
print(f"Saved: nn_pareto_front.png")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user