docs: Reorganize documentation structure

- Create DEVELOPMENT.md for tactical development tracking
- Simplify README.md to user-focused overview
- Streamline DEVELOPMENT_ROADMAP.md to focus on vision
- All docs now properly cross-referenced

Documentation now has clear separation:
- README: User overview
- DEVELOPMENT: Tactical todos and status
- ROADMAP: Strategic vision
- CHANGELOG: Version history

tests/run_5trial_test.py

@@ -0,0 +1,136 @@
+"""
+Quick 5-Trial Test with Real NX Solves
+
+Tests the complete optimization pipeline with hooks and logging.
+"""
+
+from pathlib import Path
+import sys
+from datetime import datetime
+
+# Add project root to path so we can import atomizer_paths
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+# Use intelligent path resolution
+import atomizer_paths
+atomizer_paths.ensure_imports()
+
+from optimization_engine.runner import OptimizationRunner
+from optimization_engine.nx_solver import run_nx_simulation
+from optimization_engine.result_extractors.extractors import (
+    stress_extractor,
+    displacement_extractor
+)
+
+# Global variable to store current design variables
+_current_design_vars = {}
+
+
+def bracket_model_updater(design_vars: dict):
+    """Store design variables for the simulation runner."""
+    global _current_design_vars
+    _current_design_vars = design_vars.copy()
+
+    print(f"\n[MODEL UPDATE] Design variables prepared")
+    for name, value in design_vars.items():
+        print(f"  {name} = {value:.4f}")
+
+
+def bracket_simulation_runner() -> Path:
+    """Run NX solver via journal on running NX GUI session."""
+    global _current_design_vars
+    sim_file = atomizer_paths.studies() / "bracket_stress_minimization/model/Bracket_sim1.sim"
+
+    print("\n[SIMULATION] Running via journal on NX GUI...")
+    print(f"  SIM file: {sim_file.name}")
+    if _current_design_vars:
+        print(f"  Expression updates: {_current_design_vars}")
+
+    try:
+        op2_file = run_nx_simulation(
+            sim_file=sim_file,
+            nastran_version="2412",
+            timeout=300,
+            cleanup=True,
+            use_journal=True,
+            expression_updates=_current_design_vars
+        )
+
+        print(f"[SIMULATION] Complete! Results: {op2_file.name}")
+        return op2_file
+
+    except Exception as e:
+        print(f"[SIMULATION] FAILED: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    print("="*60)
+    print("5-TRIAL REAL OPTIMIZATION TEST WITH LOGGING")
+    print("="*60)
+    print("\nThis will:")
+    print("- Run 5 real NX solves (not dummy data)")
+    print("- Create detailed log files in optimization_logs/")
+    print("- Test all hooks (pre_solve, post_solve, post_extraction)")
+    print("- Verify design variables actually change results")
+    print("\nTime: ~3-5 minutes")
+    print("="*60)
+
+    input("\nPress ENTER to continue (Ctrl+C to cancel)...")
+
+    config_path = atomizer_paths.studies() / "bracket_stress_minimization/optimization_config_stress_displacement.json"
+
+    runner = OptimizationRunner(
+        config_path=config_path,
+        model_updater=bracket_model_updater,
+        simulation_runner=bracket_simulation_runner,
+        result_extractors={
+            'stress_extractor': stress_extractor,
+            'displacement_extractor': displacement_extractor
+        }
+    )
+
+    # Generate unique study name with timestamp
+    study_name = f"test_5trials_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+
+    print("\n" + "="*60)
+    print("RUNNING 5-TRIAL OPTIMIZATION")
+    print("="*60)
+    print(f"Study name: {study_name}")
+    print("="*60)
+
+    try:
+        study = runner.run(
+            study_name=study_name,
+            n_trials=5,
+            resume=False
+        )
+
+        print("\n" + "="*60)
+        print("OPTIMIZATION COMPLETE!")
+        print("="*60)
+        print(f"\nBest stress: {study.best_value:.2f} MPa")
+        print(f"\nBest parameters:")
+        for param, value in study.best_params.items():
+            print(f"  {param}: {value:.4f}")
+
+        # Show log files created
+        log_dir = runner.output_dir / "trial_logs"
+        if log_dir.exists():
+            log_files = sorted(log_dir.glob("trial_*.log"))
+            print(f"\n{len(log_files)} detailed trial logs created:")
+            print(f"  {log_dir}")
+            print("\nExample log file (open to see detailed iteration trace):")
+            if log_files:
+                print(f"  {log_files[0].name}")
+
+        print(f"\nResults also saved to: {runner.output_dir}")
+
+    except Exception as e:
+        print(f"\n{'='*60}")
+        print("ERROR DURING OPTIMIZATION")
+        print("="*60)
+        print(f"{e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)

tests/test_hooks_with_bracket.py

@@ -0,0 +1,152 @@
+"""
+Test: Validate Hook System with Bracket Optimization
+
+Quick test (3 trials) to verify:
+1. Hooks load correctly
+2. Hooks execute at proper points
+3. Optimization still works with hooks enabled
+"""
+
+from pathlib import Path
+import sys
+
+# Add project root to path so we can import atomizer_paths
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+# Use intelligent path resolution
+import atomizer_paths
+atomizer_paths.ensure_imports()
+
+from optimization_engine.runner import OptimizationRunner
+from optimization_engine.result_extractors.extractors import (
+    stress_extractor,
+    displacement_extractor
+)
+
+
+# Dummy functions for quick testing (no actual NX calls)
+_trial_count = 0
+
+def dummy_model_updater(design_vars: dict):
+    """Simulate model update."""
+    print(f"\n[MODEL UPDATE] Design variables prepared")
+    for name, value in design_vars.items():
+        print(f"  {name} = {value:.4f}")
+
+
+def dummy_simulation_runner() -> Path:
+    """Simulate simulation run - return existing OP2 file."""
+    global _trial_count
+    _trial_count += 1
+
+    # Use existing OP2 file from bracket study
+    op2_file = atomizer_paths.studies() / "bracket_stress_minimization/optimization_results/bracket_sim1-solution_1.op2"
+
+    if not op2_file.exists():
+        raise FileNotFoundError(
+            f"Test OP2 file not found: {op2_file}\n"
+            "Please run a real solve first to generate this file."
+        )
+
+    print(f"\n[SIMULATION {_trial_count}] Using existing OP2: {op2_file.name}")
+    return op2_file
+
+
+if __name__ == "__main__":
+    print("="*60)
+    print("HOOK SYSTEM VALIDATION TEST")
+    print("="*60)
+    print("\nThis test will:")
+    print("- Run 3 quick optimization trials")
+    print("- Use dummy simulations (re-use existing OP2 file)")
+    print("- Verify hooks execute at all lifecycle points")
+    print("- Check that optimization completes successfully")
+    print("="*60)
+
+    config_path = atomizer_paths.studies() / "bracket_stress_minimization/optimization_config_stress_displacement.json"
+
+    # Create runner (this should load plugins automatically)
+    print("\n[INIT] Creating OptimizationRunner...")
+    runner = OptimizationRunner(
+        config_path=config_path,
+        model_updater=dummy_model_updater,
+        simulation_runner=dummy_simulation_runner,
+        result_extractors={
+            'stress_extractor': stress_extractor,
+            'displacement_extractor': displacement_extractor
+        }
+    )
+
+    # Check if hooks were loaded
+    print("\n[HOOKS] Checking hook system...")
+    hook_summary = runner.hook_manager.get_summary()
+    print(f"  Total hooks: {hook_summary['total_hooks']}")
+    print(f"  Enabled hooks: {hook_summary['enabled_hooks']}")
+
+    if hook_summary['total_hooks'] > 0:
+        print("\n  Hooks by point:")
+        for point, info in hook_summary['by_hook_point'].items():
+            if info['total'] > 0:
+                print(f"    {point}: {info['names']}")
+    else:
+        print("  WARNING: No hooks loaded! Plugin directory may be empty.")
+
+    print("\n" + "="*60)
+    print("RUNNING 3-TRIAL TEST")
+    print("="*60)
+
+    try:
+        # Run just 3 trials for quick validation
+        study = runner.run(
+            study_name="hook_validation_test",
+            n_trials=3,
+            resume=False
+        )
+
+        print("\n" + "="*60)
+        print("TEST COMPLETE!")
+        print("="*60)
+
+        # Check hook execution history
+        hook_history = runner.hook_manager.get_history()
+        successful_hooks = [h for h in hook_history if h['success']]
+        failed_hooks = [h for h in hook_history if not h['success']]
+
+        print(f"\nHook Execution Summary:")
+        print(f"  Total hook executions: {len(hook_history)}")
+        print(f"  Successful: {len(successful_hooks)}")
+        print(f"  Failed: {len(failed_hooks)}")
+
+        if successful_hooks:
+            print("\n  Successful hook executions:")
+            for h in successful_hooks[:10]:  # Show first 10
+                print(f"    Trial {h.get('trial_number', '?')}: {h['hook_name']} at {h['hook_point']}")
+
+        if failed_hooks:
+            print("\n  FAILED hook executions:")
+            for h in failed_hooks:
+                print(f"    Trial {h.get('trial_number', '?')}: {h['hook_name']} - {h.get('error', 'unknown error')}")
+
+        print(f"\nOptimization Results:")
+        print(f"  Best value: {study.best_value:.2f}")
+        print(f"  Best parameters:")
+        for param, value in study.best_params.items():
+            print(f"    {param}: {value:.4f}")
+
+        print("\n" + "="*60)
+        print("VALIDATION PASSED!")
+        print("="*60)
+        print("\nThe hook system is working correctly:")
+        print("  [OK] Hooks loaded from plugin directory")
+        print("  [OK] Hooks executed during optimization")
+        print("  [OK] Optimization completed successfully")
+        print("  [OK] Results extracted correctly")
+
+    except Exception as e:
+        print(f"\n{'='*60}")
+        print("TEST FAILED")
+        print("="*60)
+        print(f"{e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)

tests/test_journal_optimization.py

@@ -0,0 +1,208 @@
+"""
+Test: Complete Optimization with Journal-Based NX Solver
+
+This tests the complete workflow:
+1. Update model parameters in .prt
+2. Solve via journal (using running NX GUI)
+3. Extract results from OP2
+4. Run optimization loop
+
+REQUIREMENTS:
+- Simcenter3D must be open (but no files need to be loaded)
+- test_env conda environment activated
+"""
+
+from pathlib import Path
+import sys
+
+# Add project root to path so we can import atomizer_paths
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+# Use intelligent path resolution
+import atomizer_paths
+atomizer_paths.ensure_imports()
+
+from optimization_engine.runner import OptimizationRunner
+from optimization_engine.nx_updater import update_nx_model
+from optimization_engine.nx_solver import run_nx_simulation
+from optimization_engine.result_extractors.extractors import (
+    stress_extractor,
+    displacement_extractor
+)
+
+
+# Global variable to store current design variables for the simulation runner
+_current_design_vars = {}
+
+
+def bracket_model_updater(design_vars: dict):
+    """
+    Store design variables for the simulation runner.
+
+    Note: We no longer directly update the .prt file here.
+    Instead, design variables are passed to the journal which applies them in NX.
+    """
+    global _current_design_vars
+    _current_design_vars = design_vars.copy()
+
+    print(f"\n[MODEL UPDATE] Design variables prepared")
+    for name, value in design_vars.items():
+        print(f"  {name} = {value:.4f}")
+
+
+def bracket_simulation_runner() -> Path:
+    """
+    Run NX solver via journal on running NX GUI session.
+
+    This connects to the running Simcenter3D GUI and:
+    1. Opens the .sim file
+    2. Applies expression updates in the journal
+    3. Updates geometry and FEM
+    4. Solves the simulation
+    5. Returns path to .op2 file
+    """
+    global _current_design_vars
+    sim_file = atomizer_paths.studies() / "bracket_stress_minimization/model/Bracket_sim1.sim"
+
+    print("\n[SIMULATION] Running via journal on NX GUI...")
+    print(f"  SIM file: {sim_file.name}")
+    if _current_design_vars:
+        print(f"  Expression updates: {_current_design_vars}")
+
+    try:
+        # Run solver via journal (connects to running NX GUI)
+        # Pass expression updates directly to the journal
+        op2_file = run_nx_simulation(
+            sim_file=sim_file,
+            nastran_version="2412",
+            timeout=300,  # 5 minute timeout
+            cleanup=True,  # Clean up temp files
+            use_journal=True,  # Use journal mode (requires NX GUI open)
+            expression_updates=_current_design_vars  # Pass design vars to journal
+        )
+
+        print(f"[SIMULATION] Complete! Results: {op2_file.name}")
+        return op2_file
+
+    except Exception as e:
+        print(f"[SIMULATION] FAILED: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    print("="*60)
+    print("JOURNAL-BASED OPTIMIZATION TEST")
+    print("="*60)
+    print("\nREQUIREMENTS:")
+    print("- Simcenter3D must be OPEN (no files need to be loaded)")
+    print("- Will run 50 optimization trials (~3-4 minutes)")
+    print("- Strategy: 20 random trials (exploration) + 30 TPE trials (exploitation)")
+    print("- Each trial: update params -> solve via journal -> extract results")
+    print("="*60)
+
+    response = input("\nIs Simcenter3D open? (yes/no): ")
+    if response.lower() not in ['yes', 'y']:
+        print("Please open Simcenter3D and try again.")
+        sys.exit(0)
+
+    config_path = atomizer_paths.studies() / "bracket_stress_minimization/optimization_config_stress_displacement.json"
+
+    runner = OptimizationRunner(
+        config_path=config_path,
+        model_updater=bracket_model_updater,
+        simulation_runner=bracket_simulation_runner,  # Journal-based solver!
+        result_extractors={
+            'stress_extractor': stress_extractor,
+            'displacement_extractor': displacement_extractor
+        }
+    )
+
+    # Use the configured number of trials (50 by default)
+    n_trials = runner.config['optimization_settings']['n_trials']
+
+    # Check for existing studies
+    existing_studies = runner.list_studies()
+
+    print("\n" + "="*60)
+    print("STUDY MANAGEMENT")
+    print("="*60)
+
+    if existing_studies:
+        print(f"\nFound {len(existing_studies)} existing studies:")
+        for study in existing_studies:
+            print(f"  - {study['study_name']}: {study.get('total_trials', 0)} trials")
+
+        print("\nOptions:")
+        print("1. Create NEW study (fresh start)")
+        print("2. RESUME existing study (add more trials)")
+        choice = input("\nChoose option (1 or 2): ").strip()
+
+        if choice == '2':
+            # Resume existing study
+            if len(existing_studies) == 1:
+                study_name = existing_studies[0]['study_name']
+                print(f"\nResuming study: {study_name}")
+            else:
+                print("\nAvailable studies:")
+                for i, study in enumerate(existing_studies):
+                    print(f"{i+1}. {study['study_name']}")
+                study_idx = int(input("Select study number: ")) - 1
+                study_name = existing_studies[study_idx]['study_name']
+
+            resume_mode = True
+        else:
+            # New study
+            study_name = input("\nEnter study name (default: bracket_stress_opt): ").strip()
+            if not study_name:
+                study_name = "bracket_stress_opt"
+            resume_mode = False
+    else:
+        print("\nNo existing studies found. Creating new study.")
+        study_name = input("\nEnter study name (default: bracket_stress_opt): ").strip()
+        if not study_name:
+            study_name = "bracket_stress_opt"
+        resume_mode = False
+
+    print("\n" + "="*60)
+    if resume_mode:
+        print(f"RESUMING STUDY: {study_name}")
+        print(f"Adding {n_trials} additional trials")
+    else:
+        print(f"STARTING NEW STUDY: {study_name}")
+        print(f"Running {n_trials} trials")
+    print("="*60)
+    print("Objective: Minimize max von Mises stress")
+    print("Constraint: Max displacement <= 1.0 mm")
+    print("Solver: Journal-based (using running NX GUI)")
+    print(f"Sampler: TPE (20 random startup + {n_trials-20} TPE)")
+    print("="*60)
+
+    try:
+        study = runner.run(
+            study_name=study_name,
+            n_trials=n_trials,
+            resume=resume_mode
+        )
+
+        print("\n" + "="*60)
+        print("OPTIMIZATION COMPLETE!")
+        print("="*60)
+        print(f"\nBest stress: {study.best_value:.2f} MPa")
+        print(f"\nBest parameters:")
+        for param, value in study.best_params.items():
+            print(f"  {param}: {value:.4f}")
+
+        print(f"\nResults saved to: {runner.output_dir}")
+        print("\nCheck history.csv to see optimization progress!")
+
+    except Exception as e:
+        print(f"\n{'='*60}")
+        print("ERROR DURING OPTIMIZATION")
+        print("="*60)
+        print(f"{e}")
+        import traceback
+        traceback.print_exc()
+        print("\nMake sure:")
+        print("  - Simcenter3D is open and running")
+        print("  - .sim file is valid and solvable")
+        print("  - No other processes are locking the files")