feat(config): AtomizerSpec v2.0 Pydantic models, validators, and tests
Config Layer: - spec_models.py: Pydantic models for AtomizerSpec v2.0 - spec_validator.py: Semantic validation with detailed error reporting Extractors: - custom_extractor_loader.py: Runtime custom extractor loading - spec_extractor_builder.py: Build extractors from spec definitions Tools: - migrate_to_spec_v2.py: CLI tool for batch migration Tests: - test_migrator.py: Migration tests - test_spec_manager.py: SpecManager service tests - test_spec_api.py: REST API tests - test_mcp_tools.py: MCP tool tests - test_e2e_unified_config.py: End-to-end config tests
This commit is contained in:
674
optimization_engine/config/spec_models.py
Normal file
674
optimization_engine/config/spec_models.py
Normal file
@@ -0,0 +1,674 @@
|
||||
"""
|
||||
AtomizerSpec v2.0 Pydantic Models
|
||||
|
||||
These models match the JSON Schema at optimization_engine/schemas/atomizer_spec_v2.json
|
||||
They provide validation and type safety for the unified configuration system.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
import re
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Enums
|
||||
# ============================================================================
|
||||
|
||||
class SpecCreatedBy(str, Enum):
|
||||
"""Who/what created the spec."""
|
||||
CANVAS = "canvas"
|
||||
CLAUDE = "claude"
|
||||
API = "api"
|
||||
MIGRATION = "migration"
|
||||
MANUAL = "manual"
|
||||
|
||||
|
||||
class SolverType(str, Enum):
|
||||
"""Supported solver types."""
|
||||
NASTRAN = "nastran"
|
||||
NX_NASTRAN = "NX_Nastran"
|
||||
ABAQUS = "abaqus"
|
||||
|
||||
|
||||
class SubcaseType(str, Enum):
|
||||
"""Subcase analysis types."""
|
||||
STATIC = "static"
|
||||
MODAL = "modal"
|
||||
THERMAL = "thermal"
|
||||
BUCKLING = "buckling"
|
||||
|
||||
|
||||
class DesignVariableType(str, Enum):
|
||||
"""Design variable types."""
|
||||
CONTINUOUS = "continuous"
|
||||
INTEGER = "integer"
|
||||
CATEGORICAL = "categorical"
|
||||
|
||||
|
||||
class ExtractorType(str, Enum):
|
||||
"""Physics extractor types."""
|
||||
DISPLACEMENT = "displacement"
|
||||
FREQUENCY = "frequency"
|
||||
STRESS = "stress"
|
||||
MASS = "mass"
|
||||
MASS_EXPRESSION = "mass_expression"
|
||||
ZERNIKE_OPD = "zernike_opd"
|
||||
ZERNIKE_CSV = "zernike_csv"
|
||||
TEMPERATURE = "temperature"
|
||||
CUSTOM_FUNCTION = "custom_function"
|
||||
|
||||
|
||||
class OptimizationDirection(str, Enum):
|
||||
"""Optimization direction."""
|
||||
MINIMIZE = "minimize"
|
||||
MAXIMIZE = "maximize"
|
||||
|
||||
|
||||
class ConstraintType(str, Enum):
|
||||
"""Constraint types."""
|
||||
HARD = "hard"
|
||||
SOFT = "soft"
|
||||
|
||||
|
||||
class ConstraintOperator(str, Enum):
|
||||
"""Constraint comparison operators."""
|
||||
LE = "<="
|
||||
GE = ">="
|
||||
LT = "<"
|
||||
GT = ">"
|
||||
EQ = "=="
|
||||
|
||||
|
||||
class PenaltyMethod(str, Enum):
|
||||
"""Penalty methods for constraints."""
|
||||
LINEAR = "linear"
|
||||
QUADRATIC = "quadratic"
|
||||
EXPONENTIAL = "exponential"
|
||||
|
||||
|
||||
class AlgorithmType(str, Enum):
|
||||
"""Optimization algorithm types."""
|
||||
TPE = "TPE"
|
||||
CMA_ES = "CMA-ES"
|
||||
NSGA_II = "NSGA-II"
|
||||
RANDOM_SEARCH = "RandomSearch"
|
||||
SAT_V3 = "SAT_v3"
|
||||
GP_BO = "GP-BO"
|
||||
|
||||
|
||||
class SurrogateType(str, Enum):
|
||||
"""Surrogate model types."""
|
||||
MLP = "MLP"
|
||||
GNN = "GNN"
|
||||
ENSEMBLE = "ensemble"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Position Model
|
||||
# ============================================================================
|
||||
|
||||
class CanvasPosition(BaseModel):
|
||||
"""Canvas position for nodes."""
|
||||
x: float = 0
|
||||
y: float = 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Meta Models
|
||||
# ============================================================================
|
||||
|
||||
class SpecMeta(BaseModel):
|
||||
"""Metadata about the spec."""
|
||||
version: str = Field(
|
||||
...,
|
||||
pattern=r"^2\.\d+$",
|
||||
description="Schema version (e.g., '2.0')"
|
||||
)
|
||||
created: Optional[datetime] = Field(
|
||||
default=None,
|
||||
description="When the spec was created"
|
||||
)
|
||||
modified: Optional[datetime] = Field(
|
||||
default=None,
|
||||
description="When the spec was last modified"
|
||||
)
|
||||
created_by: Optional[SpecCreatedBy] = Field(
|
||||
default=None,
|
||||
description="Who/what created the spec"
|
||||
)
|
||||
modified_by: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Who/what last modified the spec"
|
||||
)
|
||||
study_name: str = Field(
|
||||
...,
|
||||
min_length=3,
|
||||
max_length=100,
|
||||
pattern=r"^[a-z0-9_]+$",
|
||||
description="Unique study identifier (snake_case)"
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
default=None,
|
||||
max_length=1000,
|
||||
description="Human-readable description"
|
||||
)
|
||||
tags: Optional[List[str]] = Field(
|
||||
default=None,
|
||||
description="Tags for categorization"
|
||||
)
|
||||
engineering_context: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Real-world engineering context"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Model Configuration Models
|
||||
# ============================================================================
|
||||
|
||||
class NxPartConfig(BaseModel):
|
||||
"""NX geometry part file configuration."""
|
||||
path: Optional[str] = Field(default=None, description="Path to .prt file")
|
||||
hash: Optional[str] = Field(default=None, description="File hash for change detection")
|
||||
idealized_part: Optional[str] = Field(default=None, description="Idealized part filename (_i.prt)")
|
||||
|
||||
|
||||
class FemConfig(BaseModel):
|
||||
"""FEM mesh file configuration."""
|
||||
path: Optional[str] = Field(default=None, description="Path to .fem file")
|
||||
element_count: Optional[int] = Field(default=None, description="Number of elements")
|
||||
node_count: Optional[int] = Field(default=None, description="Number of nodes")
|
||||
|
||||
|
||||
class Subcase(BaseModel):
|
||||
"""Simulation subcase definition."""
|
||||
id: int
|
||||
name: Optional[str] = None
|
||||
type: Optional[SubcaseType] = None
|
||||
|
||||
|
||||
class SimConfig(BaseModel):
|
||||
"""Simulation file configuration."""
|
||||
path: str = Field(..., description="Path to .sim file")
|
||||
solver: SolverType = Field(..., description="Solver type")
|
||||
solution_type: Optional[str] = Field(
|
||||
default=None,
|
||||
pattern=r"^SOL\d+$",
|
||||
description="Solution type (e.g., SOL101)"
|
||||
)
|
||||
subcases: Optional[List[Subcase]] = Field(default=None, description="Defined subcases")
|
||||
|
||||
|
||||
class NxSettings(BaseModel):
|
||||
"""NX runtime settings."""
|
||||
nx_install_path: Optional[str] = None
|
||||
simulation_timeout_s: Optional[int] = Field(default=None, ge=60, le=7200)
|
||||
auto_start_nx: Optional[bool] = None
|
||||
|
||||
|
||||
class ModelConfig(BaseModel):
|
||||
"""NX model files and configuration."""
|
||||
nx_part: Optional[NxPartConfig] = None
|
||||
fem: Optional[FemConfig] = None
|
||||
sim: SimConfig
|
||||
nx_settings: Optional[NxSettings] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Design Variable Models
|
||||
# ============================================================================
|
||||
|
||||
class DesignVariableBounds(BaseModel):
|
||||
"""Design variable bounds."""
|
||||
min: float
|
||||
max: float
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_bounds(self) -> 'DesignVariableBounds':
|
||||
if self.min >= self.max:
|
||||
raise ValueError(f"min ({self.min}) must be less than max ({self.max})")
|
||||
return self
|
||||
|
||||
|
||||
class DesignVariable(BaseModel):
|
||||
"""A design variable to optimize."""
|
||||
id: str = Field(
|
||||
...,
|
||||
pattern=r"^dv_\d{3}$",
|
||||
description="Unique identifier (pattern: dv_XXX)"
|
||||
)
|
||||
name: str = Field(..., description="Human-readable name")
|
||||
expression_name: str = Field(
|
||||
...,
|
||||
pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$",
|
||||
description="NX expression name (must match model)"
|
||||
)
|
||||
type: DesignVariableType = Field(..., description="Variable type")
|
||||
bounds: DesignVariableBounds = Field(..., description="Value bounds")
|
||||
baseline: Optional[float] = Field(default=None, description="Current/initial value")
|
||||
units: Optional[str] = Field(default=None, description="Physical units (mm, deg, etc.)")
|
||||
step: Optional[float] = Field(default=None, description="Step size for integer/discrete")
|
||||
enabled: bool = Field(default=True, description="Whether to include in optimization")
|
||||
description: Optional[str] = None
|
||||
canvas_position: Optional[CanvasPosition] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Extractor Models
|
||||
# ============================================================================
|
||||
|
||||
class ExtractorConfig(BaseModel):
|
||||
"""Type-specific extractor configuration."""
|
||||
inner_radius_mm: Optional[float] = None
|
||||
outer_radius_mm: Optional[float] = None
|
||||
n_modes: Optional[int] = None
|
||||
filter_low_orders: Optional[int] = None
|
||||
displacement_unit: Optional[str] = None
|
||||
reference_subcase: Optional[int] = None
|
||||
expression_name: Optional[str] = None
|
||||
mode_number: Optional[int] = None
|
||||
element_type: Optional[str] = None
|
||||
result_type: Optional[str] = None
|
||||
metric: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
extra = "allow" # Allow additional fields for flexibility
|
||||
|
||||
|
||||
class CustomFunction(BaseModel):
|
||||
"""Custom function definition for custom_function extractors."""
|
||||
name: Optional[str] = Field(default=None, description="Function name")
|
||||
module: Optional[str] = Field(default=None, description="Python module path")
|
||||
signature: Optional[str] = Field(default=None, description="Function signature")
|
||||
source_code: Optional[str] = Field(default=None, description="Python source code")
|
||||
|
||||
|
||||
class ExtractorOutput(BaseModel):
|
||||
"""Output definition for an extractor."""
|
||||
name: str = Field(..., description="Output name (used by objectives/constraints)")
|
||||
metric: Optional[str] = Field(default=None, description="Specific metric (max, total, rms, etc.)")
|
||||
subcase: Optional[int] = Field(default=None, description="Subcase ID for this output")
|
||||
units: Optional[str] = None
|
||||
|
||||
|
||||
class Extractor(BaseModel):
|
||||
"""Physics extractor that computes outputs from FEA."""
|
||||
id: str = Field(
|
||||
...,
|
||||
pattern=r"^ext_\d{3}$",
|
||||
description="Unique identifier (pattern: ext_XXX)"
|
||||
)
|
||||
name: str = Field(..., description="Human-readable name")
|
||||
type: ExtractorType = Field(..., description="Extractor type")
|
||||
builtin: bool = Field(default=True, description="Whether this is a built-in extractor")
|
||||
config: Optional[ExtractorConfig] = Field(default=None, description="Type-specific configuration")
|
||||
function: Optional[CustomFunction] = Field(
|
||||
default=None,
|
||||
description="Custom function definition (for custom_function type)"
|
||||
)
|
||||
outputs: List[ExtractorOutput] = Field(..., min_length=1, description="Output values")
|
||||
canvas_position: Optional[CanvasPosition] = None
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_custom_function(self) -> 'Extractor':
|
||||
if self.type == ExtractorType.CUSTOM_FUNCTION and self.function is None:
|
||||
raise ValueError("custom_function extractor requires function definition")
|
||||
return self
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Objective Models
|
||||
# ============================================================================
|
||||
|
||||
class ObjectiveSource(BaseModel):
|
||||
"""Source reference for objective value."""
|
||||
extractor_id: str = Field(..., description="Reference to extractor")
|
||||
output_name: str = Field(..., description="Which output from the extractor")
|
||||
|
||||
|
||||
class Objective(BaseModel):
|
||||
"""Optimization objective."""
|
||||
id: str = Field(
|
||||
...,
|
||||
pattern=r"^obj_\d{3}$",
|
||||
description="Unique identifier (pattern: obj_XXX)"
|
||||
)
|
||||
name: str = Field(..., description="Human-readable name")
|
||||
direction: OptimizationDirection = Field(..., description="Optimization direction")
|
||||
weight: float = Field(default=1.0, ge=0, description="Weight for weighted sum")
|
||||
source: ObjectiveSource = Field(..., description="Where the value comes from")
|
||||
target: Optional[float] = Field(default=None, description="Target value (for goal programming)")
|
||||
units: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
canvas_position: Optional[CanvasPosition] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Constraint Models
|
||||
# ============================================================================
|
||||
|
||||
class ConstraintSource(BaseModel):
|
||||
"""Source reference for constraint value."""
|
||||
extractor_id: str
|
||||
output_name: str
|
||||
|
||||
|
||||
class PenaltyConfig(BaseModel):
|
||||
"""Penalty method configuration for constraints."""
|
||||
method: Optional[PenaltyMethod] = None
|
||||
weight: Optional[float] = None
|
||||
margin: Optional[float] = Field(default=None, description="Soft margin before penalty kicks in")
|
||||
|
||||
|
||||
class Constraint(BaseModel):
|
||||
"""Hard or soft constraint."""
|
||||
id: str = Field(
|
||||
...,
|
||||
pattern=r"^con_\d{3}$",
|
||||
description="Unique identifier (pattern: con_XXX)"
|
||||
)
|
||||
name: str
|
||||
type: ConstraintType = Field(..., description="Constraint type")
|
||||
operator: ConstraintOperator = Field(..., description="Comparison operator")
|
||||
threshold: float = Field(..., description="Constraint threshold value")
|
||||
source: ConstraintSource = Field(..., description="Where the value comes from")
|
||||
penalty_config: Optional[PenaltyConfig] = None
|
||||
description: Optional[str] = None
|
||||
canvas_position: Optional[CanvasPosition] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Optimization Models
|
||||
# ============================================================================
|
||||
|
||||
class AlgorithmConfig(BaseModel):
|
||||
"""Algorithm-specific settings."""
|
||||
population_size: Optional[int] = None
|
||||
n_generations: Optional[int] = None
|
||||
mutation_prob: Optional[float] = None
|
||||
crossover_prob: Optional[float] = None
|
||||
seed: Optional[int] = None
|
||||
n_startup_trials: Optional[int] = None
|
||||
sigma0: Optional[float] = None
|
||||
|
||||
class Config:
|
||||
extra = "allow" # Allow additional algorithm-specific fields
|
||||
|
||||
|
||||
class Algorithm(BaseModel):
|
||||
"""Optimization algorithm configuration."""
|
||||
type: AlgorithmType
|
||||
config: Optional[AlgorithmConfig] = None
|
||||
|
||||
|
||||
class OptimizationBudget(BaseModel):
|
||||
"""Computational budget for optimization."""
|
||||
max_trials: Optional[int] = Field(default=None, ge=1, le=10000)
|
||||
max_time_hours: Optional[float] = None
|
||||
convergence_patience: Optional[int] = Field(
|
||||
default=None,
|
||||
description="Stop if no improvement for N trials"
|
||||
)
|
||||
|
||||
|
||||
class SurrogateConfig(BaseModel):
|
||||
"""Neural surrogate model configuration."""
|
||||
n_models: Optional[int] = None
|
||||
architecture: Optional[List[int]] = None
|
||||
train_every_n_trials: Optional[int] = None
|
||||
min_training_samples: Optional[int] = None
|
||||
acquisition_candidates: Optional[int] = None
|
||||
fea_validations_per_round: Optional[int] = None
|
||||
|
||||
|
||||
class Surrogate(BaseModel):
|
||||
"""Surrogate model settings."""
|
||||
enabled: Optional[bool] = None
|
||||
type: Optional[SurrogateType] = None
|
||||
config: Optional[SurrogateConfig] = None
|
||||
|
||||
|
||||
class OptimizationConfig(BaseModel):
|
||||
"""Optimization algorithm configuration."""
|
||||
algorithm: Algorithm
|
||||
budget: OptimizationBudget
|
||||
surrogate: Optional[Surrogate] = None
|
||||
canvas_position: Optional[CanvasPosition] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Workflow Models
|
||||
# ============================================================================
|
||||
|
||||
class WorkflowStage(BaseModel):
|
||||
"""A stage in a multi-stage optimization workflow."""
|
||||
id: str
|
||||
name: str
|
||||
algorithm: Optional[str] = None
|
||||
trials: Optional[int] = None
|
||||
purpose: Optional[str] = None
|
||||
|
||||
|
||||
class WorkflowTransition(BaseModel):
|
||||
"""Transition between workflow stages."""
|
||||
from_: str = Field(..., alias="from")
|
||||
to: str
|
||||
condition: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
|
||||
|
||||
class Workflow(BaseModel):
|
||||
"""Multi-stage optimization workflow."""
|
||||
stages: Optional[List[WorkflowStage]] = None
|
||||
transitions: Optional[List[WorkflowTransition]] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Reporting Models
|
||||
# ============================================================================
|
||||
|
||||
class InsightConfig(BaseModel):
|
||||
"""Insight-specific configuration."""
|
||||
include_html: Optional[bool] = None
|
||||
show_pareto_evolution: Optional[bool] = None
|
||||
|
||||
class Config:
|
||||
extra = "allow"
|
||||
|
||||
|
||||
class Insight(BaseModel):
|
||||
"""Reporting insight definition."""
|
||||
type: Optional[str] = None
|
||||
for_trials: Optional[str] = None
|
||||
config: Optional[InsightConfig] = None
|
||||
|
||||
|
||||
class ReportingConfig(BaseModel):
|
||||
"""Reporting configuration."""
|
||||
auto_report: Optional[bool] = None
|
||||
report_triggers: Optional[List[str]] = None
|
||||
insights: Optional[List[Insight]] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Canvas Models
|
||||
# ============================================================================
|
||||
|
||||
class CanvasViewport(BaseModel):
|
||||
"""Canvas viewport settings."""
|
||||
x: float = 0
|
||||
y: float = 0
|
||||
zoom: float = 1.0
|
||||
|
||||
|
||||
class CanvasEdge(BaseModel):
|
||||
"""Connection between canvas nodes."""
|
||||
source: str
|
||||
target: str
|
||||
sourceHandle: Optional[str] = None
|
||||
targetHandle: Optional[str] = None
|
||||
|
||||
|
||||
class CanvasGroup(BaseModel):
|
||||
"""Grouping of canvas nodes."""
|
||||
id: str
|
||||
name: str
|
||||
node_ids: List[str]
|
||||
|
||||
|
||||
class CanvasConfig(BaseModel):
|
||||
"""Canvas UI state (persisted for reconstruction)."""
|
||||
layout_version: Optional[str] = None
|
||||
viewport: Optional[CanvasViewport] = None
|
||||
edges: Optional[List[CanvasEdge]] = None
|
||||
groups: Optional[List[CanvasGroup]] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Main AtomizerSpec Model
|
||||
# ============================================================================
|
||||
|
||||
class AtomizerSpec(BaseModel):
|
||||
"""
|
||||
AtomizerSpec v2.0 - The unified configuration schema for Atomizer optimization studies.
|
||||
|
||||
This is the single source of truth used by:
|
||||
- Canvas UI (rendering and editing)
|
||||
- Backend API (validation and storage)
|
||||
- Claude Assistant (reading and modifying)
|
||||
- Optimization Engine (execution)
|
||||
"""
|
||||
meta: SpecMeta = Field(..., description="Metadata about the spec")
|
||||
model: ModelConfig = Field(..., description="NX model files and configuration")
|
||||
design_variables: List[DesignVariable] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=50,
|
||||
description="Design variables to optimize"
|
||||
)
|
||||
extractors: List[Extractor] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Physics extractors"
|
||||
)
|
||||
objectives: List[Objective] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=5,
|
||||
description="Optimization objectives"
|
||||
)
|
||||
constraints: Optional[List[Constraint]] = Field(
|
||||
default=None,
|
||||
description="Hard and soft constraints"
|
||||
)
|
||||
optimization: OptimizationConfig = Field(..., description="Algorithm configuration")
|
||||
workflow: Optional[Workflow] = Field(default=None, description="Multi-stage workflow")
|
||||
reporting: Optional[ReportingConfig] = Field(default=None, description="Reporting config")
|
||||
canvas: Optional[CanvasConfig] = Field(default=None, description="Canvas UI state")
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_references(self) -> 'AtomizerSpec':
|
||||
"""Validate that all references are valid."""
|
||||
# Collect valid extractor IDs and their outputs
|
||||
extractor_outputs: Dict[str, set] = {}
|
||||
for ext in self.extractors:
|
||||
extractor_outputs[ext.id] = {o.name for o in ext.outputs}
|
||||
|
||||
# Validate objective sources
|
||||
for obj in self.objectives:
|
||||
if obj.source.extractor_id not in extractor_outputs:
|
||||
raise ValueError(
|
||||
f"Objective '{obj.name}' references unknown extractor: {obj.source.extractor_id}"
|
||||
)
|
||||
if obj.source.output_name not in extractor_outputs[obj.source.extractor_id]:
|
||||
raise ValueError(
|
||||
f"Objective '{obj.name}' references unknown output: {obj.source.output_name}"
|
||||
)
|
||||
|
||||
# Validate constraint sources
|
||||
if self.constraints:
|
||||
for con in self.constraints:
|
||||
if con.source.extractor_id not in extractor_outputs:
|
||||
raise ValueError(
|
||||
f"Constraint '{con.name}' references unknown extractor: {con.source.extractor_id}"
|
||||
)
|
||||
if con.source.output_name not in extractor_outputs[con.source.extractor_id]:
|
||||
raise ValueError(
|
||||
f"Constraint '{con.name}' references unknown output: {con.source.output_name}"
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def get_enabled_design_variables(self) -> List[DesignVariable]:
|
||||
"""Return only enabled design variables."""
|
||||
return [dv for dv in self.design_variables if dv.enabled]
|
||||
|
||||
def get_extractor_by_id(self, extractor_id: str) -> Optional[Extractor]:
|
||||
"""Find an extractor by ID."""
|
||||
for ext in self.extractors:
|
||||
if ext.id == extractor_id:
|
||||
return ext
|
||||
return None
|
||||
|
||||
def get_objective_by_id(self, objective_id: str) -> Optional[Objective]:
|
||||
"""Find an objective by ID."""
|
||||
for obj in self.objectives:
|
||||
if obj.id == objective_id:
|
||||
return obj
|
||||
return None
|
||||
|
||||
def get_constraint_by_id(self, constraint_id: str) -> Optional[Constraint]:
|
||||
"""Find a constraint by ID."""
|
||||
if not self.constraints:
|
||||
return None
|
||||
for con in self.constraints:
|
||||
if con.id == constraint_id:
|
||||
return con
|
||||
return None
|
||||
|
||||
def has_custom_extractors(self) -> bool:
|
||||
"""Check if spec has any custom function extractors."""
|
||||
return any(ext.type == ExtractorType.CUSTOM_FUNCTION for ext in self.extractors)
|
||||
|
||||
def is_multi_objective(self) -> bool:
|
||||
"""Check if this is a multi-objective optimization."""
|
||||
return len(self.objectives) > 1
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Validation Response Models
|
||||
# ============================================================================
|
||||
|
||||
class ValidationError(BaseModel):
|
||||
"""A validation error."""
|
||||
type: str # 'schema', 'semantic', 'reference'
|
||||
path: List[str]
|
||||
message: str
|
||||
|
||||
|
||||
class ValidationWarning(BaseModel):
|
||||
"""A validation warning."""
|
||||
type: str
|
||||
path: List[str]
|
||||
message: str
|
||||
|
||||
|
||||
class ValidationSummary(BaseModel):
|
||||
"""Summary of spec contents."""
|
||||
design_variables: int
|
||||
extractors: int
|
||||
objectives: int
|
||||
constraints: int
|
||||
custom_functions: int
|
||||
|
||||
|
||||
class ValidationReport(BaseModel):
|
||||
"""Full validation report."""
|
||||
valid: bool
|
||||
errors: List[ValidationError]
|
||||
warnings: List[ValidationWarning]
|
||||
summary: ValidationSummary
|
||||
654
optimization_engine/config/spec_validator.py
Normal file
654
optimization_engine/config/spec_validator.py
Normal file
@@ -0,0 +1,654 @@
|
||||
"""
|
||||
AtomizerSpec v2.0 Validator
|
||||
|
||||
Provides comprehensive validation including:
|
||||
- JSON Schema validation
|
||||
- Pydantic model validation
|
||||
- Semantic validation (bounds, references, dependencies)
|
||||
- Extractor-specific validation
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from pydantic import ValidationError as PydanticValidationError
|
||||
|
||||
try:
|
||||
import jsonschema
|
||||
HAS_JSONSCHEMA = True
|
||||
except ImportError:
|
||||
HAS_JSONSCHEMA = False
|
||||
|
||||
from .spec_models import (
|
||||
AtomizerSpec,
|
||||
ValidationReport,
|
||||
ValidationError,
|
||||
ValidationWarning,
|
||||
ValidationSummary,
|
||||
ExtractorType,
|
||||
AlgorithmType,
|
||||
ConstraintType,
|
||||
)
|
||||
|
||||
|
||||
class SpecValidationError(Exception):
|
||||
"""Raised when spec validation fails."""
|
||||
|
||||
def __init__(self, message: str, errors: List[ValidationError] = None):
|
||||
super().__init__(message)
|
||||
self.errors = errors or []
|
||||
|
||||
|
||||
class SpecValidator:
|
||||
"""
|
||||
Validates AtomizerSpec v2.0 configurations.
|
||||
|
||||
Provides three levels of validation:
|
||||
1. JSON Schema validation (structural)
|
||||
2. Pydantic model validation (type safety)
|
||||
3. Semantic validation (business logic)
|
||||
"""
|
||||
|
||||
# Path to JSON Schema file
|
||||
SCHEMA_PATH = Path(__file__).parent.parent / "schemas" / "atomizer_spec_v2.json"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize validator with schema."""
|
||||
self._schema: Optional[Dict] = None
|
||||
|
||||
@property
|
||||
def schema(self) -> Dict:
|
||||
"""Lazy load the JSON Schema."""
|
||||
if self._schema is None:
|
||||
if self.SCHEMA_PATH.exists():
|
||||
with open(self.SCHEMA_PATH) as f:
|
||||
self._schema = json.load(f)
|
||||
else:
|
||||
self._schema = {}
|
||||
return self._schema
|
||||
|
||||
def validate(
|
||||
self,
|
||||
spec_data: Union[Dict[str, Any], AtomizerSpec],
|
||||
strict: bool = True
|
||||
) -> ValidationReport:
|
||||
"""
|
||||
Validate a spec and return a detailed report.
|
||||
|
||||
Args:
|
||||
spec_data: Either a dict or AtomizerSpec instance
|
||||
strict: If True, raise exception on errors; if False, return report only
|
||||
|
||||
Returns:
|
||||
ValidationReport with errors, warnings, and summary
|
||||
|
||||
Raises:
|
||||
SpecValidationError: If strict=True and validation fails
|
||||
"""
|
||||
errors: List[ValidationError] = []
|
||||
warnings: List[ValidationWarning] = []
|
||||
|
||||
# Convert to dict if needed
|
||||
if isinstance(spec_data, AtomizerSpec):
|
||||
data = spec_data.model_dump(mode='json')
|
||||
else:
|
||||
data = spec_data
|
||||
|
||||
# Phase 1: JSON Schema validation
|
||||
schema_errors = self._validate_json_schema(data)
|
||||
errors.extend(schema_errors)
|
||||
|
||||
# Phase 2: Pydantic model validation (only if schema passes)
|
||||
if not schema_errors:
|
||||
pydantic_errors = self._validate_pydantic(data)
|
||||
errors.extend(pydantic_errors)
|
||||
|
||||
# Phase 3: Semantic validation (only if pydantic passes)
|
||||
if not errors:
|
||||
spec = AtomizerSpec.model_validate(data)
|
||||
semantic_errors, semantic_warnings = self._validate_semantic(spec)
|
||||
errors.extend(semantic_errors)
|
||||
warnings.extend(semantic_warnings)
|
||||
|
||||
# Build summary
|
||||
summary = self._build_summary(data)
|
||||
|
||||
# Build report
|
||||
report = ValidationReport(
|
||||
valid=len(errors) == 0,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
summary=summary
|
||||
)
|
||||
|
||||
# Raise if strict mode and errors found
|
||||
if strict and not report.valid:
|
||||
error_messages = "; ".join(e.message for e in report.errors[:3])
|
||||
raise SpecValidationError(
|
||||
f"Spec validation failed: {error_messages}",
|
||||
errors=report.errors
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
def validate_partial(
|
||||
self,
|
||||
path: str,
|
||||
value: Any,
|
||||
current_spec: AtomizerSpec
|
||||
) -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate a partial update before applying.
|
||||
|
||||
Args:
|
||||
path: JSONPath to the field being updated
|
||||
value: New value
|
||||
current_spec: Current full spec
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, list of error messages)
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Parse path
|
||||
parts = self._parse_path(path)
|
||||
if not parts:
|
||||
return False, ["Invalid path format"]
|
||||
|
||||
# Get target type from path
|
||||
root = parts[0]
|
||||
|
||||
# Validate based on root section
|
||||
if root == "design_variables":
|
||||
errors.extend(self._validate_dv_update(parts, value, current_spec))
|
||||
elif root == "extractors":
|
||||
errors.extend(self._validate_extractor_update(parts, value, current_spec))
|
||||
elif root == "objectives":
|
||||
errors.extend(self._validate_objective_update(parts, value, current_spec))
|
||||
elif root == "constraints":
|
||||
errors.extend(self._validate_constraint_update(parts, value, current_spec))
|
||||
elif root == "optimization":
|
||||
errors.extend(self._validate_optimization_update(parts, value))
|
||||
elif root == "meta":
|
||||
errors.extend(self._validate_meta_update(parts, value))
|
||||
|
||||
return len(errors) == 0, errors
|
||||
|
||||
def _validate_json_schema(self, data: Dict) -> List[ValidationError]:
|
||||
"""Validate against JSON Schema."""
|
||||
errors = []
|
||||
|
||||
if not HAS_JSONSCHEMA or not self.schema:
|
||||
return errors # Skip if jsonschema not available
|
||||
|
||||
try:
|
||||
jsonschema.validate(instance=data, schema=self.schema)
|
||||
except jsonschema.ValidationError as e:
|
||||
errors.append(ValidationError(
|
||||
type="schema",
|
||||
path=list(e.absolute_path),
|
||||
message=e.message
|
||||
))
|
||||
except jsonschema.SchemaError as e:
|
||||
errors.append(ValidationError(
|
||||
type="schema",
|
||||
path=[],
|
||||
message=f"Invalid schema: {e.message}"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_pydantic(self, data: Dict) -> List[ValidationError]:
|
||||
"""Validate using Pydantic models."""
|
||||
errors = []
|
||||
|
||||
try:
|
||||
AtomizerSpec.model_validate(data)
|
||||
except PydanticValidationError as e:
|
||||
for err in e.errors():
|
||||
errors.append(ValidationError(
|
||||
type="schema",
|
||||
path=[str(p) for p in err.get("loc", [])],
|
||||
message=err.get("msg", "Validation error")
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_semantic(
|
||||
self,
|
||||
spec: AtomizerSpec
|
||||
) -> Tuple[List[ValidationError], List[ValidationWarning]]:
|
||||
"""
|
||||
Perform semantic validation.
|
||||
|
||||
Checks business logic and constraints that can't be expressed in schema.
|
||||
"""
|
||||
errors: List[ValidationError] = []
|
||||
warnings: List[ValidationWarning] = []
|
||||
|
||||
# Validate design variable bounds
|
||||
errors.extend(self._validate_dv_bounds(spec))
|
||||
|
||||
# Validate extractor configurations
|
||||
errors.extend(self._validate_extractor_configs(spec))
|
||||
warnings.extend(self._warn_extractor_configs(spec))
|
||||
|
||||
# Validate reference integrity (done in Pydantic, but double-check)
|
||||
errors.extend(self._validate_references(spec))
|
||||
|
||||
# Validate optimization settings
|
||||
errors.extend(self._validate_optimization_settings(spec))
|
||||
warnings.extend(self._warn_optimization_settings(spec))
|
||||
|
||||
# Validate canvas edges
|
||||
warnings.extend(self._validate_canvas_edges(spec))
|
||||
|
||||
# Check for duplicate IDs
|
||||
errors.extend(self._validate_unique_ids(spec))
|
||||
|
||||
# Validate custom function syntax
|
||||
errors.extend(self._validate_custom_functions(spec))
|
||||
|
||||
return errors, warnings
|
||||
|
||||
def _validate_dv_bounds(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate design variable bounds."""
|
||||
errors = []
|
||||
|
||||
for i, dv in enumerate(spec.design_variables):
|
||||
# Check baseline within bounds
|
||||
if dv.baseline is not None:
|
||||
if dv.baseline < dv.bounds.min or dv.baseline > dv.bounds.max:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["design_variables", str(i), "baseline"],
|
||||
message=f"Baseline {dv.baseline} outside bounds [{dv.bounds.min}, {dv.bounds.max}]"
|
||||
))
|
||||
|
||||
# Check step size for integer type
|
||||
if dv.type.value == "integer":
|
||||
range_size = dv.bounds.max - dv.bounds.min
|
||||
if range_size < 1:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["design_variables", str(i), "bounds"],
|
||||
message="Integer variable must have range >= 1"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_extractor_configs(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate extractor-specific configurations."""
|
||||
errors = []
|
||||
|
||||
for i, ext in enumerate(spec.extractors):
|
||||
# Zernike extractors need specific config
|
||||
if ext.type in [ExtractorType.ZERNIKE_OPD, ExtractorType.ZERNIKE_CSV]:
|
||||
if not ext.config:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "config"],
|
||||
message=f"Zernike extractor requires config with radius settings"
|
||||
))
|
||||
elif ext.config:
|
||||
if ext.config.inner_radius_mm is None:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "config", "inner_radius_mm"],
|
||||
message="Zernike extractor requires inner_radius_mm"
|
||||
))
|
||||
if ext.config.outer_radius_mm is None:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "config", "outer_radius_mm"],
|
||||
message="Zernike extractor requires outer_radius_mm"
|
||||
))
|
||||
|
||||
# Mass expression extractor needs expression_name
|
||||
if ext.type == ExtractorType.MASS_EXPRESSION:
|
||||
if not ext.config or not ext.config.expression_name:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "config", "expression_name"],
|
||||
message="Mass expression extractor requires expression_name in config"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _warn_extractor_configs(self, spec: AtomizerSpec) -> List[ValidationWarning]:
|
||||
"""Generate warnings for extractor configurations."""
|
||||
warnings = []
|
||||
|
||||
for i, ext in enumerate(spec.extractors):
|
||||
# Zernike mode count warning
|
||||
if ext.type in [ExtractorType.ZERNIKE_OPD, ExtractorType.ZERNIKE_CSV]:
|
||||
if ext.config and ext.config.n_modes:
|
||||
if ext.config.n_modes > 66:
|
||||
warnings.append(ValidationWarning(
|
||||
type="performance",
|
||||
path=["extractors", str(i), "config", "n_modes"],
|
||||
message=f"n_modes={ext.config.n_modes} is high; consider <=66 for performance"
|
||||
))
|
||||
|
||||
return warnings
|
||||
|
||||
def _validate_references(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate reference integrity."""
|
||||
errors = []
|
||||
|
||||
# Collect all valid IDs
|
||||
dv_ids = {dv.id for dv in spec.design_variables}
|
||||
ext_ids = {ext.id for ext in spec.extractors}
|
||||
ext_outputs: Dict[str, set] = {}
|
||||
for ext in spec.extractors:
|
||||
ext_outputs[ext.id] = {o.name for o in ext.outputs}
|
||||
|
||||
# Validate canvas edges
|
||||
if spec.canvas and spec.canvas.edges:
|
||||
all_ids = dv_ids | ext_ids
|
||||
all_ids.add("model")
|
||||
all_ids.add("solver")
|
||||
all_ids.add("optimization")
|
||||
all_ids.update(obj.id for obj in spec.objectives)
|
||||
if spec.constraints:
|
||||
all_ids.update(con.id for con in spec.constraints)
|
||||
|
||||
for i, edge in enumerate(spec.canvas.edges):
|
||||
if edge.source not in all_ids:
|
||||
errors.append(ValidationError(
|
||||
type="reference",
|
||||
path=["canvas", "edges", str(i), "source"],
|
||||
message=f"Edge source '{edge.source}' not found"
|
||||
))
|
||||
if edge.target not in all_ids:
|
||||
errors.append(ValidationError(
|
||||
type="reference",
|
||||
path=["canvas", "edges", str(i), "target"],
|
||||
message=f"Edge target '{edge.target}' not found"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_optimization_settings(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate optimization settings."""
|
||||
errors = []
|
||||
|
||||
algo_type = spec.optimization.algorithm.type
|
||||
|
||||
# NSGA-II requires multiple objectives
|
||||
if algo_type == AlgorithmType.NSGA_II and len(spec.objectives) < 2:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["optimization", "algorithm", "type"],
|
||||
message="NSGA-II requires at least 2 objectives"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _warn_optimization_settings(self, spec: AtomizerSpec) -> List[ValidationWarning]:
|
||||
"""Generate warnings for optimization settings."""
|
||||
warnings = []
|
||||
|
||||
budget = spec.optimization.budget
|
||||
|
||||
# Warn about small trial budgets
|
||||
if budget.max_trials and budget.max_trials < 20:
|
||||
warnings.append(ValidationWarning(
|
||||
type="recommendation",
|
||||
path=["optimization", "budget", "max_trials"],
|
||||
message=f"max_trials={budget.max_trials} is low; recommend >= 20 for convergence"
|
||||
))
|
||||
|
||||
# Warn about large design space with small budget
|
||||
num_dvs = len(spec.get_enabled_design_variables())
|
||||
if budget.max_trials and num_dvs > 5 and budget.max_trials < num_dvs * 10:
|
||||
warnings.append(ValidationWarning(
|
||||
type="recommendation",
|
||||
path=["optimization", "budget", "max_trials"],
|
||||
message=f"{num_dvs} DVs suggest at least {num_dvs * 10} trials"
|
||||
))
|
||||
|
||||
return warnings
|
||||
|
||||
def _validate_canvas_edges(self, spec: AtomizerSpec) -> List[ValidationWarning]:
|
||||
"""Validate canvas edge structure."""
|
||||
warnings = []
|
||||
|
||||
if not spec.canvas or not spec.canvas.edges:
|
||||
warnings.append(ValidationWarning(
|
||||
type="completeness",
|
||||
path=["canvas", "edges"],
|
||||
message="No canvas edges defined; canvas may not render correctly"
|
||||
))
|
||||
|
||||
return warnings
|
||||
|
||||
def _validate_unique_ids(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate that all IDs are unique."""
|
||||
errors = []
|
||||
seen_ids: Dict[str, str] = {}
|
||||
|
||||
# Check all ID-bearing elements
|
||||
for i, dv in enumerate(spec.design_variables):
|
||||
if dv.id in seen_ids:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["design_variables", str(i), "id"],
|
||||
message=f"Duplicate ID '{dv.id}' (also in {seen_ids[dv.id]})"
|
||||
))
|
||||
seen_ids[dv.id] = f"design_variables[{i}]"
|
||||
|
||||
for i, ext in enumerate(spec.extractors):
|
||||
if ext.id in seen_ids:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "id"],
|
||||
message=f"Duplicate ID '{ext.id}' (also in {seen_ids[ext.id]})"
|
||||
))
|
||||
seen_ids[ext.id] = f"extractors[{i}]"
|
||||
|
||||
for i, obj in enumerate(spec.objectives):
|
||||
if obj.id in seen_ids:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["objectives", str(i), "id"],
|
||||
message=f"Duplicate ID '{obj.id}' (also in {seen_ids[obj.id]})"
|
||||
))
|
||||
seen_ids[obj.id] = f"objectives[{i}]"
|
||||
|
||||
if spec.constraints:
|
||||
for i, con in enumerate(spec.constraints):
|
||||
if con.id in seen_ids:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["constraints", str(i), "id"],
|
||||
message=f"Duplicate ID '{con.id}' (also in {seen_ids[con.id]})"
|
||||
))
|
||||
seen_ids[con.id] = f"constraints[{i}]"
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_custom_functions(self, spec: AtomizerSpec) -> List[ValidationError]:
|
||||
"""Validate custom function Python syntax."""
|
||||
errors = []
|
||||
|
||||
for i, ext in enumerate(spec.extractors):
|
||||
if ext.type == ExtractorType.CUSTOM_FUNCTION and ext.function:
|
||||
if ext.function.source_code:
|
||||
try:
|
||||
compile(ext.function.source_code, f"<custom:{ext.name}>", "exec")
|
||||
except SyntaxError as e:
|
||||
errors.append(ValidationError(
|
||||
type="semantic",
|
||||
path=["extractors", str(i), "function", "source_code"],
|
||||
message=f"Python syntax error: {e.msg} at line {e.lineno}"
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
def _build_summary(self, data: Dict) -> ValidationSummary:
|
||||
"""Build validation summary."""
|
||||
extractors = data.get("extractors", [])
|
||||
custom_count = sum(
|
||||
1 for e in extractors
|
||||
if e.get("type") == "custom_function" or not e.get("builtin", True)
|
||||
)
|
||||
|
||||
return ValidationSummary(
|
||||
design_variables=len(data.get("design_variables", [])),
|
||||
extractors=len(extractors),
|
||||
objectives=len(data.get("objectives", [])),
|
||||
constraints=len(data.get("constraints", []) or []),
|
||||
custom_functions=custom_count
|
||||
)
|
||||
|
||||
def _parse_path(self, path: str) -> List[str]:
|
||||
"""Parse a JSONPath-style path into parts."""
|
||||
import re
|
||||
# Handle both dot notation and bracket notation
|
||||
# e.g., "design_variables[0].bounds.max" or "objectives.0.weight"
|
||||
parts = []
|
||||
for part in re.split(r'\.|\[|\]', path):
|
||||
if part:
|
||||
parts.append(part)
|
||||
return parts
|
||||
|
||||
def _validate_dv_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any,
|
||||
spec: AtomizerSpec
|
||||
) -> List[str]:
|
||||
"""Validate a design variable update."""
|
||||
errors = []
|
||||
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
idx = int(parts[1])
|
||||
if idx >= len(spec.design_variables):
|
||||
errors.append(f"Design variable index {idx} out of range")
|
||||
except ValueError:
|
||||
errors.append(f"Invalid design variable index: {parts[1]}")
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_extractor_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any,
|
||||
spec: AtomizerSpec
|
||||
) -> List[str]:
|
||||
"""Validate an extractor update."""
|
||||
errors = []
|
||||
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
idx = int(parts[1])
|
||||
if idx >= len(spec.extractors):
|
||||
errors.append(f"Extractor index {idx} out of range")
|
||||
except ValueError:
|
||||
errors.append(f"Invalid extractor index: {parts[1]}")
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_objective_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any,
|
||||
spec: AtomizerSpec
|
||||
) -> List[str]:
|
||||
"""Validate an objective update."""
|
||||
errors = []
|
||||
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
idx = int(parts[1])
|
||||
if idx >= len(spec.objectives):
|
||||
errors.append(f"Objective index {idx} out of range")
|
||||
except ValueError:
|
||||
errors.append(f"Invalid objective index: {parts[1]}")
|
||||
|
||||
# Validate weight
|
||||
if len(parts) >= 3 and parts[2] == "weight":
|
||||
if not isinstance(value, (int, float)) or value < 0:
|
||||
errors.append("Weight must be a non-negative number")
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_constraint_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any,
|
||||
spec: AtomizerSpec
|
||||
) -> List[str]:
|
||||
"""Validate a constraint update."""
|
||||
errors = []
|
||||
|
||||
if not spec.constraints:
|
||||
errors.append("No constraints defined")
|
||||
return errors
|
||||
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
idx = int(parts[1])
|
||||
if idx >= len(spec.constraints):
|
||||
errors.append(f"Constraint index {idx} out of range")
|
||||
except ValueError:
|
||||
errors.append(f"Invalid constraint index: {parts[1]}")
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_optimization_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any
|
||||
) -> List[str]:
|
||||
"""Validate an optimization update."""
|
||||
errors = []
|
||||
|
||||
if len(parts) >= 2:
|
||||
if parts[1] == "algorithm" and len(parts) >= 3:
|
||||
if parts[2] == "type":
|
||||
valid_types = [t.value for t in AlgorithmType]
|
||||
if value not in valid_types:
|
||||
errors.append(f"Invalid algorithm type. Valid: {valid_types}")
|
||||
|
||||
return errors
|
||||
|
||||
def _validate_meta_update(
|
||||
self,
|
||||
parts: List[str],
|
||||
value: Any
|
||||
) -> List[str]:
|
||||
"""Validate a meta update."""
|
||||
errors = []
|
||||
|
||||
if len(parts) >= 2:
|
||||
if parts[1] == "study_name":
|
||||
import re
|
||||
if not re.match(r"^[a-z0-9_]+$", str(value)):
|
||||
errors.append("study_name must be snake_case (lowercase, numbers, underscores)")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# Module-level convenience function
|
||||
def validate_spec(
|
||||
spec_data: Union[Dict[str, Any], AtomizerSpec],
|
||||
strict: bool = True
|
||||
) -> ValidationReport:
|
||||
"""
|
||||
Validate an AtomizerSpec.
|
||||
|
||||
Args:
|
||||
spec_data: Spec data (dict or AtomizerSpec)
|
||||
strict: Raise exception on errors
|
||||
|
||||
Returns:
|
||||
ValidationReport
|
||||
|
||||
Raises:
|
||||
SpecValidationError: If strict=True and validation fails
|
||||
"""
|
||||
validator = SpecValidator()
|
||||
return validator.validate(spec_data, strict=strict)
|
||||
541
optimization_engine/extractors/custom_extractor_loader.py
Normal file
541
optimization_engine/extractors/custom_extractor_loader.py
Normal file
@@ -0,0 +1,541 @@
|
||||
"""
|
||||
Custom Extractor Loader
|
||||
|
||||
Dynamically loads and executes custom Python extractors defined in AtomizerSpec v2.0.
|
||||
Provides sandboxed execution with access to FEA results and common analysis libraries.
|
||||
|
||||
P3.9: Custom extractor runtime loader
|
||||
"""
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Lazy imports for optional dependencies
|
||||
_PYOP2 = None
|
||||
_SCIPY = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Allowed modules for custom extractors (sandboxed environment)
|
||||
# ============================================================================
|
||||
|
||||
ALLOWED_MODULES = {
|
||||
# Core Python
|
||||
"math",
|
||||
"statistics",
|
||||
"collections",
|
||||
"itertools",
|
||||
"functools",
|
||||
# Scientific computing
|
||||
"numpy",
|
||||
"scipy",
|
||||
"scipy.interpolate",
|
||||
"scipy.optimize",
|
||||
"scipy.integrate",
|
||||
"scipy.linalg",
|
||||
# FEA result parsing
|
||||
"pyNastran",
|
||||
"pyNastran.op2",
|
||||
"pyNastran.op2.op2",
|
||||
"pyNastran.bdf",
|
||||
"pyNastran.bdf.bdf",
|
||||
# Atomizer extractors
|
||||
"optimization_engine.extractors",
|
||||
}
|
||||
|
||||
BLOCKED_MODULES = {
|
||||
"os",
|
||||
"subprocess",
|
||||
"shutil",
|
||||
"sys",
|
||||
"builtins",
|
||||
"__builtins__",
|
||||
"importlib",
|
||||
"eval",
|
||||
"exec",
|
||||
"compile",
|
||||
"open",
|
||||
"file",
|
||||
"socket",
|
||||
"requests",
|
||||
"urllib",
|
||||
"http",
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Code Validation
|
||||
# ============================================================================
|
||||
|
||||
class ExtractorSecurityError(Exception):
|
||||
"""Raised when custom extractor code contains disallowed patterns."""
|
||||
pass
|
||||
|
||||
|
||||
class ExtractorValidationError(Exception):
|
||||
"""Raised when custom extractor code is invalid."""
|
||||
pass
|
||||
|
||||
|
||||
def validate_extractor_code(code: str, function_name: str) -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate custom extractor code for security and correctness.
|
||||
|
||||
Args:
|
||||
code: Python source code string
|
||||
function_name: Expected function name to find in code
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, list of error messages)
|
||||
|
||||
Raises:
|
||||
ExtractorSecurityError: If dangerous patterns detected
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Check for syntax errors first
|
||||
try:
|
||||
tree = ast.parse(code)
|
||||
except SyntaxError as e:
|
||||
return False, [f"Syntax error: {e}"]
|
||||
|
||||
# Check for disallowed patterns
|
||||
dangerous_patterns = [
|
||||
(r'\bexec\s*\(', 'exec() is not allowed'),
|
||||
(r'\beval\s*\(', 'eval() is not allowed'),
|
||||
(r'\bcompile\s*\(', 'compile() is not allowed'),
|
||||
(r'\b__import__\s*\(', '__import__() is not allowed'),
|
||||
(r'\bopen\s*\(', 'open() is not allowed - use op2_path parameter'),
|
||||
(r'\bos\.(system|popen|spawn|exec)', 'os.system/popen/spawn/exec is not allowed'),
|
||||
(r'\bsubprocess\.', 'subprocess module is not allowed'),
|
||||
(r'\bshutil\.', 'shutil module is not allowed'),
|
||||
(r'import\s+os\b', 'import os is not allowed'),
|
||||
(r'from\s+os\b', 'from os import is not allowed'),
|
||||
(r'import\s+subprocess', 'import subprocess is not allowed'),
|
||||
(r'import\s+sys\b', 'import sys is not allowed'),
|
||||
]
|
||||
|
||||
for pattern, message in dangerous_patterns:
|
||||
if re.search(pattern, code):
|
||||
raise ExtractorSecurityError(message)
|
||||
|
||||
# Check that the expected function exists
|
||||
function_found = False
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.FunctionDef) and node.name == function_name:
|
||||
function_found = True
|
||||
|
||||
# Check function signature
|
||||
args = node.args
|
||||
arg_names = [arg.arg for arg in args.args]
|
||||
|
||||
# Must have op2_path as first argument (or op2_result/results)
|
||||
valid_first_args = {'op2_path', 'op2_result', 'results', 'data'}
|
||||
if not arg_names or arg_names[0] not in valid_first_args:
|
||||
errors.append(
|
||||
f"Function {function_name} must have first argument from: "
|
||||
f"{valid_first_args}, got: {arg_names[0] if arg_names else 'none'}"
|
||||
)
|
||||
break
|
||||
|
||||
if not function_found:
|
||||
errors.append(f"Function '{function_name}' not found in code")
|
||||
|
||||
# Check imports
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
module = alias.name.split('.')[0]
|
||||
if module in BLOCKED_MODULES:
|
||||
errors.append(f"Import of '{alias.name}' is not allowed")
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module:
|
||||
module = node.module.split('.')[0]
|
||||
if module in BLOCKED_MODULES:
|
||||
errors.append(f"Import from '{node.module}' is not allowed")
|
||||
|
||||
return len(errors) == 0, errors
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Extractor Compilation and Execution
|
||||
# ============================================================================
|
||||
|
||||
class CustomExtractorContext:
|
||||
"""
|
||||
Execution context for custom extractors.
|
||||
Provides safe access to FEA results and common utilities.
|
||||
"""
|
||||
|
||||
def __init__(self, op2_path: Optional[Path] = None,
|
||||
bdf_path: Optional[Path] = None,
|
||||
working_dir: Optional[Path] = None,
|
||||
params: Optional[Dict[str, float]] = None):
|
||||
"""
|
||||
Initialize extractor context.
|
||||
|
||||
Args:
|
||||
op2_path: Path to OP2 results file
|
||||
bdf_path: Path to BDF model file
|
||||
working_dir: Working directory for the trial
|
||||
params: Current design parameters
|
||||
"""
|
||||
self.op2_path = Path(op2_path) if op2_path else None
|
||||
self.bdf_path = Path(bdf_path) if bdf_path else None
|
||||
self.working_dir = Path(working_dir) if working_dir else None
|
||||
self.params = params or {}
|
||||
|
||||
# Lazy-loaded results
|
||||
self._op2_result = None
|
||||
self._bdf_model = None
|
||||
|
||||
@property
|
||||
def op2_result(self):
|
||||
"""Lazy-load OP2 results."""
|
||||
if self._op2_result is None and self.op2_path and self.op2_path.exists():
|
||||
global _PYOP2
|
||||
if _PYOP2 is None:
|
||||
from pyNastran.op2.op2 import OP2
|
||||
_PYOP2 = OP2
|
||||
self._op2_result = _PYOP2(str(self.op2_path), debug=False)
|
||||
return self._op2_result
|
||||
|
||||
@property
|
||||
def bdf_model(self):
|
||||
"""Lazy-load BDF model."""
|
||||
if self._bdf_model is None and self.bdf_path and self.bdf_path.exists():
|
||||
from pyNastran.bdf.bdf import BDF
|
||||
self._bdf_model = BDF(debug=False)
|
||||
self._bdf_model.read_bdf(str(self.bdf_path))
|
||||
return self._bdf_model
|
||||
|
||||
|
||||
class CustomExtractor:
|
||||
"""
|
||||
Compiled custom extractor ready for execution.
|
||||
"""
|
||||
|
||||
def __init__(self, extractor_id: str, name: str, function_name: str,
|
||||
code: str, outputs: List[Dict[str, Any]], dependencies: List[str] = None):
|
||||
"""
|
||||
Initialize custom extractor.
|
||||
|
||||
Args:
|
||||
extractor_id: Unique extractor ID
|
||||
name: Human-readable name
|
||||
function_name: Name of the extraction function
|
||||
code: Python source code
|
||||
outputs: List of output definitions
|
||||
dependencies: Optional list of required pip packages
|
||||
"""
|
||||
self.extractor_id = extractor_id
|
||||
self.name = name
|
||||
self.function_name = function_name
|
||||
self.code = code
|
||||
self.outputs = outputs
|
||||
self.dependencies = dependencies or []
|
||||
|
||||
# Compiled function
|
||||
self._compiled_func: Optional[Callable] = None
|
||||
self._code_hash: Optional[str] = None
|
||||
|
||||
def compile(self) -> None:
|
||||
"""
|
||||
Compile the extractor code and extract the function.
|
||||
|
||||
Raises:
|
||||
ExtractorValidationError: If code is invalid
|
||||
ExtractorSecurityError: If code contains dangerous patterns
|
||||
"""
|
||||
# Validate code
|
||||
is_valid, errors = validate_extractor_code(self.code, self.function_name)
|
||||
if not is_valid:
|
||||
raise ExtractorValidationError(f"Validation failed: {'; '.join(errors)}")
|
||||
|
||||
# Compute code hash for caching
|
||||
self._code_hash = hashlib.sha256(self.code.encode()).hexdigest()[:12]
|
||||
|
||||
# Create execution namespace with allowed imports
|
||||
namespace = {
|
||||
'np': np,
|
||||
'numpy': np,
|
||||
'math': __import__('math'),
|
||||
'statistics': __import__('statistics'),
|
||||
'collections': __import__('collections'),
|
||||
'itertools': __import__('itertools'),
|
||||
'functools': __import__('functools'),
|
||||
}
|
||||
|
||||
# Add scipy if available
|
||||
try:
|
||||
import scipy
|
||||
namespace['scipy'] = scipy
|
||||
from scipy import interpolate, optimize, integrate, linalg
|
||||
namespace['interpolate'] = interpolate
|
||||
namespace['optimize'] = optimize
|
||||
namespace['integrate'] = integrate
|
||||
namespace['linalg'] = linalg
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Add pyNastran if available
|
||||
try:
|
||||
from pyNastran.op2.op2 import OP2
|
||||
from pyNastran.bdf.bdf import BDF
|
||||
namespace['OP2'] = OP2
|
||||
namespace['BDF'] = BDF
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Add Atomizer extractors
|
||||
try:
|
||||
from optimization_engine import extractors
|
||||
namespace['extractors'] = extractors
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Execute the code to define the function
|
||||
try:
|
||||
exec(self.code, namespace)
|
||||
except Exception as e:
|
||||
raise ExtractorValidationError(f"Failed to compile: {e}")
|
||||
|
||||
# Extract the function
|
||||
if self.function_name not in namespace:
|
||||
raise ExtractorValidationError(f"Function '{self.function_name}' not defined")
|
||||
|
||||
self._compiled_func = namespace[self.function_name]
|
||||
logger.info(f"Compiled custom extractor: {self.name} ({self._code_hash})")
|
||||
|
||||
def execute(self, context: CustomExtractorContext) -> Dict[str, float]:
|
||||
"""
|
||||
Execute the extractor and return results.
|
||||
|
||||
Args:
|
||||
context: Execution context with FEA results
|
||||
|
||||
Returns:
|
||||
Dictionary of output_name -> value
|
||||
|
||||
Raises:
|
||||
RuntimeError: If execution fails
|
||||
"""
|
||||
if self._compiled_func is None:
|
||||
self.compile()
|
||||
|
||||
try:
|
||||
# Call the function with appropriate arguments
|
||||
result = self._compiled_func(
|
||||
op2_path=str(context.op2_path) if context.op2_path else None,
|
||||
bdf_path=str(context.bdf_path) if context.bdf_path else None,
|
||||
params=context.params,
|
||||
working_dir=str(context.working_dir) if context.working_dir else None,
|
||||
)
|
||||
|
||||
# Normalize result to dict
|
||||
if isinstance(result, dict):
|
||||
return result
|
||||
elif isinstance(result, (int, float)):
|
||||
# Single value - use first output name
|
||||
if self.outputs:
|
||||
return {self.outputs[0]['name']: float(result)}
|
||||
return {'value': float(result)}
|
||||
elif isinstance(result, (list, tuple)):
|
||||
# Multiple values - map to output names
|
||||
output_dict = {}
|
||||
for i, val in enumerate(result):
|
||||
if i < len(self.outputs):
|
||||
output_dict[self.outputs[i]['name']] = float(val)
|
||||
else:
|
||||
output_dict[f'output_{i}'] = float(val)
|
||||
return output_dict
|
||||
else:
|
||||
raise RuntimeError(f"Unexpected result type: {type(result)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Custom extractor {self.name} failed: {e}")
|
||||
logger.debug(traceback.format_exc())
|
||||
raise RuntimeError(f"Extractor {self.name} failed: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Extractor Loader
|
||||
# ============================================================================
|
||||
|
||||
class CustomExtractorLoader:
|
||||
"""
|
||||
Loads and manages custom extractors from AtomizerSpec.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize loader with empty cache."""
|
||||
self._cache: Dict[str, CustomExtractor] = {}
|
||||
|
||||
def load_from_spec(self, spec: Dict[str, Any]) -> Dict[str, CustomExtractor]:
|
||||
"""
|
||||
Load all custom extractors from an AtomizerSpec.
|
||||
|
||||
Args:
|
||||
spec: AtomizerSpec dictionary
|
||||
|
||||
Returns:
|
||||
Dictionary of extractor_id -> CustomExtractor
|
||||
"""
|
||||
extractors = {}
|
||||
|
||||
for ext_def in spec.get('extractors', []):
|
||||
# Skip builtin extractors
|
||||
if ext_def.get('builtin', True):
|
||||
continue
|
||||
|
||||
# Custom extractor must have function definition
|
||||
func_def = ext_def.get('function', {})
|
||||
if not func_def.get('source'):
|
||||
logger.warning(f"Custom extractor {ext_def.get('id')} has no source code")
|
||||
continue
|
||||
|
||||
extractor = CustomExtractor(
|
||||
extractor_id=ext_def.get('id', 'custom'),
|
||||
name=ext_def.get('name', 'Custom Extractor'),
|
||||
function_name=func_def.get('name', 'extract'),
|
||||
code=func_def.get('source', ''),
|
||||
outputs=ext_def.get('outputs', []),
|
||||
dependencies=func_def.get('dependencies', []),
|
||||
)
|
||||
|
||||
try:
|
||||
extractor.compile()
|
||||
extractors[extractor.extractor_id] = extractor
|
||||
self._cache[extractor.extractor_id] = extractor
|
||||
except (ExtractorValidationError, ExtractorSecurityError) as e:
|
||||
logger.error(f"Failed to load extractor {extractor.name}: {e}")
|
||||
|
||||
return extractors
|
||||
|
||||
def get(self, extractor_id: str) -> Optional[CustomExtractor]:
|
||||
"""Get a cached extractor by ID."""
|
||||
return self._cache.get(extractor_id)
|
||||
|
||||
def execute_all(self, extractors: Dict[str, CustomExtractor],
|
||||
context: CustomExtractorContext) -> Dict[str, Dict[str, float]]:
|
||||
"""
|
||||
Execute all custom extractors and collect results.
|
||||
|
||||
Args:
|
||||
extractors: Dictionary of extractor_id -> CustomExtractor
|
||||
context: Execution context
|
||||
|
||||
Returns:
|
||||
Dictionary of extractor_id -> {output_name: value}
|
||||
"""
|
||||
results = {}
|
||||
|
||||
for ext_id, extractor in extractors.items():
|
||||
try:
|
||||
results[ext_id] = extractor.execute(context)
|
||||
except Exception as e:
|
||||
logger.error(f"Extractor {ext_id} failed: {e}")
|
||||
# Return NaN for failed extractors
|
||||
results[ext_id] = {
|
||||
out['name']: float('nan')
|
||||
for out in extractor.outputs
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the extractor cache."""
|
||||
self._cache.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Convenience Functions
|
||||
# ============================================================================
|
||||
|
||||
# Global loader instance
|
||||
_loader = CustomExtractorLoader()
|
||||
|
||||
|
||||
def load_custom_extractors(spec: Dict[str, Any]) -> Dict[str, CustomExtractor]:
|
||||
"""
|
||||
Load custom extractors from an AtomizerSpec.
|
||||
|
||||
Args:
|
||||
spec: AtomizerSpec dictionary
|
||||
|
||||
Returns:
|
||||
Dictionary of extractor_id -> CustomExtractor
|
||||
"""
|
||||
return _loader.load_from_spec(spec)
|
||||
|
||||
|
||||
def execute_custom_extractor(extractor_id: str,
|
||||
op2_path: Union[str, Path],
|
||||
bdf_path: Optional[Union[str, Path]] = None,
|
||||
working_dir: Optional[Union[str, Path]] = None,
|
||||
params: Optional[Dict[str, float]] = None) -> Dict[str, float]:
|
||||
"""
|
||||
Execute a single cached custom extractor.
|
||||
|
||||
Args:
|
||||
extractor_id: ID of the extractor to run
|
||||
op2_path: Path to OP2 results file
|
||||
bdf_path: Optional path to BDF file
|
||||
working_dir: Optional working directory
|
||||
params: Optional design parameters
|
||||
|
||||
Returns:
|
||||
Dictionary of output_name -> value
|
||||
|
||||
Raises:
|
||||
KeyError: If extractor not found in cache
|
||||
"""
|
||||
extractor = _loader.get(extractor_id)
|
||||
if extractor is None:
|
||||
raise KeyError(f"Extractor '{extractor_id}' not found in cache")
|
||||
|
||||
context = CustomExtractorContext(
|
||||
op2_path=op2_path,
|
||||
bdf_path=bdf_path,
|
||||
working_dir=working_dir,
|
||||
params=params
|
||||
)
|
||||
|
||||
return extractor.execute(context)
|
||||
|
||||
|
||||
def validate_custom_extractor(code: str, function_name: str = "extract") -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate custom extractor code without executing it.
|
||||
|
||||
Args:
|
||||
code: Python source code
|
||||
function_name: Expected function name
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, list of error/warning messages)
|
||||
"""
|
||||
return validate_extractor_code(code, function_name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'CustomExtractor',
|
||||
'CustomExtractorLoader',
|
||||
'CustomExtractorContext',
|
||||
'ExtractorSecurityError',
|
||||
'ExtractorValidationError',
|
||||
'load_custom_extractors',
|
||||
'execute_custom_extractor',
|
||||
'validate_custom_extractor',
|
||||
]
|
||||
328
optimization_engine/extractors/spec_extractor_builder.py
Normal file
328
optimization_engine/extractors/spec_extractor_builder.py
Normal file
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
Spec Extractor Builder
|
||||
|
||||
Builds result extractors from AtomizerSpec v2.0 configuration.
|
||||
Combines builtin extractors with custom Python extractors.
|
||||
|
||||
P3.10: Integration with optimization runner
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from optimization_engine.extractors.custom_extractor_loader import (
|
||||
CustomExtractor,
|
||||
CustomExtractorContext,
|
||||
CustomExtractorLoader,
|
||||
load_custom_extractors,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Builtin Extractor Registry
|
||||
# ============================================================================
|
||||
|
||||
# Map of builtin extractor types to their extraction functions
|
||||
BUILTIN_EXTRACTORS = {}
|
||||
|
||||
|
||||
def _register_builtin_extractors():
|
||||
"""Lazily register builtin extractors to avoid circular imports."""
|
||||
global BUILTIN_EXTRACTORS
|
||||
if BUILTIN_EXTRACTORS:
|
||||
return
|
||||
|
||||
try:
|
||||
# Zernike OPD (recommended for mirrors)
|
||||
from optimization_engine.extractors.extract_zernike_figure import (
|
||||
ZernikeOPDExtractor,
|
||||
)
|
||||
BUILTIN_EXTRACTORS['zernike_opd'] = ZernikeOPDExtractor
|
||||
|
||||
# Mass extractors
|
||||
from optimization_engine.extractors.bdf_mass_extractor import extract_mass_from_bdf
|
||||
BUILTIN_EXTRACTORS['mass'] = extract_mass_from_bdf
|
||||
|
||||
from optimization_engine.extractors.extract_mass_from_expression import (
|
||||
extract_mass_from_expression,
|
||||
)
|
||||
BUILTIN_EXTRACTORS['mass_expression'] = extract_mass_from_expression
|
||||
|
||||
# Displacement
|
||||
from optimization_engine.extractors.extract_displacement import extract_displacement
|
||||
BUILTIN_EXTRACTORS['displacement'] = extract_displacement
|
||||
|
||||
# Stress
|
||||
from optimization_engine.extractors.extract_von_mises_stress import extract_solid_stress
|
||||
BUILTIN_EXTRACTORS['stress'] = extract_solid_stress
|
||||
|
||||
from optimization_engine.extractors.extract_principal_stress import (
|
||||
extract_principal_stress,
|
||||
)
|
||||
BUILTIN_EXTRACTORS['principal_stress'] = extract_principal_stress
|
||||
|
||||
# Frequency
|
||||
from optimization_engine.extractors.extract_frequency import extract_frequency
|
||||
BUILTIN_EXTRACTORS['frequency'] = extract_frequency
|
||||
|
||||
# Temperature
|
||||
from optimization_engine.extractors.extract_temperature import extract_temperature
|
||||
BUILTIN_EXTRACTORS['temperature'] = extract_temperature
|
||||
|
||||
# Strain energy
|
||||
from optimization_engine.extractors.extract_strain_energy import (
|
||||
extract_strain_energy,
|
||||
extract_total_strain_energy,
|
||||
)
|
||||
BUILTIN_EXTRACTORS['strain_energy'] = extract_strain_energy
|
||||
BUILTIN_EXTRACTORS['total_strain_energy'] = extract_total_strain_energy
|
||||
|
||||
# SPC forces
|
||||
from optimization_engine.extractors.extract_spc_forces import (
|
||||
extract_spc_forces,
|
||||
extract_total_reaction_force,
|
||||
)
|
||||
BUILTIN_EXTRACTORS['spc_forces'] = extract_spc_forces
|
||||
BUILTIN_EXTRACTORS['reaction_force'] = extract_total_reaction_force
|
||||
|
||||
logger.debug(f"Registered {len(BUILTIN_EXTRACTORS)} builtin extractors")
|
||||
|
||||
except ImportError as e:
|
||||
logger.warning(f"Some builtin extractors unavailable: {e}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Spec Extractor Builder
|
||||
# ============================================================================
|
||||
|
||||
class SpecExtractorBuilder:
|
||||
"""
|
||||
Builds extraction functions from AtomizerSpec extractor definitions.
|
||||
"""
|
||||
|
||||
def __init__(self, spec: Dict[str, Any]):
|
||||
"""
|
||||
Initialize builder with an AtomizerSpec.
|
||||
|
||||
Args:
|
||||
spec: AtomizerSpec dictionary
|
||||
"""
|
||||
self.spec = spec
|
||||
self.custom_loader = CustomExtractorLoader()
|
||||
self._extractors: Dict[str, Callable] = {}
|
||||
self._custom_extractors: Dict[str, CustomExtractor] = {}
|
||||
|
||||
# Register builtin extractors
|
||||
_register_builtin_extractors()
|
||||
|
||||
def build(self) -> Dict[str, Callable]:
|
||||
"""
|
||||
Build all extractors from the spec.
|
||||
|
||||
Returns:
|
||||
Dictionary of extractor_id -> extraction_function
|
||||
"""
|
||||
for ext_def in self.spec.get('extractors', []):
|
||||
ext_id = ext_def.get('id', 'unknown')
|
||||
|
||||
if ext_def.get('builtin', True):
|
||||
# Builtin extractor
|
||||
extractor_func = self._build_builtin_extractor(ext_def)
|
||||
else:
|
||||
# Custom extractor
|
||||
extractor_func = self._build_custom_extractor(ext_def)
|
||||
|
||||
if extractor_func:
|
||||
self._extractors[ext_id] = extractor_func
|
||||
else:
|
||||
logger.warning(f"Failed to build extractor: {ext_id}")
|
||||
|
||||
return self._extractors
|
||||
|
||||
def _build_builtin_extractor(self, ext_def: Dict[str, Any]) -> Optional[Callable]:
|
||||
"""
|
||||
Build a builtin extractor function.
|
||||
|
||||
Args:
|
||||
ext_def: Extractor definition from spec
|
||||
|
||||
Returns:
|
||||
Callable extraction function or None
|
||||
"""
|
||||
ext_type = ext_def.get('type', '')
|
||||
ext_id = ext_def.get('id', '')
|
||||
config = ext_def.get('config', {})
|
||||
outputs = ext_def.get('outputs', [])
|
||||
|
||||
# Get base extractor
|
||||
base_extractor = BUILTIN_EXTRACTORS.get(ext_type)
|
||||
if base_extractor is None:
|
||||
logger.warning(f"Unknown builtin extractor type: {ext_type}")
|
||||
return None
|
||||
|
||||
# Create configured wrapper
|
||||
def create_extractor_wrapper(base, cfg, outs):
|
||||
"""Create a wrapper that applies config and extracts specified outputs."""
|
||||
def wrapper(op2_path: str, **kwargs) -> Dict[str, float]:
|
||||
"""Execute extractor and return outputs dict."""
|
||||
try:
|
||||
# Handle class-based extractors (like ZernikeOPDExtractor)
|
||||
if isinstance(base, type):
|
||||
# Instantiate with config
|
||||
instance = base(
|
||||
inner_radius=cfg.get('inner_radius_mm', 0),
|
||||
n_modes=cfg.get('n_modes', 21),
|
||||
**{k: v for k, v in cfg.items()
|
||||
if k not in ['inner_radius_mm', 'n_modes']}
|
||||
)
|
||||
raw_result = instance.extract(op2_path, **kwargs)
|
||||
else:
|
||||
# Function-based extractor
|
||||
raw_result = base(op2_path, **cfg, **kwargs)
|
||||
|
||||
# Map to output names
|
||||
result = {}
|
||||
if isinstance(raw_result, dict):
|
||||
# Use output definitions to select values
|
||||
for out_def in outs:
|
||||
out_name = out_def.get('name', '')
|
||||
source = out_def.get('source', out_name)
|
||||
if source in raw_result:
|
||||
result[out_name] = float(raw_result[source])
|
||||
elif out_name in raw_result:
|
||||
result[out_name] = float(raw_result[out_name])
|
||||
|
||||
# If no outputs defined, return all
|
||||
if not outs:
|
||||
result = {k: float(v) for k, v in raw_result.items()
|
||||
if isinstance(v, (int, float))}
|
||||
elif isinstance(raw_result, (int, float)):
|
||||
# Single value - use first output name or 'value'
|
||||
out_name = outs[0]['name'] if outs else 'value'
|
||||
result[out_name] = float(raw_result)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Extractor failed: {e}")
|
||||
return {out['name']: float('nan') for out in outs}
|
||||
|
||||
return wrapper
|
||||
|
||||
return create_extractor_wrapper(base_extractor, config, outputs)
|
||||
|
||||
def _build_custom_extractor(self, ext_def: Dict[str, Any]) -> Optional[Callable]:
|
||||
"""
|
||||
Build a custom Python extractor function.
|
||||
|
||||
Args:
|
||||
ext_def: Extractor definition with function source
|
||||
|
||||
Returns:
|
||||
Callable extraction function or None
|
||||
"""
|
||||
ext_id = ext_def.get('id', 'custom')
|
||||
func_def = ext_def.get('function', {})
|
||||
|
||||
if not func_def.get('source'):
|
||||
logger.error(f"Custom extractor {ext_id} has no source code")
|
||||
return None
|
||||
|
||||
try:
|
||||
custom_ext = CustomExtractor(
|
||||
extractor_id=ext_id,
|
||||
name=ext_def.get('name', 'Custom'),
|
||||
function_name=func_def.get('name', 'extract'),
|
||||
code=func_def.get('source', ''),
|
||||
outputs=ext_def.get('outputs', []),
|
||||
dependencies=func_def.get('dependencies', []),
|
||||
)
|
||||
custom_ext.compile()
|
||||
self._custom_extractors[ext_id] = custom_ext
|
||||
|
||||
# Create wrapper function
|
||||
def create_custom_wrapper(extractor):
|
||||
def wrapper(op2_path: str, bdf_path: str = None,
|
||||
params: Dict[str, float] = None,
|
||||
working_dir: str = None, **kwargs) -> Dict[str, float]:
|
||||
context = CustomExtractorContext(
|
||||
op2_path=op2_path,
|
||||
bdf_path=bdf_path,
|
||||
working_dir=working_dir,
|
||||
params=params or {}
|
||||
)
|
||||
return extractor.execute(context)
|
||||
return wrapper
|
||||
|
||||
return create_custom_wrapper(custom_ext)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to build custom extractor {ext_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Convenience Functions
|
||||
# ============================================================================
|
||||
|
||||
def build_extractors_from_spec(spec: Union[Dict[str, Any], Path, str]) -> Dict[str, Callable]:
|
||||
"""
|
||||
Build extraction functions from an AtomizerSpec.
|
||||
|
||||
Args:
|
||||
spec: AtomizerSpec dict, or path to spec JSON file
|
||||
|
||||
Returns:
|
||||
Dictionary of extractor_id -> extraction_function
|
||||
|
||||
Example:
|
||||
extractors = build_extractors_from_spec("atomizer_spec.json")
|
||||
results = extractors['E1']("model.op2")
|
||||
"""
|
||||
if isinstance(spec, (str, Path)):
|
||||
with open(spec) as f:
|
||||
spec = json.load(f)
|
||||
|
||||
builder = SpecExtractorBuilder(spec)
|
||||
return builder.build()
|
||||
|
||||
|
||||
def get_extractor_outputs(spec: Dict[str, Any], extractor_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get output definitions for an extractor.
|
||||
|
||||
Args:
|
||||
spec: AtomizerSpec dictionary
|
||||
extractor_id: ID of the extractor
|
||||
|
||||
Returns:
|
||||
List of output definitions [{name, units, description}, ...]
|
||||
"""
|
||||
for ext in spec.get('extractors', []):
|
||||
if ext.get('id') == extractor_id:
|
||||
return ext.get('outputs', [])
|
||||
return []
|
||||
|
||||
|
||||
def list_available_builtin_extractors() -> List[str]:
|
||||
"""
|
||||
List all available builtin extractor types.
|
||||
|
||||
Returns:
|
||||
List of extractor type names
|
||||
"""
|
||||
_register_builtin_extractors()
|
||||
return list(BUILTIN_EXTRACTORS.keys())
|
||||
|
||||
|
||||
__all__ = [
|
||||
'SpecExtractorBuilder',
|
||||
'build_extractors_from_spec',
|
||||
'get_extractor_outputs',
|
||||
'list_available_builtin_extractors',
|
||||
'BUILTIN_EXTRACTORS',
|
||||
]
|
||||
Reference in New Issue
Block a user