feat: Improve dashboard performance and Claude terminal context

- Add trial limiting (300 max) and reduce polling to 15s for large studies
- Make dashboard layout wider with col-span adjustments
- Claude terminal now runs from Atomizer root for CLAUDE.md/skills access
- Add study context display in terminal on connect
- Add KaTeX math rendering styles for study reports
- Add surrogate tuner module for hyperparameter optimization
- Fix backend proxy to port 8001

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

atomizer-dashboard/frontend/src/components/ClaudeTerminal.tsx

@@ -149,21 +149,25 @@ export const ClaudeTerminal: React.FC<ClaudeTerminalProps> = ({
     setIsConnecting(true);
     setError(null);
 
-    // Determine working directory - use study path if available
-    let workingDir = '';
-    if (selectedStudy?.id) {
-      // The study directory path
-      workingDir = `?working_dir=C:/Users/Antoine/Atomizer`;
-    }
+    // Always use Atomizer root as working directory so Claude has access to:
+    // - CLAUDE.md (system instructions)
+    // - .claude/skills/ (skill definitions)
+    // Pass study_id as parameter so we can inform Claude about the context
+    const workingDir = 'C:/Users/Antoine/Atomizer';
+    const studyParam = selectedStudy?.id ? `&study_id=${selectedStudy.id}` : '';
 
     const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
-    const ws = new WebSocket(`${protocol}//${window.location.host}/api/terminal/claude${workingDir}`);
+    const ws = new WebSocket(`${protocol}//${window.location.host}/api/terminal/claude?working_dir=${workingDir}${studyParam}`);
 
     ws.onopen = () => {
       setIsConnected(true);
       setIsConnecting(false);
       xtermRef.current?.clear();
       xtermRef.current?.writeln('\x1b[1;32mConnected to Claude Code\x1b[0m');
+      if (selectedStudy?.id) {
+        xtermRef.current?.writeln(`\x1b[90mStudy context: \x1b[1;33m${selectedStudy.id}\x1b[0m`);
+        xtermRef.current?.writeln('\x1b[90mTip: Tell Claude about your study, e.g. "Help me with study ' + selectedStudy.id + '"\x1b[0m');
+      }
       xtermRef.current?.writeln('');
 
       // Send initial resize

atomizer-dashboard/frontend/src/components/StudyReportViewer.tsx

@@ -65,7 +65,7 @@ export function StudyReportViewer({ studyId, studyPath }: StudyReportViewerProps
 
   return (
     <div className="fixed inset-0 z-50 flex items-center justify-center bg-black/70">
-      <div className="bg-dark-800 rounded-xl shadow-2xl w-[90vw] max-w-5xl h-[85vh] flex flex-col border border-dark-600">
+      <div className="bg-dark-800 rounded-xl shadow-2xl w-[95vw] max-w-7xl h-[90vh] flex flex-col border border-dark-600">
         {/* Header */}
         <div className="flex items-center justify-between px-6 py-4 border-b border-dark-600">
           <div className="flex items-center gap-3">
@@ -127,8 +127,8 @@ export function StudyReportViewer({ studyId, studyPath }: StudyReportViewerProps
           {markdown && !loading && (
             <article className="markdown-body">
               <ReactMarkdown
-                remarkPlugins={[remarkGfm, remarkMath]}
-                rehypePlugins={[rehypeKatex]}
+                remarkPlugins={[remarkGfm, [remarkMath, { singleDollarTextMath: false }]]}
+                rehypePlugins={[[rehypeKatex, { strict: false, trust: true, output: 'html' }]]}
                 components={{
                   // Custom heading styles
                   h1: ({children}) => (

atomizer-dashboard/frontend/src/index.css

@@ -69,3 +69,39 @@
 ::-webkit-scrollbar-thumb:hover {
   @apply bg-dark-400;
 }
+
+/* KaTeX Math Rendering */
+.katex {
+  font-size: 1.1em !important;
+  color: inherit !important;
+}
+
+.katex-display {
+  margin: 1em 0 !important;
+  overflow-x: auto;
+  overflow-y: hidden;
+}
+
+.katex-display > .katex {
+  color: #e2e8f0 !important;
+}
+
+/* Markdown body styles */
+.markdown-body {
+  color: #e2e8f0;
+  line-height: 1.7;
+}
+
+.markdown-body .katex-display {
+  background: rgba(30, 41, 59, 0.5);
+  padding: 1rem;
+  border-radius: 0.5rem;
+  border: 1px solid #334155;
+}
+
+/* Code blocks in markdown should have proper width */
+.markdown-body pre {
+  max-width: 100%;
+  overflow-x: auto;
+  white-space: pre;
+}

atomizer-dashboard/frontend/src/pages/Dashboard.tsx

@@ -54,6 +54,8 @@ export default function Dashboard() {
   const [alertIdCounter, setAlertIdCounter] = useState(0);
   const [expandedTrials, setExpandedTrials] = useState<Set<number>>(new Set());
   const [sortBy, setSortBy] = useState<'performance' | 'chronological'>('performance');
+  const [trialsPage, setTrialsPage] = useState(0);
+  const trialsPerPage = 50; // Limit trials per page for performance
 
   // Parameter Space axis selection
   const [paramXIndex, setParamXIndex] = useState(0);
@@ -99,6 +101,9 @@ export default function Dashboard() {
   });
 
   // Load initial trial history when study changes
+  // PERFORMANCE: Use limit to avoid loading thousands of trials at once
+  const MAX_TRIALS_LOAD = 300;
+
   useEffect(() => {
     if (selectedStudyId) {
       setAllTrials([]);
@@ -106,74 +111,63 @@ export default function Dashboard() {
       setPrunedCount(0);
       setExpandedTrials(new Set());
 
-      apiClient.getStudyHistory(selectedStudyId)
+      // Single history fetch with limit - used for both trial list and charts
+      // This replaces the duplicate fetch calls
+      fetch(`/api/optimization/studies/${selectedStudyId}/history?limit=${MAX_TRIALS_LOAD}`)
+        .then(res => res.json())
         .then(data => {
-          const validTrials = data.trials.filter(t => t.objective !== null && t.objective !== undefined);
+          // Set trials for the trial list
+          const validTrials = data.trials.filter((t: any) => t.objective !== null && t.objective !== undefined);
           setAllTrials(validTrials);
           if (validTrials.length > 0) {
-            const minObj = Math.min(...validTrials.map(t => t.objective));
+            const minObj = Math.min(...validTrials.map((t: any) => t.objective));
             setBestValue(minObj);
           }
-        })
-        .catch(console.error);
 
-      apiClient.getStudyPruning(selectedStudyId)
-        .then(data => {
-          // Use count if available (new API), fallback to array length (legacy)
-          setPrunedCount(data.count ?? data.pruned_trials?.length ?? 0);
-        })
-        .catch(console.error);
-
-      // Protocol 13: Fetch metadata
-      fetch(`/api/optimization/studies/${selectedStudyId}/metadata`)
-        .then(res => res.json())
-        .then(data => {
-          setStudyMetadata(data);
-        })
-        .catch(err => console.error('Failed to load metadata:', err));
-
-      // Protocol 13: Fetch Pareto front (raw format for Protocol 13 components)
-      fetch(`/api/optimization/studies/${selectedStudyId}/pareto-front`)
-        .then(res => res.json())
-        .then(paretoData => {
-          console.log('[Dashboard] Pareto front data:', paretoData);
-          if (paretoData.is_multi_objective && paretoData.pareto_front) {
-            console.log('[Dashboard] Setting Pareto front with', paretoData.pareto_front.length, 'trials');
-            setParetoFront(paretoData.pareto_front);
-          } else {
-            console.log('[Dashboard] No Pareto front or not multi-objective');
-            setParetoFront([]);
-          }
-        })
-        .catch(err => console.error('Failed to load Pareto front:', err));
-
-      // Fetch ALL trials (not just Pareto) for parallel coordinates and charts
-      fetch(`/api/optimization/studies/${selectedStudyId}/history`)
-        .then(res => res.json())
-        .then(data => {
-          // Transform to match the format expected by charts
-          // API returns 'objectives' (array) for multi-objective, 'objective' (number) for single
+          // Transform for charts (parallel coordinates, etc.)
           const trialsData = data.trials.map((t: any) => {
-            // Build values array: use objectives if available, otherwise wrap single objective
             let values: number[] = [];
             if (t.objectives && Array.isArray(t.objectives)) {
               values = t.objectives;
             } else if (t.objective !== null && t.objective !== undefined) {
               values = [t.objective];
             }
-
             return {
               trial_number: t.trial_number,
               values,
               params: t.design_variables || {},
               user_attrs: t.user_attrs || {},
               constraint_satisfied: t.constraint_satisfied !== false,
-              source: t.source || t.user_attrs?.source || 'FEA'  // FEA vs NN differentiation
+              source: t.source || t.user_attrs?.source || 'FEA'
             };
           });
           setAllTrialsRaw(trialsData);
         })
-        .catch(err => console.error('Failed to load all trials:', err));
+        .catch(console.error);
+
+      apiClient.getStudyPruning(selectedStudyId)
+        .then(data => {
+          setPrunedCount(data.count ?? data.pruned_trials?.length ?? 0);
+        })
+        .catch(console.error);
+
+      // Fetch metadata (small payload)
+      fetch(`/api/optimization/studies/${selectedStudyId}/metadata`)
+        .then(res => res.json())
+        .then(data => setStudyMetadata(data))
+        .catch(err => console.error('Failed to load metadata:', err));
+
+      // Fetch Pareto front (usually small)
+      fetch(`/api/optimization/studies/${selectedStudyId}/pareto-front`)
+        .then(res => res.json())
+        .then(paretoData => {
+          if (paretoData.is_multi_objective && paretoData.pareto_front) {
+            setParetoFront(paretoData.pareto_front);
+          } else {
+            setParetoFront([]);
+          }
+        })
+        .catch(err => console.error('Failed to load Pareto front:', err));
     }
   }, [selectedStudyId]);
 
@@ -194,41 +188,77 @@ export default function Dashboard() {
     setDisplayedTrials(sorted);
   }, [allTrials, sortBy]);
 
-  // Auto-refresh polling (every 3 seconds) for trial history
+  // Auto-refresh polling for trial history
+  // PERFORMANCE: Use limit and longer interval for large studies
   useEffect(() => {
     if (!selectedStudyId) return;
 
     const refreshInterval = setInterval(() => {
-      apiClient.getStudyHistory(selectedStudyId)
+      // Only fetch latest trials, not the entire history
+      fetch(`/api/optimization/studies/${selectedStudyId}/history?limit=${MAX_TRIALS_LOAD}`)
+        .then(res => res.json())
         .then(data => {
-          const validTrials = data.trials.filter(t => t.objective !== null && t.objective !== undefined);
+          const validTrials = data.trials.filter((t: any) => t.objective !== null && t.objective !== undefined);
           setAllTrials(validTrials);
           if (validTrials.length > 0) {
-            const minObj = Math.min(...validTrials.map(t => t.objective));
+            const minObj = Math.min(...validTrials.map((t: any) => t.objective));
             setBestValue(minObj);
           }
+          // Also update chart data
+          const trialsData = data.trials.map((t: any) => {
+            let values: number[] = [];
+            if (t.objectives && Array.isArray(t.objectives)) {
+              values = t.objectives;
+            } else if (t.objective !== null && t.objective !== undefined) {
+              values = [t.objective];
+            }
+            return {
+              trial_number: t.trial_number,
+              values,
+              params: t.design_variables || {},
+              user_attrs: t.user_attrs || {},
+              constraint_satisfied: t.constraint_satisfied !== false,
+              source: t.source || t.user_attrs?.source || 'FEA'
+            };
+          });
+          setAllTrialsRaw(trialsData);
         })
         .catch(err => console.error('Auto-refresh failed:', err));
-    }, 3000); // Poll every 3 seconds
+    }, 15000); // Poll every 15 seconds for performance
 
     return () => clearInterval(refreshInterval);
   }, [selectedStudyId]);
 
-  // Prepare chart data with proper null/undefined handling
-  const convergenceData: ConvergenceDataPoint[] = allTrials
-    .filter(t => t.objective !== null && t.objective !== undefined)
-    .sort((a, b) => a.trial_number - b.trial_number)
-    .map((trial, idx, arr) => {
-      const previousTrials = arr.slice(0, idx + 1);
-      const validObjectives = previousTrials.map(t => t.objective).filter(o => o !== null && o !== undefined);
-      return {
-        trial_number: trial.trial_number,
-        objective: trial.objective,
-        best_so_far: validObjectives.length > 0 ? Math.min(...validObjectives) : trial.objective,
-      };
-    });
+  // Sample data for charts when there are too many trials (performance optimization)
+  const MAX_CHART_POINTS = 200; // Reduced for better performance
+  const sampleData = <T,>(data: T[], maxPoints: number): T[] => {
+    if (data.length <= maxPoints) return data;
+    const step = Math.ceil(data.length / maxPoints);
+    return data.filter((_, i) => i % step === 0 || i === data.length - 1);
+  };
 
-  const parameterSpaceData: ParameterSpaceDataPoint[] = allTrials
+  // Prepare chart data with proper null/undefined handling
+  const allValidTrials = allTrials
+    .filter(t => t.objective !== null && t.objective !== undefined)
+    .sort((a, b) => a.trial_number - b.trial_number);
+
+  // Calculate best_so_far for each trial
+  let runningBest = Infinity;
+  const convergenceDataFull: ConvergenceDataPoint[] = allValidTrials.map(trial => {
+    if (trial.objective < runningBest) {
+      runningBest = trial.objective;
+    }
+    return {
+      trial_number: trial.trial_number,
+      objective: trial.objective,
+      best_so_far: runningBest,
+    };
+  });
+
+  // Sample for chart rendering performance
+  const convergenceData = sampleData(convergenceDataFull, MAX_CHART_POINTS);
+
+  const parameterSpaceDataFull: ParameterSpaceDataPoint[] = allTrials
     .filter(t => t.objective !== null && t.objective !== undefined && t.design_variables)
     .map(trial => {
       const params = Object.values(trial.design_variables);
@@ -241,6 +271,9 @@ export default function Dashboard() {
       };
     });
 
+  // Sample for chart rendering performance
+  const parameterSpaceData = sampleData(parameterSpaceDataFull, MAX_CHART_POINTS);
+
   // Calculate average objective
   const validObjectives = allTrials.filter(t => t.objective !== null && t.objective !== undefined).map(t => t.objective);
   const avgObjective = validObjectives.length > 0
@@ -384,14 +417,14 @@ export default function Dashboard() {
         </div>
       </header>
 
-      <div className="grid grid-cols-12 gap-6">
-        {/* Control Panel - Left Sidebar */}
-        <aside className="col-span-3">
+      <div className="grid grid-cols-12 gap-4">
+        {/* Control Panel - Left Sidebar (smaller) */}
+        <aside className="col-span-2">
           <ControlPanel onStatusChange={refreshStudies} />
         </aside>
 
-        {/* Main Content - shrinks when chat is open */}
-        <main className={chatOpen ? 'col-span-5' : 'col-span-9'}>
+        {/* Main Content - takes most of the space */}
+        <main className={chatOpen ? 'col-span-6' : 'col-span-10'}>
           {/* Study Name Header */}
           {selectedStudyId && (
             <div className="mb-4 pb-3 border-b border-dark-600">
@@ -694,12 +727,12 @@ export default function Dashboard() {
             </ExpandableChart>
           </div>
 
-          {/* Trial History with Sort Controls */}
+          {/* Trial History with Sort Controls and Pagination */}
           <Card
             title={
               <div className="flex items-center justify-between w-full">
                 <span>Trial History ({displayedTrials.length} trials)</span>
-                <div className="flex gap-2">
+                <div className="flex gap-2 items-center">
                   <button
                     onClick={() => setSortBy('performance')}
                     className={`px-3 py-1 rounded text-sm ${
@@ -720,13 +753,35 @@ export default function Dashboard() {
                   >
                     Newest First
                   </button>
+                  {/* Pagination controls */}
+                  {displayedTrials.length > trialsPerPage && (
+                    <div className="flex items-center gap-1 ml-2 border-l border-dark-500 pl-2">
+                      <button
+                        onClick={() => setTrialsPage(Math.max(0, trialsPage - 1))}
+                        disabled={trialsPage === 0}
+                        className="px-2 py-1 text-sm bg-dark-500 rounded disabled:opacity-50 hover:bg-dark-400"
+                      >
+                        ‹
+                      </button>
+                      <span className="text-xs text-dark-300 px-2">
+                        {trialsPage + 1}/{Math.ceil(displayedTrials.length / trialsPerPage)}
+                      </span>
+                      <button
+                        onClick={() => setTrialsPage(Math.min(Math.ceil(displayedTrials.length / trialsPerPage) - 1, trialsPage + 1))}
+                        disabled={trialsPage >= Math.ceil(displayedTrials.length / trialsPerPage) - 1}
+                        className="px-2 py-1 text-sm bg-dark-500 rounded disabled:opacity-50 hover:bg-dark-400"
+                      >
+                        ›
+                      </button>
+                    </div>
+                  )}
                 </div>
               </div>
             }
           >
             <div className="space-y-2 max-h-[600px] overflow-y-auto">
               {displayedTrials.length > 0 ? (
-                displayedTrials.map(trial => {
+                displayedTrials.slice(trialsPage * trialsPerPage, (trialsPage + 1) * trialsPerPage).map(trial => {
                   const isExpanded = expandedTrials.has(trial.trial_number);
                   const isBest = trial.objective === bestValue;
 
@@ -879,9 +934,9 @@ export default function Dashboard() {
           </div>
         </main>
 
-        {/* Claude Code Terminal - Right Sidebar */}
+        {/* Claude Code Terminal - Right Sidebar (taller for better visibility) */}
         {chatOpen && (
-          <aside className="col-span-4 h-[calc(100vh-12rem)] sticky top-24">
+          <aside className="col-span-4 h-[calc(100vh-8rem)] sticky top-20">
             <ClaudeTerminal
               isExpanded={chatExpanded}
               onToggleExpand={() => setChatExpanded(!chatExpanded)}

atomizer-dashboard/frontend/src/pages/Home.tsx

@@ -21,6 +21,7 @@ import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import remarkMath from 'remark-math';
 import rehypeKatex from 'rehype-katex';
+import 'katex/dist/katex.min.css';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
 import { apiClient } from '../api/client';
@@ -101,11 +102,24 @@ const Home: React.FC = () => {
     }
   };
 
-  // Sort studies: running first, then by trial count
+  // Study sort options
+  const [studySort, setStudySort] = useState<'date' | 'running' | 'trials'>('date');
+
+  // Sort studies based on selected sort option
   const sortedStudies = [...studies].sort((a, b) => {
-    if (a.status === 'running' && b.status !== 'running') return -1;
-    if (b.status === 'running' && a.status !== 'running') return 1;
-    return b.progress.current - a.progress.current;
+    if (studySort === 'running') {
+      // Running first, then by date
+      if (a.status === 'running' && b.status !== 'running') return -1;
+      if (b.status === 'running' && a.status !== 'running') return 1;
+    }
+    if (studySort === 'trials') {
+      // By trial count (most trials first)
+      return b.progress.current - a.progress.current;
+    }
+    // Default: sort by date (newest first)
+    const aDate = a.last_modified || a.created_at || '';
+    const bDate = b.last_modified || b.created_at || '';
+    return bDate.localeCompare(aDate);
   });
 
   const displayedStudies = showAllStudies ? sortedStudies : sortedStudies.slice(0, 6);
@@ -114,7 +128,7 @@ const Home: React.FC = () => {
     <div className="min-h-screen bg-dark-900">
       {/* Header */}
       <header className="bg-dark-800/50 border-b border-dark-700 backdrop-blur-sm sticky top-0 z-10">
-        <div className="max-w-[1600px] mx-auto px-6 py-4">
+        <div className="max-w-[1920px] mx-auto px-6 py-4">
           <div className="flex items-center justify-between">
             <div className="flex items-center gap-4">
               <div className="w-11 h-11 bg-gradient-to-br from-primary-500 to-primary-700 rounded-xl flex items-center justify-center shadow-lg shadow-primary-500/20">
@@ -138,7 +152,7 @@ const Home: React.FC = () => {
         </div>
       </header>
 
-      <main className="max-w-[1600px] mx-auto px-6 py-8">
+      <main className="max-w-[1920px] mx-auto px-6 py-8">
         {/* Study Selection Section */}
         <section className="mb-8">
           <div className="flex items-center justify-between mb-4">
@@ -146,18 +160,56 @@ const Home: React.FC = () => {
               <FolderOpen className="w-5 h-5 text-primary-400" />
               Select a Study
             </h2>
-            {studies.length > 6 && (
-              <button
-                onClick={() => setShowAllStudies(!showAllStudies)}
-                className="text-sm text-primary-400 hover:text-primary-300 flex items-center gap-1"
-              >
-                {showAllStudies ? (
-                  <>Show Less <ChevronUp className="w-4 h-4" /></>
-                ) : (
-                  <>Show All ({studies.length}) <ChevronDown className="w-4 h-4" /></>
-                )}
-              </button>
-            )}
+            <div className="flex items-center gap-4">
+              {/* Sort Controls */}
+              <div className="flex items-center gap-2">
+                <span className="text-sm text-dark-400">Sort:</span>
+                <div className="flex rounded-lg overflow-hidden border border-dark-600">
+                  <button
+                    onClick={() => setStudySort('date')}
+                    className={`px-3 py-1.5 text-sm transition-colors ${
+                      studySort === 'date'
+                        ? 'bg-primary-500 text-white'
+                        : 'bg-dark-700 text-dark-300 hover:bg-dark-600'
+                    }`}
+                  >
+                    Newest
+                  </button>
+                  <button
+                    onClick={() => setStudySort('running')}
+                    className={`px-3 py-1.5 text-sm transition-colors ${
+                      studySort === 'running'
+                        ? 'bg-primary-500 text-white'
+                        : 'bg-dark-700 text-dark-300 hover:bg-dark-600'
+                    }`}
+                  >
+                    Running
+                  </button>
+                  <button
+                    onClick={() => setStudySort('trials')}
+                    className={`px-3 py-1.5 text-sm transition-colors ${
+                      studySort === 'trials'
+                        ? 'bg-primary-500 text-white'
+                        : 'bg-dark-700 text-dark-300 hover:bg-dark-600'
+                    }`}
+                  >
+                    Most Trials
+                  </button>
+                </div>
+              </div>
+              {studies.length > 6 && (
+                <button
+                  onClick={() => setShowAllStudies(!showAllStudies)}
+                  className="text-sm text-primary-400 hover:text-primary-300 flex items-center gap-1"
+                >
+                  {showAllStudies ? (
+                    <>Show Less <ChevronUp className="w-4 h-4" /></>
+                  ) : (
+                    <>Show All ({studies.length}) <ChevronDown className="w-4 h-4" /></>
+                  )}
+                </button>
+              )}
+            </div>
           </div>
 
           {isLoading ? (
@@ -273,8 +325,8 @@ const Home: React.FC = () => {
                 <div className="p-8 overflow-x-auto">
                   <article className="markdown-body max-w-none">
                     <ReactMarkdown
-                      remarkPlugins={[remarkGfm, remarkMath]}
-                      rehypePlugins={[rehypeKatex]}
+                      remarkPlugins={[remarkGfm, [remarkMath, { singleDollarTextMath: false }]]}
+                      rehypePlugins={[[rehypeKatex, { strict: false, trust: true, output: 'html' }]]}
                       components={{
                         // Custom heading styles
                         h1: ({ children }) => (

atomizer-dashboard/frontend/src/types/index.ts

@@ -10,6 +10,8 @@ export interface Study {
   best_value: number | null;
   target: number | null;
   path: string;
+  created_at?: string;
+  last_modified?: string;
 }
 
 export interface StudyListResponse {