ATOCore/src/atocore/memory/service.py

"""Memory Core — structured memory management.

Memory types (per Master Plan):
  - identity:    who the user is, role, background
  - preference:  how they like to work, style, tools
  - project:     project-specific knowledge and context
  - episodic:    what happened, conversations, events
  - knowledge:   verified facts, technical knowledge
  - adaptation:  learned corrections, behavioral adjustments

Memories have:
  - confidence (0.0–1.0): how certain we are
  - status: lifecycle state, one of MEMORY_STATUSES
      * candidate:  extracted from an interaction, awaiting human review
                    (Phase 9 Commit C). Candidates are NEVER included in
                    context packs.
      * active:     promoted/curated, visible to retrieval and context
      * superseded: replaced by a newer entry
      * invalid:    rejected / error-corrected
  - last_referenced_at / reference_count: reinforcement signal
      (Phase 9 Commit B). Bumped whenever a captured interaction's
      response content echoes this memory.
  - optional link to source chunk: traceability
"""

import uuid
from dataclasses import dataclass
from datetime import datetime, timezone

from atocore.models.database import get_connection
from atocore.observability.logger import get_logger
from atocore.projects.registry import resolve_project_name

log = get_logger("memory")

MEMORY_TYPES = [
    "identity",
    "preference",
    "project",
    "episodic",
    "knowledge",
    "adaptation",
]

MEMORY_STATUSES = [
    "candidate",
    "active",
    "superseded",
    "invalid",
    "graduated",  # Phase 5: memory has become an entity; content frozen, forward pointer in properties
]


@dataclass
class Memory:
    id: str
    memory_type: str
    content: str
    project: str
    source_chunk_id: str
    confidence: float
    status: str
    created_at: str
    updated_at: str
    last_referenced_at: str = ""
    reference_count: int = 0
    domain_tags: list[str] | None = None
    valid_until: str = ""  # ISO UTC; empty = permanent


def _audit_memory(
    memory_id: str,
    action: str,
    actor: str = "api",
    before: dict | None = None,
    after: dict | None = None,
    note: str = "",
) -> None:
    """Append an entry to memory_audit.

    Phase 4 Robustness V1. Every memory mutation flows through this
    helper so we can answer "how did this memory get to its current
    state?" and "when did we learn X?".

    ``action`` is a short verb: created, updated, promoted, rejected,
    superseded, invalidated, reinforced, auto_promoted, expired.
    ``actor`` identifies the caller: api (default), auto-triage,
    human-triage, host-cron, reinforcement, phase10-auto-promote,
    etc. ``before`` / ``after`` are field snapshots (JSON-serialized).
    Fail-open: a logging failure never breaks the mutation itself.
    """
    import json as _json
    try:
        with get_connection() as conn:
            conn.execute(
                "INSERT INTO memory_audit (id, memory_id, action, actor, "
                "before_json, after_json, note) VALUES (?, ?, ?, ?, ?, ?, ?)",
                (
                    str(uuid.uuid4()),
                    memory_id,
                    action,
                    actor or "api",
                    _json.dumps(before or {}),
                    _json.dumps(after or {}),
                    (note or "")[:500],
                ),
            )
    except Exception as e:
        log.warning("memory_audit_failed", memory_id=memory_id, action=action, error=str(e))


def get_memory_audit(memory_id: str, limit: int = 100) -> list[dict]:
    """Fetch audit entries for a memory, newest first."""
    import json as _json
    with get_connection() as conn:
        rows = conn.execute(
            "SELECT id, memory_id, action, actor, before_json, after_json, note, timestamp "
            "FROM memory_audit WHERE memory_id = ? ORDER BY timestamp DESC LIMIT ?",
            (memory_id, limit),
        ).fetchall()
    out = []
    for r in rows:
        try:
            before = _json.loads(r["before_json"] or "{}")
        except Exception:
            before = {}
        try:
            after = _json.loads(r["after_json"] or "{}")
        except Exception:
            after = {}
        out.append({
            "id": r["id"],
            "memory_id": r["memory_id"],
            "action": r["action"],
            "actor": r["actor"] or "api",
            "before": before,
            "after": after,
            "note": r["note"] or "",
            "timestamp": r["timestamp"],
        })
    return out


def get_recent_audit(limit: int = 50) -> list[dict]:
    """Fetch recent memory_audit entries across all memories, newest first."""
    import json as _json
    with get_connection() as conn:
        rows = conn.execute(
            "SELECT id, memory_id, action, actor, before_json, after_json, note, timestamp "
            "FROM memory_audit ORDER BY timestamp DESC LIMIT ?",
            (limit,),
        ).fetchall()
    out = []
    for r in rows:
        try:
            after = _json.loads(r["after_json"] or "{}")
        except Exception:
            after = {}
        out.append({
            "id": r["id"],
            "memory_id": r["memory_id"],
            "action": r["action"],
            "actor": r["actor"] or "api",
            "note": r["note"] or "",
            "timestamp": r["timestamp"],
            "content_preview": (after.get("content") or "")[:120],
        })
    return out


def _normalize_tags(tags) -> list[str]:
    """Coerce a tags value (list, JSON string, None) to a clean lowercase list."""
    import json as _json
    if tags is None:
        return []
    if isinstance(tags, str):
        try:
            tags = _json.loads(tags) if tags.strip().startswith("[") else []
        except Exception:
            tags = []
    if not isinstance(tags, list):
        return []
    out = []
    for t in tags:
        if not isinstance(t, str):
            continue
        t = t.strip().lower()
        if t and t not in out:
            out.append(t)
    return out


def create_memory(
    memory_type: str,
    content: str,
    project: str = "",
    source_chunk_id: str = "",
    confidence: float = 1.0,
    status: str = "active",
    domain_tags: list[str] | None = None,
    valid_until: str = "",
    actor: str = "api",
) -> Memory:
    """Create a new memory entry.

    ``status`` defaults to ``active`` for backward compatibility. Pass
    ``candidate`` when the memory is being proposed by the Phase 9 Commit C
    extractor and still needs human review before it can influence context.

    Phase 3: ``domain_tags`` is a list of lowercase domain strings
    (optics, mechanics, firmware, ...) for cross-project retrieval.
    ``valid_until`` is an ISO UTC timestamp; memories with valid_until
    in the past are excluded from context packs (but remain queryable).
    """
    import json as _json

    if memory_type not in MEMORY_TYPES:
        raise ValueError(f"Invalid memory type '{memory_type}'. Must be one of: {MEMORY_TYPES}")
    if status not in MEMORY_STATUSES:
        raise ValueError(f"Invalid status '{status}'. Must be one of: {MEMORY_STATUSES}")
    _validate_confidence(confidence)

    project = resolve_project_name(project)
    tags = _normalize_tags(domain_tags)
    tags_json = _json.dumps(tags)
    valid_until = (valid_until or "").strip() or None

    memory_id = str(uuid.uuid4())
    now = datetime.now(timezone.utc).isoformat()

    with get_connection() as conn:
        existing = conn.execute(
            "SELECT id FROM memories "
            "WHERE memory_type = ? AND content = ? AND project = ? AND status = ?",
            (memory_type, content, project, status),
        ).fetchone()
        if existing:
            log.info(
                "memory_duplicate_skipped",
                memory_type=memory_type,
                status=status,
                content_preview=content[:80],
            )
            return _row_to_memory(
                conn.execute("SELECT * FROM memories WHERE id = ?", (existing["id"],)).fetchone()
            )

        conn.execute(
            "INSERT INTO memories (id, memory_type, content, project, source_chunk_id, "
            "confidence, status, domain_tags, valid_until) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (memory_id, memory_type, content, project, source_chunk_id or None,
             confidence, status, tags_json, valid_until),
        )

    log.info(
        "memory_created",
        memory_type=memory_type,
        status=status,
        content_preview=content[:80],
        tags=tags,
        valid_until=valid_until or "",
    )

    _audit_memory(
        memory_id=memory_id,
        action="created",
        actor=actor,
        after={
            "memory_type": memory_type,
            "content": content,
            "project": project,
            "status": status,
            "confidence": confidence,
            "domain_tags": tags,
            "valid_until": valid_until or "",
        },
    )

    return Memory(
        id=memory_id,
        memory_type=memory_type,
        content=content,
        project=project,
        source_chunk_id=source_chunk_id,
        confidence=confidence,
        status=status,
        created_at=now,
        updated_at=now,
        last_referenced_at="",
        reference_count=0,
        domain_tags=tags,
        valid_until=valid_until or "",
    )


def get_memories(
    memory_type: str | None = None,
    project: str | None = None,
    active_only: bool = True,
    min_confidence: float = 0.0,
    limit: int = 50,
    status: str | None = None,
) -> list[Memory]:
    """Retrieve memories, optionally filtered.

    When ``status`` is provided explicitly, it takes precedence over
    ``active_only`` so callers can list the candidate review queue via
    ``get_memories(status='candidate')``. When ``status`` is omitted the
    legacy ``active_only`` behaviour still applies.
    """
    if status is not None and status not in MEMORY_STATUSES:
        raise ValueError(f"Invalid status '{status}'. Must be one of: {MEMORY_STATUSES}")

    query = "SELECT * FROM memories WHERE 1=1"
    params: list = []

    if memory_type:
        query += " AND memory_type = ?"
        params.append(memory_type)
    if project is not None:
        # Canonicalize on the read side so a caller passing an alias
        # finds rows that were stored under the canonical id (and
        # vice versa). resolve_project_name returns the input
        # unchanged for unregistered names so empty-string queries
        # for "no project scope" still work.
        query += " AND project = ?"
        params.append(resolve_project_name(project))
    if status is not None:
        query += " AND status = ?"
        params.append(status)
    elif active_only:
        query += " AND status = 'active'"
    if min_confidence > 0:
        query += " AND confidence >= ?"
        params.append(min_confidence)

    query += " ORDER BY confidence DESC, updated_at DESC LIMIT ?"
    params.append(limit)

    with get_connection() as conn:
        rows = conn.execute(query, params).fetchall()

    return [_row_to_memory(r) for r in rows]


def update_memory(
    memory_id: str,
    content: str | None = None,
    confidence: float | None = None,
    status: str | None = None,
    memory_type: str | None = None,
    domain_tags: list[str] | None = None,
    valid_until: str | None = None,
    actor: str = "api",
    note: str = "",
) -> bool:
    """Update an existing memory."""
    import json as _json

    with get_connection() as conn:
        existing = conn.execute("SELECT * FROM memories WHERE id = ?", (memory_id,)).fetchone()
        if existing is None:
            return False

        next_content = content if content is not None else existing["content"]
        next_status = status if status is not None else existing["status"]
        if confidence is not None:
            _validate_confidence(confidence)

        if next_status == "active":
            duplicate = conn.execute(
                "SELECT id FROM memories "
                "WHERE memory_type = ? AND content = ? AND project = ? AND status = 'active' AND id != ?",
                (existing["memory_type"], next_content, existing["project"] or "", memory_id),
            ).fetchone()
            if duplicate:
                raise ValueError("Update would create a duplicate active memory")

        # Capture before-state for audit
        before_snapshot = {
            "content": existing["content"],
            "status": existing["status"],
            "confidence": existing["confidence"],
            "memory_type": existing["memory_type"],
        }
        after_snapshot = dict(before_snapshot)

        updates = []
        params: list = []

        if content is not None:
            updates.append("content = ?")
            params.append(content)
            after_snapshot["content"] = content
        if confidence is not None:
            updates.append("confidence = ?")
            params.append(confidence)
            after_snapshot["confidence"] = confidence
        if status is not None:
            if status not in MEMORY_STATUSES:
                raise ValueError(f"Invalid status '{status}'. Must be one of: {MEMORY_STATUSES}")
            updates.append("status = ?")
            params.append(status)
            after_snapshot["status"] = status
        if memory_type is not None:
            if memory_type not in MEMORY_TYPES:
                raise ValueError(f"Invalid memory type '{memory_type}'. Must be one of: {MEMORY_TYPES}")
            updates.append("memory_type = ?")
            params.append(memory_type)
            after_snapshot["memory_type"] = memory_type
        if domain_tags is not None:
            norm_tags = _normalize_tags(domain_tags)
            updates.append("domain_tags = ?")
            params.append(_json.dumps(norm_tags))
            after_snapshot["domain_tags"] = norm_tags
        if valid_until is not None:
            vu = valid_until.strip() or None
            updates.append("valid_until = ?")
            params.append(vu)
            after_snapshot["valid_until"] = vu or ""

        if not updates:
            return False

        updates.append("updated_at = CURRENT_TIMESTAMP")
        params.append(memory_id)

        result = conn.execute(
            f"UPDATE memories SET {', '.join(updates)} WHERE id = ?",
            params,
        )

    if result.rowcount > 0:
        log.info("memory_updated", memory_id=memory_id)
        # Action verb is driven by status change when applicable; otherwise "updated"
        if status == "active" and before_snapshot["status"] == "candidate":
            action = "promoted"
        elif status == "invalid" and before_snapshot["status"] == "candidate":
            action = "rejected"
        elif status == "invalid":
            action = "invalidated"
        elif status == "superseded":
            action = "superseded"
        else:
            action = "updated"
        _audit_memory(
            memory_id=memory_id,
            action=action,
            actor=actor,
            before=before_snapshot,
            after=after_snapshot,
            note=note,
        )
        return True
    return False


def invalidate_memory(memory_id: str, actor: str = "api") -> bool:
    """Mark a memory as invalid (error correction)."""
    return update_memory(memory_id, status="invalid", actor=actor)


def supersede_memory(memory_id: str, actor: str = "api") -> bool:
    """Mark a memory as superseded (replaced by newer info)."""
    return update_memory(memory_id, status="superseded", actor=actor)


def promote_memory(memory_id: str, actor: str = "api", note: str = "") -> bool:
    """Promote a candidate memory to active (Phase 9 Commit C review queue).

    Returns False if the memory does not exist or is not currently a
    candidate. Raises ValueError only if the promotion would create a
    duplicate active memory (delegates to update_memory's existing check).
    """
    with get_connection() as conn:
        row = conn.execute(
            "SELECT status FROM memories WHERE id = ?", (memory_id,)
        ).fetchone()
    if row is None:
        return False
    if row["status"] != "candidate":
        return False
    return update_memory(memory_id, status="active", actor=actor, note=note)


def reject_candidate_memory(memory_id: str, actor: str = "api", note: str = "") -> bool:
    """Reject a candidate memory (Phase 9 Commit C).

    Sets the candidate's status to ``invalid`` so it drops out of the
    review queue without polluting the active set. Returns False if the
    memory does not exist or is not currently a candidate.
    """
    with get_connection() as conn:
        row = conn.execute(
            "SELECT status FROM memories WHERE id = ?", (memory_id,)
        ).fetchone()
    if row is None:
        return False
    if row["status"] != "candidate":
        return False
    return update_memory(memory_id, status="invalid", actor=actor, note=note)


def reinforce_memory(
    memory_id: str,
    confidence_delta: float = 0.02,
) -> tuple[bool, float, float]:
    """Bump a memory's confidence and reference count (Phase 9 Commit B).

    Returns a 3-tuple ``(applied, old_confidence, new_confidence)``.
    ``applied`` is False if the memory does not exist or is not in the
    ``active`` state — reinforcement only touches live memories so the
    candidate queue and invalidated history are never silently revived.

    Confidence is capped at 1.0. last_referenced_at is set to the current
    UTC time in SQLite-comparable format. reference_count is incremented
    by one per call (not per delta amount).
    """
    if confidence_delta < 0:
        raise ValueError("confidence_delta must be non-negative for reinforcement")
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
    with get_connection() as conn:
        row = conn.execute(
            "SELECT confidence, status FROM memories WHERE id = ?", (memory_id,)
        ).fetchone()
        if row is None or row["status"] != "active":
            return False, 0.0, 0.0
        old_confidence = float(row["confidence"])
        new_confidence = min(1.0, old_confidence + confidence_delta)
        conn.execute(
            "UPDATE memories SET confidence = ?, last_referenced_at = ?, "
            "reference_count = COALESCE(reference_count, 0) + 1 "
            "WHERE id = ?",
            (new_confidence, now, memory_id),
        )
    log.info(
        "memory_reinforced",
        memory_id=memory_id,
        old_confidence=round(old_confidence, 4),
        new_confidence=round(new_confidence, 4),
    )
    # Reinforcement writes an audit row per bump. Reinforcement fires often
    # (every captured interaction); this lets you trace which interactions
    # kept which memories alive. Could become chatty but is invaluable for
    # decay/cold-memory analysis. If it becomes an issue, throttle here.
    _audit_memory(
        memory_id=memory_id,
        action="reinforced",
        actor="reinforcement",
        before={"confidence": old_confidence},
        after={"confidence": new_confidence},
    )
    return True, old_confidence, new_confidence


def auto_promote_reinforced(
    min_reference_count: int = 3,
    min_confidence: float = 0.7,
    max_age_days: int = 14,
) -> list[str]:
    """Auto-promote candidate memories with strong reinforcement signals.

    Phase 10: memories that have been reinforced by multiple interactions
    graduate from candidate to active without human review. This rewards
    knowledge that the system keeps referencing organically.

    Returns a list of promoted memory IDs.
    """
    from datetime import timedelta

    cutoff = (
        datetime.now(timezone.utc) - timedelta(days=max_age_days)
    ).strftime("%Y-%m-%d %H:%M:%S")
    promoted: list[str] = []
    with get_connection() as conn:
        rows = conn.execute(
            "SELECT id, content, memory_type, project, confidence, "
            "reference_count FROM memories "
            "WHERE status = 'candidate' "
            "AND COALESCE(reference_count, 0) >= ? "
            "AND confidence >= ? "
            "AND last_referenced_at >= ?",
            (min_reference_count, min_confidence, cutoff),
        ).fetchall()

    for row in rows:
        mid = row["id"]
        ok = promote_memory(
            mid,
            actor="phase10-auto-promote",
            note=f"ref_count={row['reference_count']} confidence={row['confidence']:.2f}",
        )
        if ok:
            promoted.append(mid)
            log.info(
                "memory_auto_promoted",
                memory_id=mid,
                memory_type=row["memory_type"],
                project=row["project"] or "(global)",
                reference_count=row["reference_count"],
                confidence=round(row["confidence"], 3),
            )
    return promoted


def expire_stale_candidates(
    max_age_days: int = 14,
) -> list[str]:
    """Reject candidate memories that sat in queue too long unreinforced.

    Candidates older than ``max_age_days`` with zero reinforcement are
    auto-rejected to prevent unbounded queue growth. Returns rejected IDs.
    """
    from datetime import timedelta

    cutoff = (
        datetime.now(timezone.utc) - timedelta(days=max_age_days)
    ).strftime("%Y-%m-%d %H:%M:%S")
    expired: list[str] = []
    with get_connection() as conn:
        rows = conn.execute(
            "SELECT id FROM memories "
            "WHERE status = 'candidate' "
            "AND COALESCE(reference_count, 0) = 0 "
            "AND created_at < ?",
            (cutoff,),
        ).fetchall()

    for row in rows:
        mid = row["id"]
        ok = reject_candidate_memory(
            mid,
            actor="candidate-expiry",
            note=f"unreinforced for {max_age_days}+ days",
        )
        if ok:
            expired.append(mid)
            log.info("memory_expired", memory_id=mid)
    return expired


def get_memories_for_context(
    memory_types: list[str] | None = None,
    project: str | None = None,
    budget: int = 500,
    header: str = "--- AtoCore Memory ---",
    footer: str = "--- End Memory ---",
    query: str | None = None,
) -> tuple[str, int]:
    """Get formatted memories for context injection.

    Returns (formatted_text, char_count).

    Budget allocation per Master Plan section 9:
      identity: 5%, preference: 5%, rest from retrieval budget

    The caller can override ``header`` / ``footer`` to distinguish
    multiple memory blocks in the same pack (e.g. identity/preference
    vs project/knowledge memories).

    When ``query`` is provided, candidates within each memory type
    are ranked by lexical overlap against the query (stemmed token
    intersection, ties broken by confidence). Without a query,
    candidates fall through in the order ``get_memories`` returns
    them — which is effectively "by confidence desc".
    """
    if memory_types is None:
        memory_types = ["identity", "preference"]

    if budget <= 0:
        return "", 0
    wrapper_chars = len(header) + len(footer) + 2
    if budget <= wrapper_chars:
        return "", 0

    available = budget - wrapper_chars
    selected_entries: list[str] = []
    used = 0

    # Pre-tokenize the query once. ``_score_memory_for_query`` is a
    # free function below that reuses the reinforcement tokenizer so
    # lexical scoring here matches the reinforcement matcher.
    query_tokens: set[str] | None = None
    if query:
        from atocore.memory.reinforcement import _normalize, _tokenize

        query_tokens = _tokenize(_normalize(query))
        if not query_tokens:
            query_tokens = None

    # Collect ALL candidates across the requested types into one
    # pool, then rank globally before the budget walk. Ranking per
    # type and walking types in order would starve later types when
    # the first type's candidates filled the budget — even if a
    # later-type candidate matched the query perfectly. Type order
    # is preserved as a stable tiebreaker inside
    # ``_rank_memories_for_query`` via Python's stable sort.
    pool: list[Memory] = []
    seen_ids: set[str] = set()
    for mtype in memory_types:
        for mem in get_memories(
            memory_type=mtype,
            project=project,
            min_confidence=0.5,
            limit=30,
        ):
            if mem.id in seen_ids:
                continue
            seen_ids.add(mem.id)
            pool.append(mem)

    # Phase 3: filter out expired memories (valid_until in the past).
    # Raw API queries still return them (for audit/history) but context
    # packs must not surface stale facts.
    if pool:
        pool = _filter_expired(pool)

    if query_tokens is not None:
        pool = _rank_memories_for_query(pool, query_tokens, query=query)

    # Per-entry cap prevents a single long memory from monopolizing
    # the band. With 16 p06 memories competing for ~700 chars, an
    # uncapped 530-char overview memory fills the entire budget before
    # a query-relevant 150-char memory gets a slot. The cap ensures at
    # least 2-3 entries fit regardless of individual memory length.
    max_entry_chars = 250
    for mem in pool:
        content = mem.content
        if len(content) > max_entry_chars:
            content = content[:max_entry_chars - 3].rstrip() + "..."
        entry = f"[{mem.memory_type}] {content}"
        entry_len = len(entry) + 1
        if entry_len > available - used:
            continue
        selected_entries.append(entry)
        used += entry_len

    if not selected_entries:
        return "", 0

    lines = [header, *selected_entries, footer]
    text = "\n".join(lines)

    log.info("memories_for_context", count=len(selected_entries), chars=len(text))
    return text, len(text)


def _filter_expired(memories: list["Memory"]) -> list["Memory"]:
    """Drop memories whose valid_until is in the past (UTC comparison)."""
    now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d")
    out = []
    for m in memories:
        vu = (m.valid_until or "").strip()
        if not vu:
            out.append(m)
            continue
        # Compare as string (ISO dates/timestamps sort correctly lexicographically
        # when they have the same format; date-only vs full ts both start YYYY-MM-DD).
        if vu[:10] >= now_iso:
            out.append(m)
        # else: expired, drop silently
    return out


def _rank_memories_for_query(
    memories: list["Memory"],
    query_tokens: set[str],
    query: str | None = None,
) -> list["Memory"]:
    """Rerank a memory list by lexical overlap with a pre-tokenized query.

    Primary key: overlap_density (overlap_count / memory_token_count),
    which rewards short focused memories that match the query precisely
    over long overview memories that incidentally share a few tokens.
    Secondary: absolute overlap count. Tertiary: domain-tag match.
    Quaternary: confidence.

    Phase 3: domain_tags contribute a boost when they appear in the
    query text. A memory tagged [optics, thermal] for a query about
    "optics coating" gets promoted above a memory without those tags.
    Tag boost fires AFTER content-overlap density so it only breaks
    ties among content-similar candidates.
    """
    from atocore.memory.reinforcement import _normalize, _tokenize

    query_lower = (query or "").lower()

    scored: list[tuple[float, int, int, float, Memory]] = []
    for mem in memories:
        mem_tokens = _tokenize(_normalize(mem.content))
        overlap = len(mem_tokens & query_tokens) if mem_tokens else 0
        density = overlap / len(mem_tokens) if mem_tokens else 0.0

        # Tag boost: count how many of the memory's domain_tags appear
        # as substrings in the raw query. Strong signal for topical match.
        tag_hits = 0
        for tag in (mem.domain_tags or []):
            if tag and tag in query_lower:
                tag_hits += 1

        scored.append((density, overlap, tag_hits, mem.confidence, mem))
    scored.sort(key=lambda t: (t[0], t[1], t[2], t[3]), reverse=True)
    return [mem for _, _, _, _, mem in scored]


def _row_to_memory(row) -> Memory:
    """Convert a DB row to Memory dataclass."""
    import json as _json
    keys = row.keys() if hasattr(row, "keys") else []
    last_ref = row["last_referenced_at"] if "last_referenced_at" in keys else None
    ref_count = row["reference_count"] if "reference_count" in keys else 0
    tags_raw = row["domain_tags"] if "domain_tags" in keys else None
    try:
        tags = _json.loads(tags_raw) if tags_raw else []
        if not isinstance(tags, list):
            tags = []
    except Exception:
        tags = []
    valid_until = row["valid_until"] if "valid_until" in keys else None
    return Memory(
        id=row["id"],
        memory_type=row["memory_type"],
        content=row["content"],
        project=row["project"] or "",
        source_chunk_id=row["source_chunk_id"] or "",
        confidence=row["confidence"],
        status=row["status"],
        created_at=row["created_at"],
        updated_at=row["updated_at"],
        last_referenced_at=last_ref or "",
        reference_count=int(ref_count or 0),
        domain_tags=tags,
        valid_until=valid_until or "",
    )


def _validate_confidence(confidence: float) -> None:
    if not 0.0 <= confidence <= 1.0:
        raise ValueError("Confidence must be between 0.0 and 1.0")