{
  "id": "evaluations_observability_prelude_insight",
  "type": "prelude_insight",
  "theme": "evaluations_observability",
  "title": "Evaluations 前导洞察：先定义 AI 功能的发布门槛",
  "label": "Evaluations & Observability",
  "status": "prelude_ready",
  "evidence_grade": "A- · Apple-source indexed transcript",
  "intent": "进入 PoC 或分工前的认知对齐材料，不承诺排期。",
  "core_thesis": "Evaluations / Instruments 的核心意义是把 agentic feature 从 demo 变成可回归、可比较、可发布的工程资产；没有 evaluation baseline，就不应该进入 beta。",
  "what_to_align_first": [
    "不要用传统 unit test 取代 AI evaluation。",
    "不要只看平均效果，要看失败模式、回退路径和长尾。",
    "不要把 model-as-judge 当绝对真理，需要人审 sample 和指标解释。",
    "不要等产品化后补评估，evaluation dataset 应该和功能一起设计。",
    "不要只评估输出文本，还要评估 tool calling、流程正确性和用户控制权。"
  ],
  "prelude_questions": [
    {
      "area": "AI 平台",
      "question": "哪些指标必须标准化：correctness、latency、fallback、toxicity、instruction following？"
    },
    {
      "area": "QA",
      "question": "evaluation dataset 如何设计、维护和版本化？"
    },
    {
      "area": "客户端工程",
      "question": "telemetry 和 trace 需要记录哪些字段才足以复现问题？"
    },
    {
      "area": "产品",
      "question": "什么质量阈值才允许进入灰度或 beta？"
    },
    {
      "area": "设计",
      "question": "用户控制权、确认、撤销是否能被纳入 evaluation？"
    },
    {
      "area": "管理",
      "question": "谁拥有 AI feature 的 release gate？"
    }
  ],
  "recommended_next_step": "先开一次 60-90 分钟 Evaluations & Observability 前导洞察评审会，只做认知对齐、方向排序和分工准备。",
  "deferred_items": [
    "Evaluation baseline template",
    "Model-as-judge trial",
    "Synthetic dataset policy",
    "Agent trace schema"
  ],
  "source_sessions": [
    {
      "session_id": "243",
      "title": "Debug and profile agentic app experiences with Instruments",
      "priority": "P0",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/243"
    },
    {
      "session_id": "298",
      "title": "Meet the Evaluations framework",
      "priority": "P1",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298"
    },
    {
      "session_id": "299",
      "title": "Create robust evaluations for agentic apps",
      "priority": "P1",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/299"
    },
    {
      "session_id": "268",
      "title": "Profile, fix, and verify: Improve app responsiveness with Instruments",
      "priority": "P2",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/268"
    },
    {
      "session_id": "335",
      "title": "Improve your prompts by hill-climbing with Evaluations",
      "priority": "P2",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/335"
    }
  ],
  "last_updated": "2026-06-09"
}