{
  "id": "evaluations_observability_claim_level_evidence_map",
  "generated_at": "2026-06-09",
  "theme": "evaluations_observability",
  "name": "评估与可观测",
  "status": "ready_for_human_review_of_claims",
  "evidence_grade": "A- · Apple-source indexed transcript",
  "coverage": {
    "source_sessions": 5,
    "transcripts_analyzed": 5,
    "total_words": 17189,
    "total_segments": 2067
  },
  "source_sessions": [
    {
      "session_id": "243",
      "title": "Debug and profile agentic app experiences with Instruments",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/243",
      "has_transcript": true,
      "word_count": 2101,
      "segments": 280
    },
    {
      "session_id": "298",
      "title": "Meet the Evaluations framework",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298",
      "has_transcript": true,
      "word_count": 3703,
      "segments": 438
    },
    {
      "session_id": "299",
      "title": "Create robust evaluations for agentic apps",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/299",
      "has_transcript": true,
      "word_count": 3385,
      "segments": 397
    },
    {
      "session_id": "268",
      "title": "Profile, fix, and verify: Improve app responsiveness with Instruments",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/268",
      "has_transcript": true,
      "word_count": 3825,
      "segments": 461
    },
    {
      "session_id": "335",
      "title": "Improve your prompts by hill-climbing with Evaluations",
      "apple_url": "https://developer.apple.com/videos/play/wwdc2026/335",
      "has_transcript": true,
      "word_count": 4175,
      "segments": 491
    }
  ],
  "review_policy": "No claim should be accepted until a human reviewer checks each mapped subtitle anchor against the claim wording.",
  "claims": [
    {
      "id": "EVAL-C1",
      "claim": "Evaluations framework 把 AI 功能质量从主观体验转向可定义、可比较的评估流程。",
      "claim_type": "Apple 明示 + Insight 综合",
      "confidence": "High",
      "evidence_status": "ready_for_human_review",
      "review_question": "证据是否支撑“可定义、可比较”，还是只是介绍 framework？",
      "anchors": [
        {
          "session_id": "298",
          "session_title": "Meet the Evaluations framework",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298",
          "segment_index": 2,
          "start_seconds": 10.0,
          "timecode": "0:10",
          "matched_terms": [
            "Evaluations framework"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/298/transcript.json"
        },
        {
          "session_id": "299",
          "session_title": "Create robust evaluations for agentic apps",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/299",
          "segment_index": 2,
          "start_seconds": 11.0,
          "timecode": "0:11",
          "matched_terms": [
            "evaluations"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/299/transcript.json"
        },
        {
          "session_id": "335",
          "session_title": "Improve your prompts by hill-climbing with Evaluations",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/335",
          "segment_index": 0,
          "start_seconds": 7.0,
          "timecode": "0:07",
          "matched_terms": [
            "Evaluations"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/335/transcript.json"
        }
      ],
      "anchor_count": 3,
      "anchor_sessions": [
        "298",
        "299",
        "335"
      ]
    },
    {
      "id": "EVAL-C2",
      "claim": "Agentic apps 的质量评估必须覆盖工具调用、动态条件和多步骤行为，而不只是最终文本输出。",
      "claim_type": "Apple 明示 + 多证据综合",
      "confidence": "High",
      "evidence_status": "ready_for_human_review",
      "review_question": "是否覆盖多步骤行为，还是主要覆盖工具调用？",
      "anchors": [
        {
          "session_id": "243",
          "session_title": "Debug and profile agentic app experiences with Instruments",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/243",
          "segment_index": 1,
          "start_seconds": 10.0,
          "timecode": "0:10",
          "matched_terms": [
            "Instruments"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/243/transcript.json"
        },
        {
          "session_id": "298",
          "session_title": "Meet the Evaluations framework",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298",
          "segment_index": 38,
          "start_seconds": 140.0,
          "timecode": "2:20",
          "matched_terms": [
            "evaluate"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/298/transcript.json"
        }
      ],
      "anchor_count": 2,
      "anchor_sessions": [
        "243",
        "298"
      ]
    },
    {
      "id": "EVAL-C3",
      "claim": "Instruments 在 agentic app 场景中承担调试与剖析作用，使 AI 体验具备运行时可观测性。",
      "claim_type": "Apple 明示 + Insight 综合",
      "confidence": "High",
      "evidence_status": "ready_for_human_review",
      "review_question": "“运行时可观测性”是否是合理综合，还是应具体称为 debug/profile？",
      "anchors": [
        {
          "session_id": "243",
          "session_title": "Debug and profile agentic app experiences with Instruments",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/243",
          "segment_index": 25,
          "start_seconds": 82.0,
          "timecode": "1:22",
          "matched_terms": [
            "Instruments"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/243/transcript.json"
        },
        {
          "session_id": "268",
          "session_title": "Profile, fix, and verify: Improve app responsiveness with Instruments",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/268",
          "segment_index": 1,
          "start_seconds": 9.0,
          "timecode": "0:09",
          "matched_terms": [
            "Instruments"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/268/transcript.json"
        },
        {
          "session_id": "299",
          "session_title": "Create robust evaluations for agentic apps",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/299",
          "segment_index": 22,
          "start_seconds": 83.0,
          "timecode": "1:23",
          "matched_terms": [
            "evaluate"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/299/transcript.json"
        }
      ],
      "anchor_count": 3,
      "anchor_sessions": [
        "243",
        "268",
        "299"
      ]
    },
    {
      "id": "EVAL-C4",
      "claim": "Synthetic datasets、model-as-judge 和 hill-climbing 等机制说明 prompt 和 agent 行为会进入持续优化闭环。",
      "claim_type": "Apple 明示 + Insight 推论",
      "confidence": "Medium-High",
      "evidence_status": "ready_for_human_review",
      "review_question": "这些机制是否足以支撑“持续优化闭环”？",
      "anchors": [
        {
          "session_id": "298",
          "session_title": "Meet the Evaluations framework",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298",
          "segment_index": 41,
          "start_seconds": 151.0,
          "timecode": "2:31",
          "matched_terms": [
            "dataset"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/298/transcript.json"
        },
        {
          "session_id": "299",
          "session_title": "Create robust evaluations for agentic apps",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/299",
          "segment_index": 17,
          "start_seconds": 63.0,
          "timecode": "1:03",
          "matched_terms": [
            "agentic"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/299/transcript.json"
        },
        {
          "session_id": "335",
          "session_title": "Improve your prompts by hill-climbing with Evaluations",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/335",
          "segment_index": 2,
          "start_seconds": 14.0,
          "timecode": "0:14",
          "matched_terms": [
            "improve"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/335/transcript.json"
        }
      ],
      "anchor_count": 3,
      "anchor_sessions": [
        "298",
        "299",
        "335"
      ]
    },
    {
      "id": "EVAL-C5",
      "claim": "评估与观测将成为 AI 功能进入产品化前的关键门槛，需要在功能设计阶段就前置。",
      "claim_type": "Insight 综合判断",
      "confidence": "Medium",
      "evidence_status": "ready_for_human_review",
      "review_question": "“产品化前门槛”属于推论，需要确认措辞是否过强。",
      "anchors": [
        {
          "session_id": "298",
          "session_title": "Meet the Evaluations framework",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/298",
          "segment_index": 33,
          "start_seconds": 123.0,
          "timecode": "2:03",
          "matched_terms": [
            "Evaluations framework"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/298/transcript.json"
        },
        {
          "session_id": "243",
          "session_title": "Debug and profile agentic app experiences with Instruments",
          "apple_url": "https://developer.apple.com/videos/play/wwdc2026/243",
          "segment_index": 31,
          "start_seconds": 99.0,
          "timecode": "1:39",
          "matched_terms": [
            "Instruments"
          ],
          "evidence_role": "supports claim component",
          "review_instruction": "Open internal transcript and verify support strength. Full text is not republished.",
          "internal_transcript_path": "wwdc_2026/internal_data/wwdc_sessions/sessions/243/transcript.json"
        }
      ],
      "anchor_count": 2,
      "anchor_sessions": [
        "243",
        "298"
      ]
    }
  ],
  "counts": {
    "claims": 5,
    "claims_ready_for_review": 5,
    "anchors": 13,
    "unique_anchor_sessions": 5
  },
  "known_limits": [
    "Anchors are locator references, not proof by themselves.",
    "Some claims are synthesis or inference and must be checked for overreach.",
    "Full transcript text is not republished; reviewer must open internal transcript paths."
  ]
}