{
  "schema": "lri.research-domain-workbench/v1",
  "topic": {
    "id": "russian_semantic_search",
    "parent_domain": "information_retrieval",
    "title": "俄语语义检索与 RAG 评测",
    "title_en": "Russian Semantic Retrieval and RAG Evaluation",
    "subtitle": "以 RusBEIR、混合检索、长文档召回和 RAG 联合评测为核心的俄语检索能力专题",
    "goal": "判断俄语检索 Benchmark、模型组合和场景路由能否形成可复现、可迁移的组织能力，并把 MSU/RSF 团队映射为合作资源而非单一项目新闻。",
    "canonical_entry": "workbench.html",
    "status": "evidence_backed_seed",
    "last_verified": "2026-07-17"
  },
  "scope": {
    "include": [
      "Russian IR benchmark engineering",
      "BM25 / dense / sparse / hybrid retrieval",
      "neural reranking",
      "long-document retrieval",
      "RAG context selection and answer evaluation",
      "query and corpus routing",
      "Russian scientific and enterprise search"
    ],
    "exclude": [
      "KV Cache implementation",
      "Paged Attention kernels",
      "GPU memory scheduling",
      "generic long-context inference infrastructure"
    ]
  },
  "thesis": [
    {
      "id": "t1",
      "claim": "该项目的核心资产是俄语检索与 RAG 评测基础设施，而不是 KV Cache 或长上下文推理内核。",
      "grade": "Inferred",
      "decision_use": "主归属 Research Domains / Information Retrieval Intelligence；长上下文只作为检索压缩上下文的相邻问题。",
      "evidence_refs": [
        "src_rsf_project",
        "src_user_project_brief"
      ]
    },
    {
      "id": "t2",
      "claim": "最有价值的技术路线不是单一 Dense Retriever，而是查询分类、词法/神经召回、候选融合、神经重排和 RAG 上下文选择的组合。",
      "grade": "Research synthesis",
      "decision_use": "Pilot 必须比较多阶段 Pipeline，不能只做 Embedding 排名。",
      "evidence_refs": [
        "src_rusbeir_paper"
      ]
    },
    {
      "id": "t3",
      "claim": "RusBEIR 应作为版本化 Benchmark 资产管理，必须区分原生俄语、翻译、多语言抽取和新构造数据集。",
      "grade": "Methodological requirement",
      "decision_use": "所有总榜结果必须按数据来源和文档长度拆分。",
      "evidence_refs": [
        "src_rsf_project",
        "src_rusbeir_paper"
      ]
    },
    {
      "id": "t4",
      "claim": "合作入口应同时覆盖项目负责人和实际 Benchmark/NLP 核心成员。",
      "grade": "Organizational inference",
      "decision_use": "Dobrov 负责组织接口；Loukachevitch、Kovalev、Tikhomirov 应进入技术尽调。",
      "evidence_refs": [
        "src_user_project_brief"
      ]
    }
  ],
  "tracks": [
    {
      "id": "benchmark",
      "name": "Benchmark Engineering",
      "description": "数据集纳入、Qrels、版本、License、Hash、Leaderboard 与可复现脚本。"
    },
    {
      "id": "retrieval",
      "name": "Hybrid Retrieval",
      "description": "BM25、Dense、Sparse、Fusion 与 Neural Reranking 的组合。"
    },
    {
      "id": "long_document",
      "name": "Long-document Retrieval",
      "description": "全文、段落、滑窗和句子粒度下的召回、成本与时延。"
    },
    {
      "id": "rag",
      "name": "RAG Evaluation",
      "description": "检索、上下文选择、答案生成和答案质量联合评测。"
    },
    {
      "id": "routing",
      "name": "Scenario Routing",
      "description": "按查询类型、文档长度、领域和成本选择模型与 Pipeline。"
    }
  ],
  "overview_map": [
    {
      "layer": "L1",
      "title": "Benchmark",
      "description": "版本化俄语检索数据集、Qrels 和复现实验。"
    },
    {
      "layer": "L2",
      "title": "Retrieval",
      "description": "词法、神经、混合召回与重排。"
    },
    {
      "layer": "L3",
      "title": "Long documents",
      "description": "粒度切分、输入长度和召回成本。"
    },
    {
      "layer": "L4",
      "title": "RAG",
      "description": "上下文选择与答案评测。"
    },
    {
      "layer": "L5",
      "title": "Model routing",
      "description": "数据类型 × 查询类型 × 模型组合推荐矩阵。"
    }
  ],
  "capability_matrix": [
    {
      "capability": "Russian IR benchmark adapter",
      "track": "Benchmark Engineering",
      "primary_resources": [
        "RusBEIR",
        "RSF project 25-21-00206"
      ],
      "gap": "需要固定 release、dataset manifest、Qrels provenance 和 license matrix。",
      "evidence_refs": [
        "src_rsf_project",
        "src_rusbeir_paper"
      ]
    },
    {
      "capability": "Hybrid retrieval evaluation",
      "track": "Hybrid Retrieval",
      "primary_resources": [
        "BM25",
        "multilingual/Russian dense models",
        "rerankers"
      ],
      "gap": "需要统一模型版本、索引参数、延迟和成本采集。",
      "evidence_refs": [
        "src_rusbeir_paper"
      ]
    },
    {
      "capability": "Long-document stress test",
      "track": "Long-document Retrieval",
      "primary_resources": [
        "scientific papers",
        "technical reports",
        "enterprise documents"
      ],
      "gap": "需要 512/2K/8K/32K token 分层和全文/段落/滑窗对照。",
      "evidence_refs": [
        "src_user_project_brief"
      ]
    },
    {
      "capability": "Russian RAG joint evaluation",
      "track": "RAG Evaluation",
      "primary_resources": [
        "retrieval benchmark",
        "context selection",
        "answer evaluation"
      ],
      "gap": "需要答案真值、引用正确性和 Token 成本指标。",
      "evidence_refs": [
        "src_rsf_project"
      ]
    },
    {
      "capability": "Scenario/model routing",
      "track": "Scenario Routing",
      "primary_resources": [
        "query taxonomy",
        "corpus taxonomy",
        "pipeline scorecards"
      ],
      "gap": "需要真实 CRI/CSI/CBG-AI/AI-scenario 语料和查询分布。",
      "evidence_refs": [
        "src_user_project_brief"
      ]
    }
  ],
  "project_profile": {
    "project_number": "25-21-00206",
    "project_name": "Semantic search based on neural information-retrieval models for the Russian language: creating benchmark and developing new models",
    "leader": {
      "name": "Boris Dobrov",
      "role": "Head of the Laboratory of Information Resources Analysis",
      "affiliation": "Research Computing Center, Lomonosov Moscow State University",
      "research_boundary": "Organization and project interface; technical execution must include benchmark/NLP core members."
    },
    "laboratory": {
      "name": "Laboratory of Information Resources Analysis",
      "core_members": [
        "Grigory Kovalev",
        "Natalia Loukachevitch",
        "Mikhail Tikhomirov"
      ]
    },
    "cooperation_status": "qualified_clue_no_cooperation",
    "target_organizations": [
      "CRI",
      "CSI",
      "CBG-AI",
      "AI-scenario"
    ]
  },
  "claims": [
    {
      "id": "c1",
      "statement": "项目计划创建并发布不少于 20 个多样化俄语信息检索数据集的 Benchmark。",
      "grade": "Provided + project record",
      "status": "supported",
      "evidence_refs": [
        "src_rsf_project",
        "src_user_project_brief"
      ],
      "allowed_wording": "项目目标是建立不少于 20 个数据集的俄语检索 Benchmark。"
    },
    {
      "id": "c2",
      "statement": "项目将评估 Transformer encoder、LLM 等现有检索模型，并开发俄语专项模型。",
      "grade": "Project objective",
      "status": "supported_as_objective",
      "evidence_refs": [
        "src_rsf_project",
        "src_user_project_brief"
      ],
      "allowed_wording": "这是公开项目目标；专项模型是否已发布需要单独核实。"
    },
    {
      "id": "c3",
      "statement": "RusBEIR 是当前最重要的可复用技术资产。",
      "grade": "Inferred",
      "status": "supported",
      "evidence_refs": [
        "src_rusbeir_paper",
        "src_rsf_project"
      ],
      "allowed_wording": "RusBEIR 是该线索最适合进入内部复现和模型路由实验的资产。"
    },
    {
      "id": "c4",
      "statement": "项目适合长文档检索和 RAG 上下文选择，但不等同于 KV Cache 技术研究。",
      "grade": "Scope judgment",
      "status": "supported",
      "evidence_refs": [
        "src_user_project_brief"
      ],
      "allowed_wording": "它可提供长上下文系统的检索工作负载与评测，不是 KV Cache 内核合作方。"
    },
    {
      "id": "c5",
      "statement": "团队具备 Benchmark、俄语 NLP、生成模型评测和科学信息处理的连续能力。",
      "grade": "Provided project profile",
      "status": "candidate_supported",
      "evidence_refs": [
        "src_user_project_brief"
      ],
      "allowed_wording": "公开项目与成员信息显示相关能力，需要通过代码、数据和论文复现进一步验证。"
    },
    {
      "id": "c6",
      "statement": "最合适的首轮合作是 4–6 周可复现 Pilot。",
      "grade": "Recommendation",
      "status": "proposed",
      "evidence_refs": [
        "src_rsf_project",
        "src_rusbeir_paper"
      ],
      "allowed_wording": "建议先做 Benchmark Adapter、长文档压力测试和 RAG 联合评测。"
    }
  ],
  "evaluation_matrix": [
    {
      "dimension": "Data origin",
      "values": [
        "native Russian",
        "translated",
        "multilingual subset",
        "newly constructed"
      ],
      "reason": "避免统一平均分掩盖俄语真实性偏差。"
    },
    {
      "dimension": "Document length",
      "values": [
        "512",
        "2K",
        "8K",
        "32K tokens"
      ],
      "reason": "定位模型输入长度与粒度切分影响。"
    },
    {
      "dimension": "Retrieval stage",
      "values": [
        "BM25",
        "Dense",
        "Sparse neural",
        "Hybrid",
        "Reranker"
      ],
      "reason": "比较多阶段组合而不是单模型排行榜。"
    },
    {
      "dimension": "Outcome",
      "values": [
        "NDCG@10",
        "Recall@10/100",
        "latency",
        "index size",
        "GPU/CPU cost",
        "RAG answer quality",
        "input tokens"
      ],
      "reason": "把离线效果、系统成本和最终答案联合评估。"
    }
  ],
  "pilot_plan": [
    {
      "phase": "P1",
      "title": "Official reproduction",
      "output": "固定 RusBEIR release、数据 Hash、Qrels、官方 baseline。"
    },
    {
      "phase": "P2",
      "title": "Internal model adapter",
      "output": "接入内部 Retriever、Embedding、Reranker 与统一指标。"
    },
    {
      "phase": "P3",
      "title": "Long-document corpus",
      "output": "增加俄语科学、技术和企业长文档。"
    },
    {
      "phase": "P4",
      "title": "Retrieval + RAG",
      "output": "联合评估上下文选择、答案质量和 Token 成本。"
    },
    {
      "phase": "P5",
      "title": "Routing recommendation",
      "output": "形成数据类型 × 查询类型 × Pipeline 推荐矩阵。"
    }
  ],
  "evidence_sources": [
    {
      "id": "src_rsf_project",
      "title": "RSF project 25-21-00206",
      "url": "https://rscf.ru/project/25-21-00206/",
      "grade": "Direct project record",
      "summary": "Project identity, objectives, team and reporting boundary for Russian semantic search and information-retrieval evaluation."
    },
    {
      "id": "src_rusbeir_paper",
      "title": "Building Russian Benchmark for Evaluation of Information Retrieval Models",
      "url": "https://arxiv.org/abs/2504.12879",
      "grade": "Direct research paper",
      "summary": "Public paper describing a stage of the RusBEIR benchmark and model evaluation."
    },
    {
      "id": "src_user_project_brief",
      "title": "User-provided RSF project brief",
      "url": "local://conversation/rsf-25-21-00206",
      "grade": "Provided project brief",
      "summary": "Structured project, laboratory, objective, achievement and internal-routing information supplied for this topic."
    }
  ],
  "evidence_gaps": [
    {
      "gap": "专项俄语 Retriever 的公开发布状态不明确。",
      "severity": "P0",
      "next_action": "向团队索取模型卡、代码、权重和训练数据边界。"
    },
    {
      "gap": "Benchmark 不同阶段的数据集数量和 release 边界需要固定。",
      "severity": "P0",
      "next_action": "建立 manifest、版本、Hash、License 和 Qrels provenance。"
    },
    {
      "gap": "真实企业查询分布和权限过滤尚未进入公开 Benchmark。",
      "severity": "P1",
      "next_action": "由 CBG-AI/AI-scenario 提供脱敏场景集。"
    },
    {
      "gap": "长文档与 RAG 联合评测缺少统一成本指标。",
      "severity": "P1",
      "next_action": "补 Token、延迟、GPU/CPU、索引体积与答案正确率。"
    }
  ],
  "risks": [
    {
      "risk": "把 Benchmark 总榜当作生产选型结论",
      "impact": "忽略数据来源、查询类型、文档长度和成本。",
      "mitigation": "强制输出分桶结果和场景路由矩阵。"
    },
    {
      "risk": "把翻译数据等同原生俄语",
      "impact": "高估真实俄语形态和领域术语能力。",
      "mitigation": "按 native/translated/multilingual/newly-created 分组。"
    },
    {
      "risk": "把长文档检索误写成 KV Cache 研究",
      "impact": "错误分派团队与合作目标。",
      "mitigation": "KV Cache 只作为相邻基础设施标签，不作为主能力。"
    },
    {
      "risk": "只联系项目负责人",
      "impact": "技术尽调缺少实际 Benchmark/NLP 执行者。",
      "mitigation": "同时覆盖 Dobrov、Loukachevitch、Kovalev、Tikhomirov。"
    }
  ],
  "resource_profiles": [
    {
      "name": "RSF project 25-21-00206",
      "type": "funded research project",
      "status": "qualified_clue",
      "value": "俄语检索 Benchmark、模型评估、专项模型与应用建议。",
      "evidence_refs": [
        "src_rsf_project"
      ]
    },
    {
      "name": "RusBEIR",
      "type": "benchmark",
      "status": "public_reproduction_candidate",
      "value": "俄语检索模型统一评测入口。",
      "evidence_refs": [
        "src_rusbeir_paper"
      ]
    },
    {
      "name": "MSU Laboratory of Information Resources Analysis",
      "type": "research laboratory",
      "status": "regional_resource_mirror",
      "value": "俄语 NLP、科学信息处理和 Benchmark 合作团队。",
      "evidence_refs": [
        "src_user_project_brief"
      ]
    },
    {
      "name": "CRI / CSI / CBG-AI / AI-scenario",
      "type": "internal routing",
      "status": "spread_candidate",
      "value": "分别承接科研检索、搜索系统、企业 RAG 和场景评测。",
      "evidence_refs": [
        "src_user_project_brief"
      ]
    }
  ],
  "next_steps": [
    {
      "title": "固定 RusBEIR 版本",
      "description": "收集 release、数据列表、Hash、Qrels、License 和官方 baseline。"
    },
    {
      "title": "复现公开结果",
      "description": "在统一环境复现 BM25、Dense 和 Reranking。"
    },
    {
      "title": "建立长文档压力测试",
      "description": "覆盖全文、段落、滑窗、句子和四档长度。"
    },
    {
      "title": "组织技术尽调",
      "description": "由 CRI/CSI 主持，邀请项目负责人和三位核心成员。"
    },
    {
      "title": "形成合作决策",
      "description": "按复现性、场景增益、数据边界和合作可达性评级。"
    }
  ],
  "review_methodology_status": {
    "review_type": "evidence-backed resource/topic seed",
    "protocol": "sources/search_protocol.yaml",
    "search_log": "sources/search_log.csv",
    "screening_log": "sources/screening_log.csv",
    "traceability": "data/claim_traceability_matrix.json"
  }
}