xas-test-library/catalog/test-definitions.json

{
  "schema_version": 1,
  "reviewed_at": "2026-10-20",
  "basis": "Reviewed README entry points, script purpose comments and call references; registration does not authorize service/GPU actions.",
  "example_script_ids": [
    "499b615a7dc7435eb30789b7a7dc090e",
    "fe5a2b28c8b4449591ef55bd88299a6d"
  ],
  "tests": [
    {
      "key": "gpu-suite",
      "source_group": "company-gpu-suite",
      "entry": "xas-gpu-test.sh",
      "name": "GPU 综合验收套件",
      "description": "公司 XAS GPU Test v3.5.0 的统一入口，包含硬件压力、多卡通信、模型性能及报告。内部模块和套件自身测试统一收在详情中。",
      "kind": "suite",
      "members": [
        {
          "source_group": "company-gpu-suite",
          "pattern": "**/*"
        }
      ]
    },
    {
      "key": "deepseek-suite",
      "source_group": "dsv41-trial-20261010",
      "entry": "run-b-suite.sh",
      "name": "DeepSeek 综合性能测试",
      "description": "在第二套 DeepSeek 服务上完成单路曲线、并发吞吐、自然语言生成、多轮缓存和工具调用的一整轮测试。",
      "kind": "suite",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "bench-serve.sh"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "curves.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "multiturn_cache.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "spec_bench.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "spec_prompts.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "toolcall_stress.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-report-phase*.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-d2.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-d3.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-d456.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-unc*.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "model-suite",
      "source_group": "model-trials-20261011",
      "entry": "run-suite.sh",
      "name": "模型综合测试套件",
      "description": "面向被测模型的一整轮速度、长上下文、功能与稳定性测试。不同模型的历史调度脚本收在相关文件中。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-serve.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "curves.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "functional_check.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "multiturn_cache.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "needle.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_bench.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_prompts.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "toolcall_stress.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-glm-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-mimo-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-qwen-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "serve-model.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "functional",
      "source_group": "model-trials-20261011",
      "entry": "functional_check.py",
      "name": "模型功能检查",
      "description": "检查接口功能、结构化输出、多轮交互等可用性，不等同于答案准确率评测。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "functional_check.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "curves",
      "source_group": "model-trials-20261011",
      "entry": "curves.py",
      "name": "单路性能曲线",
      "description": "单请求逐项测量生成长度和上下文长度变化时的性能曲线。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "curves.py",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-curves.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "context_probe.py",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-mimo-recheck.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "language",
      "source_group": "model-trials-20261011",
      "entry": "spec_bench.py",
      "name": "自然语言生成速度",
      "description": "用固定自然语言题库测量生成速度、并发和不同解码设置的表现。",
      "kind": "test",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_prompts.py"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "spec_bench.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "conversation-cache",
      "source_group": "model-trials-20261011",
      "entry": "multiturn_cache.py",
      "name": "多轮对话缓存效果",
      "description": "模拟多轮对话，观察前缀缓存对每轮首字时间的影响。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "multiturn_cache.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "needle",
      "source_group": "model-trials-20261011",
      "entry": "needle.py",
      "name": "长上下文检索",
      "description": "在指定上下文长度和位置中检索目标信息，检查长上下文能力。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "needle.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "concurrent-input",
      "source_group": "model-trials-20261011",
      "entry": "concurrent_prefill.py",
      "name": "并发长输入读入",
      "description": "检查多个长输入同时到达时的读入表现。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "concurrent_prefill.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "cache-revisit",
      "source_group": "model-trials-20261011",
      "entry": "kv_revisit.py",
      "name": "长会话缓存回访",
      "description": "检查长会话缓存被挤出后再次访问时的等待时间。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "kv_revisit.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "interference",
      "source_group": "model-trials-20261011",
      "entry": "prefill_interference.py",
      "name": "长输入对聊天延迟的影响",
      "description": "测量超长输入进入时，对其他交互请求延迟造成的影响。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "prefill_interference.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "tool-calls",
      "source_group": "model-trials-20261011",
      "entry": "toolcall_stress.py",
      "name": "工具调用并发稳定性",
      "description": "检查并发工具调用、参数 JSON 和必填字段的稳定性。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "toolcall_stress.py",
          "role": "variant"
        }
      ]
    },
    {
      "key": "throughput",
      "source_group": "model-trials-20261011",
      "entry": "bench-serve.sh",
      "name": "单服务并发吞吐",
      "description": "对一个已部署的模型服务测量不同并发下的吞吐和延迟。",
      "kind": "test",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-one.sh"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "bench-serve.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "bench.py",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-fresh-seed-dsv41.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "math",
      "source_group": "dsv41-trial-20261010",
      "entry": "math_check.py",
      "name": "数学应用题回归",
      "description": "通过可自动判对错的算术应用题检查解码配置变更后是否出现明显退化。",
      "kind": "test",
      "members": []
    },
    {
      "key": "vision",
      "source_group": "dsv41-trial-20261010",
      "entry": "vision_check.py",
      "name": "图片理解功能检查",
      "description": "检查模型图片输入功能是否可用，需要准备图片素材。",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-d7v-d8.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "robust",
      "source_group": "model-trials-20261011",
      "entry": "robust_check.py",
      "name": "异常输入与断连检查",
      "description": "检查异常输入、请求中断等情况下的模型服务稳定性。",
      "kind": "test",
      "members": []
    },
    {
      "key": "agent",
      "source_group": "model-trials-20261011",
      "entry": "agent_sim.py",
      "name": "Agent 多轮场景测试",
      "description": "用多轮 Agent 场景请求测量运行表现和 token 消耗，汇总脚本和历史调度链作为配套文件。",
      "kind": "test",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "agent_summary.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "agent-chain.sh"
        }
      ]
    },
    {
      "key": "replicas",
      "source_group": "model-trials-20261011",
      "entry": "run-replicas.sh",
      "name": "多实例并行吞吐",
      "description": "测量多个模型实例同时运行时的合计能力；实例数、GPU 和端口仍由原配置确定。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-replicas.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-multi.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_bench.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_prompts.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "serve-replicas.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "serve-sets.sh"
        }
      ]
    },
    {
      "key": "engine-comparison",
      "source_group": "model-trials-20261011",
      "entry": "run-compare.sh",
      "name": "推理引擎与解码配置对比",
      "description": "比较推理引擎、权重精度和解码配置的速度表现；同类历史对照脚本不再单独平铺。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-serve.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "curves.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_bench.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "spec_prompts.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "qwen38-compare-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "qwen38-fast-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "qwen38-sglang-track.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-glm-mtp-retry.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-qwen-pin-check.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "run-qwen-shm-check.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-spec-compare.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-dspark-tests.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "soak",
      "source_group": "model-trials-20261011",
      "entry": "soak.sh",
      "name": "长时间稳定性测试",
      "description": "持续施加并发和长输入负载，观察失败、吞吐和资源变化。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-serve.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "concurrent_prefill.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "soak24-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "run-d8.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "capacity",
      "source_group": "model-trials-20261011",
      "entry": "cap-8x-chain.sh",
      "name": "单卡与整机并发上限",
      "description": "测量单卡或单套服务的并发上限，以及多实例整机满载表现。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-multi.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "serve-replicas.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "serve-sets.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "qwen38-cap-chain.sh",
          "role": "variant"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "x16-chain.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "cost",
      "source_group": "model-trials-20261011",
      "entry": "value-chain.sh",
      "name": "性能与成本评估",
      "description": "按聊天、检索和代码生成场景测量性能，结合既定价格口径汇总成本。",
      "kind": "suite",
      "members": [
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-multi.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "bench-one.sh"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "value_ladder.py"
        },
        {
          "source_group": "model-trials-20261011",
          "pattern": "value2-chain.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "light-smoke",
      "managed_profile": "legacy_smoke",
      "kind": "test",
      "members": [
        {
          "source_group": "dsv41-trial-20261010",
          "pattern": "smoke.sh",
          "role": "variant"
        }
      ]
    },
    {
      "key": "drama3-asr",
      "source_group": "drama3-tests-20261015",
      "entry": "test_asr.py",
      "name": "语音识别文件与实时转写",
      "description": "用现有 Drama3 合成三段录音，核验 Qwen3-ASR 文件转写、实时转写、字错率和响应延迟；会占用共享语音服务。",
      "kind": "suite",
      "members": []
    },
    {
      "key": "drama3-api",
      "source_group": "drama3-tests-20261015",
      "entry": "test_api.py",
      "name": "语音合成网关兼容性与质量回归",
      "description": "核验本地语音网关的多格式、四种音色、流式、变速、SDK 兼容和八路并发；保留原 selftest 凭据读取方式。",
      "kind": "suite",
      "members": []
    },
    {
      "key": "drama3-voice-chat",
      "source_group": "drama3-tests-20261015",
      "entry": "test_voice_chat.py",
      "name": "实时语音对话端到端回归",
      "description": "核验输入、回答、打断和四人对话的完整链路；当前语音服务预设仍指向旧 DeepSeek 模型名，待服务方确认后再开放。",
      "kind": "suite",
      "members": [
        {
          "source_group": "drama3-tests-20261015",
          "pattern": "test_voice_logic.py"
        }
      ]
    },
    {
      "key": "drama3-engine",
      "source_group": "drama3-tests-20261015",
      "entry": "test_drama3.py",
      "name": "Drama3 引擎合成与流式回归",
      "description": "核验 Drama3 引擎的合成、克隆、流式和并发；依赖原管理凭据及外网参考音频下载，暂未接通执行。",
      "kind": "suite",
      "members": []
    },
    {
      "key": "drama3-voice-capacity",
      "source_group": "drama3-tests-20261015",
      "entry": "load_voice.py",
      "name": "语音链路并发容量测试",
      "description": "用模拟大脑测语音链路的并发容量、首声延迟和播放卡顿；需补齐临时辅助容器生命周期与每次资源上限。",
      "kind": "suite",
      "members": [
        {
          "source_group": "drama3-tests-20261015",
          "pattern": "mock_brain.py"
        },
        {
          "source_group": "drama3-tests-20261015",
          "pattern": "load_summary.py"
        }
      ]
    }
  ]
}