AdamWen95
194 lines
10 KiB
Python
194 lines
10 KiB
Python
"""Source registration and filesystem boundaries shared by API and worker."""
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import uuid
|
|
from .store import now
|
|
|
|
SECRET = re.compile(r"(?im)(?:[\w-]*(?:api[_-]?key|password|token|secret)[\w-]*)\s*[:=]\s*['\"][^'\"\r\n]{8,}['\"]|-----BEGIN [A-Z ]*PRIVATE KEY-----|Bearer\s+[A-Za-z0-9._-]{16,}")
|
|
SERVICE = re.compile(r"(?im)\b(?:docker|podman|systemctl|sudo|pkill|killall|shutdown|reboot|nvidia-smi)\b|\brm\s+-[\w]*r|\bkill\s+-")
|
|
SMOKE_HASH = "be858e914d87454240791da773578cb3f80f380e740f919546db347b7715a13d"
|
|
|
|
|
|
def clean_source(content):
|
|
try:
|
|
text = content.decode("utf-8-sig")
|
|
except UnicodeDecodeError as exc:
|
|
raise ValueError("脚本必须使用 UTF-8 编码") from exc
|
|
if "\x00" in text:
|
|
raise ValueError("不支持二进制脚本")
|
|
return text
|
|
|
|
|
|
def safe_path(path, root, must_exist=True):
|
|
candidate, base = Path(path), Path(root)
|
|
if not candidate.is_absolute():
|
|
raise ValueError("路径必须为绝对路径")
|
|
# Never follow a symlink, including symlink ancestors under the allowed root.
|
|
resolved_base = base.resolve()
|
|
resolved = candidate.resolve()
|
|
if not resolved.is_relative_to(resolved_base):
|
|
raise ValueError("路径超出允许目录")
|
|
for component in [candidate, *candidate.parents]:
|
|
if component.is_symlink():
|
|
raise ValueError("不允许符号链接")
|
|
if component == base:
|
|
break
|
|
if must_exist and not candidate.exists():
|
|
raise ValueError("文件或目录不存在")
|
|
return candidate
|
|
|
|
|
|
def source_for(settings, group):
|
|
for source in settings.sources:
|
|
if source["id"] == group:
|
|
return Path(source["path"])
|
|
raise ValueError("未知来源目录")
|
|
|
|
|
|
def validate_config(settings, item, config):
|
|
from . import native_plans
|
|
if native_plans.profile_for(settings, item):
|
|
return native_plans.validate_config(settings, item, config)
|
|
if config.get("native_options"):
|
|
raise ValueError("此项目不支持原环境参数")
|
|
defaults = {"args": [], "cwd": "", "output_paths": [], "timeout_seconds": 300, "profile": "standard"}
|
|
config = {**defaults, **config}
|
|
if config["profile"] not in {"standard", "legacy_smoke"}:
|
|
raise ValueError("未知执行环境")
|
|
if not isinstance(config["args"], list) or len(config["args"]) > 40 or any(
|
|
not isinstance(a, str) or len(a) > 2048 or "\x00" in a for a in config["args"]):
|
|
raise ValueError("参数需为最多 40 项的字符串数组")
|
|
if SECRET.search("\n".join(config["args"])):
|
|
raise ValueError("参数不能包含明文凭据")
|
|
if config["profile"] == "legacy_smoke" and config["args"]:
|
|
raise ValueError("轻量模型兼容环境使用已核对的固定参数，请新建脚本以配置其他测试")
|
|
if type(config["timeout_seconds"]) is not int or not 5 <= config["timeout_seconds"] <= 7200:
|
|
raise ValueError("运行时限须为 5 至 7200 秒")
|
|
if not isinstance(config["output_paths"], list) or len(config["output_paths"]) > 8:
|
|
raise ValueError("最多登记 8 个日志或结果路径")
|
|
if not isinstance(config["cwd"], str):
|
|
raise ValueError("工作目录格式错误")
|
|
if item["kind"] == "managed":
|
|
if config["cwd"] or config["output_paths"]:
|
|
raise ValueError("上传脚本使用本次独立运行目录；通过 XAS_RUN_DIR 写入结果")
|
|
else:
|
|
source = source_for(settings, item["source_group"])
|
|
base = source.parent if source.name == "scripts" else source
|
|
cwd = safe_path(config["cwd"] or str(base), base)
|
|
if not cwd.is_dir():
|
|
raise ValueError("工作目录不存在")
|
|
config["cwd"] = str(cwd)
|
|
for raw in config["output_paths"]:
|
|
if not isinstance(raw, str):
|
|
raise ValueError("输出路径格式错误")
|
|
# Only logs/results directories in that script family may become writable.
|
|
if not any(Path(raw).resolve().is_relative_to((base / name).resolve()) for name in ("logs", "results")):
|
|
raise ValueError("原目录输出只允许该来源的 logs/ 或 results/ 下路径")
|
|
safe_path(raw, base)
|
|
return config
|
|
|
|
|
|
def script_path(settings, item):
|
|
root = settings.library if item["kind"] == "managed" else source_for(settings, item["source_group"])
|
|
path = safe_path(item["source_path"], root)
|
|
if not path.is_file() or path.suffix not in {".py", ".sh"} or path.stat().st_size > settings.max_upload:
|
|
raise ValueError("脚本缺失、类型不支持或大于 1 MiB")
|
|
return path
|
|
|
|
|
|
def read_code(settings, item):
|
|
data = script_path(settings, item).read_bytes()
|
|
text = clean_source(data)
|
|
redacted = bool(SECRET.search(text))
|
|
if redacted:
|
|
text = SECRET.sub("[已隐藏疑似凭据]", text)
|
|
return {"code": text, "filename": Path(item["source_path"]).name,
|
|
"sha256": hashlib.sha256(data).hexdigest(), "redacted": redacted, "truncated": False}
|
|
|
|
|
|
def upload(store, data, filename, name="", description="", identity=None):
|
|
if not filename or Path(filename).name != filename or "/" in filename or "\\" in filename or not re.fullmatch(r"[\w.\- ]{1,120}\.(py|sh)", filename):
|
|
raise ValueError("只支持文件名安全的 .py 或 .sh 脚本")
|
|
if len(data) > store.settings.max_upload:
|
|
raise ValueError("脚本不能超过 1 MiB")
|
|
if not data:
|
|
raise ValueError("脚本不能为空")
|
|
text = clean_source(data)
|
|
if SECRET.search(text):
|
|
raise ValueError("脚本包含疑似明文凭据，请移除后上传")
|
|
if identity:
|
|
item = store.script(identity)
|
|
if item["kind"] != "managed":
|
|
raise ValueError("原位置脚本不支持上传覆盖，请新建脚本")
|
|
if item["config"]["profile"] == "legacy_smoke" and hashlib.sha256(data).hexdigest() != SMOKE_HASH:
|
|
raise ValueError("此兼容环境绑定已核对的原脚本；修改代码请新建脚本并另行配置认证")
|
|
if (Path(filename).suffix == ".py") != (item["language"] == "python"):
|
|
raise ValueError("新版本的语言必须与原脚本一致")
|
|
else:
|
|
item = {"id": uuid.uuid4().hex, "name": (name or filename)[:120], "description": description[:4000],
|
|
"kind": "managed", "language": "python" if filename.endswith(".py") else "shell",
|
|
"source_group": "uploaded", "source_path": "", "status": "ready", "enabled": True,
|
|
"reason": "", "version": 0, "sha256": "", "updated_at": now(), "synthetic": False,
|
|
"config": {"args": [], "cwd": "", "output_paths": [], "timeout_seconds": 300, "profile": "standard"}}
|
|
return store.upload_version(item, data, filename)
|
|
|
|
|
|
def discover(store):
|
|
existing = {s["id"]: s for s in store.scripts() if s["kind"] == "external"}
|
|
found, registered = set(), 0
|
|
for source in store.settings.sources:
|
|
root = Path(source["path"])
|
|
if not root.is_dir():
|
|
continue
|
|
count = 0
|
|
for directory, dirs, files in os.walk(root, followlinks=False):
|
|
dirs[:] = [d for d in dirs if d not in {".git", "__pycache__", ".venv", "node_modules", "logs", "results"}
|
|
and not (Path(directory) / d).is_symlink()]
|
|
for filename in sorted(files):
|
|
path = Path(directory) / filename
|
|
if path.suffix not in {".py", ".sh"} or path.is_symlink():
|
|
continue
|
|
count += 1
|
|
if count > 2000:
|
|
raise ValueError("来源脚本超过扫描上限 2000，请收窄来源目录")
|
|
identity = hashlib.sha256(str(path).encode()).hexdigest()[:32]
|
|
found.add(identity)
|
|
before = existing.get(identity)
|
|
try:
|
|
if path.stat().st_size > store.settings.max_upload:
|
|
raise ValueError("大于 1 MiB，仅登记")
|
|
raw = path.read_bytes()
|
|
code = clean_source(raw)
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
blocked = bool(SERVICE.search(code) or SECRET.search(code))
|
|
reason = "涉及服务、设备管理或疑似凭据，需管理员审查后接入" if blocked else "请确认参数、依赖和输出路径后启用"
|
|
except (OSError, ValueError):
|
|
digest, blocked, reason = "", True, "文件不可读取或格式不支持"
|
|
item = before or {"id": identity, "name": filename, "description": "",
|
|
"kind": "external", "language": "python" if path.suffix == ".py" else "shell",
|
|
"source_group": source["id"], "source_path": str(path), "version": 1, "enabled": False,
|
|
"config": {"args": [], "cwd": str(root.parent if root.name == "scripts" else root),
|
|
"output_paths": [], "timeout_seconds": 300, "profile": "standard"}}
|
|
from .native_plans import profile_for
|
|
native = profile_for(store.settings, item)
|
|
if native:
|
|
blocked = not native.get("available", True)
|
|
reason = native.get("unavailable_reason", "原环境方案已登记，请确认参数后启用")
|
|
if before and before["sha256"] != digest:
|
|
item["version"] += 1
|
|
item["enabled"] = False
|
|
reason = "源文件已变动，请重新确认配置并启用"
|
|
if blocked:
|
|
item["enabled"] = False
|
|
item.update(sha256=digest, blocked=blocked, reason="" if item["enabled"] else reason,
|
|
status="ready" if item["enabled"] else "needs_configuration", updated_at=now())
|
|
store.save_script(item)
|
|
registered += int(before is None)
|
|
for identity, item in existing.items():
|
|
if identity not in found:
|
|
item.update(status="missing", enabled=False, reason="原脚本当前不存在或来源不可访问", updated_at=now())
|
|
store.save_script(item)
|
|
return {"registered": registered, "missing": len(set(existing) - found), "total": len(found)}
|