AdamWen95
161 lines
9.4 KiB
Python
161 lines
9.4 KiB
Python
"""Manage only this console's transient units; reattach after worker restarts."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from .native_plans import build_plan, public_plan
|
|
from .store import now
|
|
|
|
|
|
def unit_name(identity):
|
|
if not re.fullmatch(r"[a-f0-9]{32}", identity):raise ValueError("运行编号不合法")
|
|
return "adamwen-native-" + identity + ".service"
|
|
|
|
|
|
def inspect(identity):
|
|
unit = unit_name(identity)
|
|
result = subprocess.run(["/usr/bin/systemctl", "show", unit, "--property=Id,LoadState,ActiveState,SubState,Description,Result,ExecMainStatus,MainPID,TasksCurrent"],
|
|
capture_output=True, timeout=8, text=True)
|
|
values = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line)
|
|
if values.get("LoadState") == "not-found":return None
|
|
if result.returncode or not values.get("LoadState"):
|
|
raise OSError("systemd 状态查询失败，任务状态未知")
|
|
if values.get("Id") != unit or values.get("Description") != "XAS native " + identity:
|
|
raise ValueError("原环境服务归属核验失败")
|
|
return values
|
|
|
|
|
|
def stop_unit(identity):
|
|
if inspect(identity):
|
|
subprocess.run(["/usr/bin/systemctl", "--no-block", "stop", unit_name(identity)], check=True, capture_output=True, timeout=8)
|
|
|
|
|
|
class NativeExecutor:
|
|
def __init__(self, worker):
|
|
self.worker, self.store, self.settings = worker, worker.store, worker.settings
|
|
|
|
def execute(self, run):
|
|
if self.store.run(run["id"])["cancel_requested"]:
|
|
self.store.finish(run["id"], "cancelled", message="启动前已取消，原脚本未执行")
|
|
return
|
|
directory = self.settings.runs / run["id"]
|
|
directory.mkdir(mode=0o750, exist_ok=False)
|
|
try:
|
|
script = {**run, "id": run["script_id"]}
|
|
plan = build_plan(self.settings, script, run_id=run["id"])
|
|
if plan["plan_token"] != run["execution"]["plan_token"] or not plan["ready"]:
|
|
raise ValueError("执行前原入口或环境已变化，请重新预检；未启动原测试")
|
|
if plan["impact"]["requires_confirmation"] and not run["execution"].get("impact_acknowledged"):
|
|
raise ValueError("缺少本次运行影响确认")
|
|
if inspect(run["id"]):raise ValueError("本次原环境服务已存在，拒绝重复启动")
|
|
run = self.store.update_execution(run["id"], {**public_plan(plan), "unit": unit_name(run["id"])})
|
|
project = Path(self.worker.config["project"])
|
|
log = directory / "console.log"
|
|
log.touch(mode=0o600)
|
|
with (directory / "native-permit").open("x") as permit:
|
|
permit.write(run["execution"]["plan_token"])
|
|
command = ["/usr/bin/systemd-run", "--unit=" + unit_name(run["id"]), "--description=XAS native " + run["id"],
|
|
"--service-type=exec", "--property=RemainAfterExit=yes", "--property=KillMode=control-group",
|
|
"--property=TimeoutStopSec=120", "--property=RuntimeMaxSec=" + str(plan["timeout_seconds"]),
|
|
"--property=WorkingDirectory=" + str(project), "--property=User=root", "--property=UMask=0022",
|
|
"--property=StandardOutput=append:" + str(log), "--property=StandardError=append:" + str(log),
|
|
"--setenv=XAS_CONSOLE_ROOT=" + str(self.settings.root), "--setenv=PYTHONDONTWRITEBYTECODE=1",
|
|
"--", "/usr/bin/python3", "-m", "console.native_job", run["id"]]
|
|
if self.store.run(run["id"])["cancel_requested"]:
|
|
(directory / "native-permit").unlink(missing_ok=True)
|
|
self.store.finish(run["id"], "cancelled", message="启动前已取消，原脚本未执行")
|
|
return
|
|
result = subprocess.run(command, capture_output=True, timeout=15, text=True)
|
|
if result.returncode:
|
|
raise ValueError("systemd 未接受原环境任务，原脚本未启动")
|
|
except Exception as exc:
|
|
message = str(exc) if isinstance(exc, ValueError) else "原环境启动失败，未自动重试"
|
|
# A timeout can leave a submitted unit; inspect it instead of submitting again.
|
|
if inspect(run["id"]):
|
|
return self.monitor(run)
|
|
(directory / "native-permit").unlink(missing_ok=True)
|
|
if (directory / "native-started").exists():
|
|
self.worker.stopping = True
|
|
self.store.finish(run["id"], "interrupted", message="原入口可能已启动但服务状态未知，执行器停止接单；请核对后再操作")
|
|
return
|
|
(directory / "console.log").write_text(message + "\n", encoding="utf-8")
|
|
self.store.finish(run["id"], "failed", message=message)
|
|
return
|
|
self.monitor(run)
|
|
|
|
def monitor(self, run):
|
|
requested = False
|
|
failures = 0
|
|
while not self.worker.stopping:
|
|
self.store.heartbeat()
|
|
try:
|
|
state = inspect(run["id"])
|
|
failures = 0
|
|
except (OSError, ValueError, subprocess.TimeoutExpired):
|
|
failures += 1
|
|
self.store.running_message(run["id"], "无法确认原环境服务状态，保留运行占位，不启动下一项")
|
|
if failures >= 10:self.worker.stopping = True
|
|
time.sleep(1)
|
|
continue
|
|
if state is None or state.get("SubState") in {"exited", "dead", "failed"}:
|
|
return self.finish(run, state)
|
|
if self.store.run(run["id"])["cancel_requested"] and not requested:
|
|
stop_unit(run["id"])
|
|
requested = True
|
|
time.sleep(.5)
|
|
# The independent unit keeps running. The next worker attaches without replaying it.
|
|
|
|
def finish(self, run, unit):
|
|
directory = self.settings.runs / run["id"]
|
|
directory.mkdir(mode=0o750, exist_ok=True)
|
|
outcome_file = directory / "unit-outcome.json"
|
|
outcome = unit
|
|
if outcome_file.exists():outcome = json.loads(outcome_file.read_text())
|
|
elif unit:outcome_file.write_text(json.dumps(unit))
|
|
result_file = directory / "native-result.json"
|
|
result = json.loads(result_file.read_text()) if result_file.is_file() else {}
|
|
current = self.store.run(run["id"])
|
|
code = result.get("exit_code")
|
|
if current["cancel_requested"]:state, message = "cancelled", "已取消本次宿主入口；请核对原脚本启动的 Docker/tmux 任务和模型恢复情况"
|
|
elif outcome and outcome.get("Result") == "timeout":state, message = "timed_out", "原环境运行超过时限；原服务是否恢复需结合收尾记录核对"
|
|
elif not result:state, message = "interrupted", "原环境任务结束但完整结果未取得；未自动重跑，请查看日志和原服务状态"
|
|
elif result.get("stopped"):state, message = "interrupted", "原环境入口收到外部停止信号，服务恢复情况未确认"
|
|
elif code == 0:state, message = "succeeded", "原入口正常退出；业务结论和共享资源状态请查看报告"
|
|
else:state, message = "failed", f"原入口退出码 {code}，请查看日志"
|
|
sessions = result.get("runtime_changes", {}).get("new_tmux_sessions", [])
|
|
if sessions and state == "succeeded":
|
|
state, message = "interrupted", "原入口已退出，但检测到新增 tmux 会话；后台任务完成状态未确认，请核对原工具"
|
|
artifacts = result.get("artifacts", [])
|
|
summary = {"run_id": run["id"], "state": state, "exit_code": code, "message": message,
|
|
"finished_at": now(), "source_sha256": run["sha256"], "execution": run["execution"],
|
|
"runtime_changes": result.get("runtime_changes"), "archive_notices": result.get("archive_notices", []),
|
|
"log_truncated": result.get("log_truncated", False)}
|
|
directory.mkdir(mode=0o750, exist_ok=True)
|
|
(directory / "artifacts").mkdir(exist_ok=True)
|
|
data = json.dumps(summary, ensure_ascii=False, indent=2).encode()
|
|
identity = hashlib.sha256((str(directory) + "/summary.json").encode()).hexdigest()[:24]
|
|
(directory / "artifacts" / identity).write_bytes(data)
|
|
artifacts.append({"id": identity, "name": "summary.json", "kind": "json", "size": len(data)})
|
|
if unit:
|
|
if not self.wait_stopped(run):return
|
|
subprocess.run(["/usr/bin/systemctl", "reset-failed", unit_name(run["id"])], capture_output=True, timeout=8)
|
|
self.store.finish(run["id"], state, message=message, exit_code=code, artifacts=artifacts)
|
|
|
|
def wait_stopped(self, run):
|
|
try:
|
|
stop_unit(run["id"])
|
|
deadline = time.monotonic() + 130
|
|
while time.monotonic() < deadline and not self.worker.stopping:
|
|
self.store.heartbeat()
|
|
current = inspect(run["id"])
|
|
if current is None:return True
|
|
if current.get("ActiveState") in {"inactive", "failed"} and current.get("MainPID", "0") == "0" and current.get("TasksCurrent", "0") in {"0", "[not set]"}:
|
|
return True
|
|
time.sleep(.5)
|
|
except (OSError, ValueError, subprocess.TimeoutExpired, subprocess.CalledProcessError):pass
|
|
self.store.running_message(run["id"], "本次进程收尾未确认，保留运行占位，执行器停止接单")
|
|
self.worker.stopping = True
|
|
return False
|