"""Manage only this console's transient units; reattach after worker restarts.""" import hashlib import json from pathlib import Path import re import subprocess import time from .native_plans import build_plan, public_plan from .store import now def unit_name(identity): if not re.fullmatch(r"[a-f0-9]{32}", identity):raise ValueError("运行编号不合法") return "adamwen-native-" + identity + ".service" def inspect(identity): unit = unit_name(identity) result = subprocess.run(["/usr/bin/systemctl", "show", unit, "--property=Id,LoadState,ActiveState,SubState,Description,Result,ExecMainStatus,MainPID,TasksCurrent"], capture_output=True, timeout=8, text=True) values = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line) if values.get("LoadState") == "not-found":return None if result.returncode or not values.get("LoadState"): raise OSError("systemd 状态查询失败，任务状态未知") if values.get("Id") != unit or values.get("Description") != "XAS native " + identity: raise ValueError("原环境服务归属核验失败") return values def stop_unit(identity): if inspect(identity): subprocess.run(["/usr/bin/systemctl", "--no-block", "stop", unit_name(identity)], check=True, capture_output=True, timeout=8) class NativeExecutor: def __init__(self, worker): self.worker, self.store, self.settings = worker, worker.store, worker.settings def execute(self, run): if self.store.run(run["id"])["cancel_requested"]: self.store.finish(run["id"], "cancelled", message="启动前已取消，原脚本未执行") return directory = self.settings.runs / run["id"] directory.mkdir(mode=0o750, exist_ok=False) try: script = {**run, "id": run["script_id"]} plan = build_plan(self.settings, script, run_id=run["id"]) if plan["plan_token"] != run["execution"]["plan_token"] or not plan["ready"]: raise ValueError("执行前原入口或环境已变化，请重新预检；未启动原测试") if plan["impact"]["requires_confirmation"] and not run["execution"].get("impact_acknowledged"): raise ValueError("缺少本次运行影响确认") if inspect(run["id"]):raise ValueError("本次原环境服务已存在，拒绝重复启动") run = self.store.update_execution(run["id"], {**public_plan(plan), "unit": unit_name(run["id"])}) project = Path(self.worker.config["project"]) log = directory / "console.log" log.touch(mode=0o600) with (directory / "native-permit").open("x") as permit: permit.write(run["execution"]["plan_token"]) command = ["/usr/bin/systemd-run", "--unit=" + unit_name(run["id"]), "--description=XAS native " + run["id"], "--service-type=exec", "--property=RemainAfterExit=yes", "--property=KillMode=control-group", "--property=TimeoutStopSec=120", "--property=RuntimeMaxSec=" + str(plan["timeout_seconds"]), "--property=WorkingDirectory=" + str(project), "--property=User=root", "--property=UMask=0022", "--property=StandardOutput=append:" + str(log), "--property=StandardError=append:" + str(log), "--setenv=XAS_CONSOLE_ROOT=" + str(self.settings.root), "--setenv=PYTHONDONTWRITEBYTECODE=1", "--", "/usr/bin/python3", "-m", "console.native_job", run["id"]] if self.store.run(run["id"])["cancel_requested"]: (directory / "native-permit").unlink(missing_ok=True) self.store.finish(run["id"], "cancelled", message="启动前已取消，原脚本未执行") return result = subprocess.run(command, capture_output=True, timeout=15, text=True) if result.returncode: raise ValueError("systemd 未接受原环境任务，原脚本未启动") except Exception as exc: message = str(exc) if isinstance(exc, ValueError) else "原环境启动失败，未自动重试" # A timeout can leave a submitted unit; inspect it instead of submitting again. if inspect(run["id"]): return self.monitor(run) (directory / "native-permit").unlink(missing_ok=True) if (directory / "native-started").exists(): self.worker.stopping = True self.store.finish(run["id"], "interrupted", message="原入口可能已启动但服务状态未知，执行器停止接单；请核对后再操作") return (directory / "console.log").write_text(message + "\n", encoding="utf-8") self.store.finish(run["id"], "failed", message=message) return self.monitor(run) def monitor(self, run): requested = False failures = 0 while not self.worker.stopping: self.store.heartbeat() try: state = inspect(run["id"]) failures = 0 except (OSError, ValueError, subprocess.TimeoutExpired): failures += 1 self.store.running_message(run["id"], "无法确认原环境服务状态，保留运行占位，不启动下一项") if failures >= 10:self.worker.stopping = True time.sleep(1) continue if state is None or state.get("SubState") in {"exited", "dead", "failed"}: return self.finish(run, state) if self.store.run(run["id"])["cancel_requested"] and not requested: stop_unit(run["id"]) requested = True time.sleep(.5) # The independent unit keeps running. The next worker attaches without replaying it. def finish(self, run, unit): directory = self.settings.runs / run["id"] directory.mkdir(mode=0o750, exist_ok=True) outcome_file = directory / "unit-outcome.json" outcome = unit if outcome_file.exists():outcome = json.loads(outcome_file.read_text()) elif unit:outcome_file.write_text(json.dumps(unit)) result_file = directory / "native-result.json" result = json.loads(result_file.read_text()) if result_file.is_file() else {} current = self.store.run(run["id"]) code = result.get("exit_code") if current["cancel_requested"]:state, message = "cancelled", "已取消本次宿主入口；请核对原脚本启动的 Docker/tmux 任务和模型恢复情况" elif outcome and outcome.get("Result") == "timeout":state, message = "timed_out", "原环境运行超过时限；原服务是否恢复需结合收尾记录核对" elif not result:state, message = "interrupted", "原环境任务结束但完整结果未取得；未自动重跑，请查看日志和原服务状态" elif result.get("stopped"):state, message = "interrupted", "原环境入口收到外部停止信号，服务恢复情况未确认" elif code == 0:state, message = "succeeded", "原入口正常退出；业务结论和共享资源状态请查看报告" else:state, message = "failed", f"原入口退出码 {code}，请查看日志" sessions = result.get("runtime_changes", {}).get("new_tmux_sessions", []) if sessions and state == "succeeded": state, message = "interrupted", "原入口已退出，但检测到新增 tmux 会话；后台任务完成状态未确认，请核对原工具" artifacts = result.get("artifacts", []) summary = {"run_id": run["id"], "state": state, "exit_code": code, "message": message, "finished_at": now(), "source_sha256": run["sha256"], "execution": run["execution"], "runtime_changes": result.get("runtime_changes"), "archive_notices": result.get("archive_notices", []), "log_truncated": result.get("log_truncated", False)} directory.mkdir(mode=0o750, exist_ok=True) (directory / "artifacts").mkdir(exist_ok=True) data = json.dumps(summary, ensure_ascii=False, indent=2).encode() identity = hashlib.sha256((str(directory) + "/summary.json").encode()).hexdigest()[:24] (directory / "artifacts" / identity).write_bytes(data) artifacts.append({"id": identity, "name": "summary.json", "kind": "json", "size": len(data)}) if unit: if not self.wait_stopped(run):return subprocess.run(["/usr/bin/systemctl", "reset-failed", unit_name(run["id"])], capture_output=True, timeout=8) self.store.finish(run["id"], state, message=message, exit_code=code, artifacts=artifacts) def wait_stopped(self, run): try: stop_unit(run["id"]) deadline = time.monotonic() + 130 while time.monotonic() < deadline and not self.worker.stopping: self.store.heartbeat() current = inspect(run["id"]) if current is None:return True if current.get("ActiveState") in {"inactive", "failed"} and current.get("MainPID", "0") == "0" and current.get("TasksCurrent", "0") in {"0", "[not set]"}: return True time.sleep(.5) except (OSError, ValueError, subprocess.TimeoutExpired, subprocess.CalledProcessError):pass self.store.running_message(run["id"], "本次进程收尾未确认，保留运行占位，执行器停止接单") self.worker.stopping = True return False