470 lines
16 KiB
Go
470 lines
16 KiB
Go
package server
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"github.com/gin-gonic/gin"
|
||
"github.com/jmoiron/sqlx"
|
||
"math"
|
||
"strings"
|
||
"time"
|
||
"vdi-cloud/platform/internal/model"
|
||
"vdi-cloud/platform/internal/store"
|
||
"vdi-cloud/platform/pkg/proxmox"
|
||
)
|
||
|
||
type hardwareInput struct {
|
||
Kind string `json:"kind" binding:"required,oneof=compute disk"`
|
||
Cores int `json:"cores"`
|
||
Sockets int `json:"sockets"`
|
||
MemoryMiB int `json:"memoryMiB"`
|
||
Disk string `json:"disk" binding:"max=20"`
|
||
SizeGiB int `json:"sizeGiB"`
|
||
Digest string `json:"digest" binding:"required,max=80"`
|
||
}
|
||
|
||
func (s *Server) loadHardware(ctx context.Context, id string, db sqlx.ExtContext) (model.Desktop, proxmox.VMHardware, map[string]any, error) {
|
||
var d model.Desktop
|
||
if err := sqlx.GetContext(ctx, db, &d, "SELECT * FROM desktops WHERE id=$1", id); err != nil {
|
||
return d, proxmox.VMHardware{}, nil, err
|
||
}
|
||
pve := s.currentPVE()
|
||
if pve == nil || d.Provider != "proxmox" {
|
||
return d, proxmox.VMHardware{}, nil, errors.New("仅支持已接入 Proxmox 的真实桌面")
|
||
}
|
||
h, config, err := pve.Hardware(ctx, d.Node, d.VMID)
|
||
return d, h, config, err
|
||
}
|
||
func (s *Server) desktopHardware(c *gin.Context) {
|
||
_, h, _, err := s.loadHardware(c.Request.Context(), c.Param("id"), s.DB)
|
||
if err != nil {
|
||
s.fail(c, 502, "hardware_unavailable", "无法读取真实规格，请检查 PVE 的 VM.Audit 权限和连接")
|
||
return
|
||
}
|
||
if h.PowerState != "stopped" {
|
||
h.Blockers = append(h.Blockers, "请先正常关闭桌面，再修改规格")
|
||
}
|
||
unresolved, err := maintenanceBlocked(c.Request.Context(), s.DB, c.Param("id"), "", true)
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if unresolved {
|
||
h.Blockers = append(h.Blockers, "有桌面维护任务正在执行或结果未确认，请先核对任务")
|
||
}
|
||
c.JSON(200, h)
|
||
}
|
||
func (s *Server) hardwarePreflight(ctx context.Context, id string, in hardwareInput, db sqlx.ExtContext, ownTask string) (model.Desktop, proxmox.VMHardware, map[string]any, error) {
|
||
d, h, config, err := s.loadHardware(ctx, id, db)
|
||
if err != nil {
|
||
return d, h, config, errors.New("无法读取 PVE 规格")
|
||
}
|
||
reject := func(message string) (model.Desktop, proxmox.VMHardware, map[string]any, error) {
|
||
return d, h, config, errors.New(message)
|
||
}
|
||
if h.PowerState != "stopped" {
|
||
return reject("桌面仍在运行，请先正常关机")
|
||
}
|
||
if len(h.Blockers) > 0 {
|
||
return reject(strings.Join(h.Blockers, "；"))
|
||
}
|
||
if h.Digest != in.Digest {
|
||
return reject("PVE 配置已变化，请重新读取规格")
|
||
}
|
||
var busy bool
|
||
busy, err = maintenanceBlocked(ctx, db, id, ownTask, true)
|
||
if err != nil {
|
||
return d, h, config, err
|
||
}
|
||
var connected bool
|
||
if err = sqlx.GetContext(ctx, db, &connected, "SELECT EXISTS(SELECT 1 FROM terminals WHERE session_desktop_id=$1 AND last_seen>now()-interval '90 seconds')", id); err != nil {
|
||
return d, h, config, err
|
||
}
|
||
busy = busy || connected
|
||
if busy {
|
||
return reject("桌面有活动连接、执行中任务或未确认规格变更")
|
||
}
|
||
pve := s.currentPVE()
|
||
permissions := []string{}
|
||
if in.Kind == "compute" {
|
||
if in.Cores < 1 || in.Cores > 256 || in.Sockets < 1 || in.Sockets > 8 || in.Cores*in.Sockets > 1024 || in.MemoryMiB < 512 || in.MemoryMiB > 1048576 || in.MemoryMiB%128 != 0 || in.Disk != "" || in.SizeGiB != 0 {
|
||
return reject("CPU 或内存规格无效；内存按 128 MiB 递增")
|
||
}
|
||
if in.MemoryMiB < h.Balloon {
|
||
return reject("目标内存低于当前 balloon 最小内存，请先在 PVE 调整")
|
||
}
|
||
if in.Cores != h.Cores || in.Sockets != h.Sockets {
|
||
permissions = append(permissions, "VM.Config.CPU")
|
||
}
|
||
if in.MemoryMiB != h.MemoryMiB {
|
||
permissions = append(permissions, "VM.Config.Memory")
|
||
}
|
||
if len(permissions) == 0 {
|
||
return reject("规格没有变化")
|
||
}
|
||
} else if in.Kind == "disk" {
|
||
if in.Cores != 0 || in.Sockets != 0 || in.MemoryMiB != 0 || in.SizeGiB < 1 || in.SizeGiB > 65536 {
|
||
return reject("磁盘目标容量无效")
|
||
}
|
||
var disk *proxmox.VMDisk
|
||
for i := range h.Disks {
|
||
if h.Disks[i].Key == in.Disk {
|
||
disk = &h.Disks[i]
|
||
}
|
||
}
|
||
if disk == nil || float64(in.SizeGiB) <= disk.SizeGiB {
|
||
return reject("只能扩容已识别的虚拟磁盘，不能缩小或保持原容量")
|
||
}
|
||
ok, e := pve.HasPermission(ctx, "/storage/"+disk.Storage, "Datastore.AllocateSpace")
|
||
if e != nil || !ok {
|
||
return reject("缺少磁盘存储分配权限")
|
||
}
|
||
storages, e := pve.Storages(ctx, d.Node)
|
||
if e != nil {
|
||
return reject("无法核查节点存储余量")
|
||
}
|
||
available := false
|
||
for _, st := range storages {
|
||
if st.Name == disk.Storage && st.Active == 1 && st.Enabled == 1 && st.Available >= (float64(in.SizeGiB)-disk.SizeGiB+1)*(1<<30) {
|
||
available = true
|
||
}
|
||
}
|
||
if !available {
|
||
return reject("存储不足或不可用；需扩容量加 1 GiB 余量")
|
||
}
|
||
permissions = append(permissions, "VM.Config.Disk")
|
||
} else {
|
||
return reject("规格操作类型无效")
|
||
}
|
||
for _, permission := range permissions {
|
||
ok, e := pve.HasPermission(ctx, fmt.Sprintf("/vms/%d", d.VMID), permission)
|
||
if e != nil || !ok {
|
||
return reject("缺少 " + permission + " 权限")
|
||
}
|
||
}
|
||
return d, h, config, nil
|
||
}
|
||
func (s *Server) checkHardware(c *gin.Context) {
|
||
var in hardwareInput
|
||
if !s.decodeConfiguration(c, &in) {
|
||
return
|
||
}
|
||
_, _, _, err := s.hardwarePreflight(c.Request.Context(), c.Param("id"), in, s.DB, "")
|
||
if err != nil {
|
||
s.fail(c, 409, "hardware_not_ready", err.Error())
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"available": true, "message": "规格、状态、权限与存储预检通过"})
|
||
}
|
||
func (s *Server) queueHardware(c *gin.Context) {
|
||
var in hardwareInput
|
||
if !s.decodeConfiguration(c, &in) {
|
||
return
|
||
}
|
||
key := strings.TrimSpace(c.GetHeader("Idempotency-Key"))
|
||
if key == "" || len(key) > 100 {
|
||
s.fail(c, 400, "idempotency_required", "请携带有效请求标识")
|
||
return
|
||
}
|
||
tx, actor := s.accountTransaction(c, true)
|
||
if tx == nil {
|
||
return
|
||
}
|
||
defer tx.Rollback()
|
||
ctx := c.Request.Context()
|
||
if !s.lockAuthorization(c, tx) {
|
||
return
|
||
}
|
||
raw, _ := json.Marshal(in)
|
||
hash := store.Hash(string(raw))
|
||
var old struct {
|
||
TaskID string `db:"task_id"`
|
||
Hash string `db:"request_hash"`
|
||
}
|
||
err := tx.GetContext(ctx, &old, "SELECT h.task_id,h.request_hash FROM hardware_jobs h JOIN tasks t ON t.id=h.task_id WHERE t.actor=$1 AND t.idempotency_key=$2", actor.Username, key)
|
||
if err == nil {
|
||
var task model.Task
|
||
if old.Hash != hash {
|
||
s.fail(c, 409, "idempotency_conflict", "请求标识已用于其他规格参数")
|
||
return
|
||
}
|
||
if e := tx.GetContext(ctx, &task, "SELECT * FROM tasks WHERE id=$1 AND target_id=$2", old.TaskID, c.Param("id")); e != nil {
|
||
s.fail(c, 409, "idempotency_conflict", "请求标识已用于其他桌面")
|
||
return
|
||
}
|
||
c.JSON(202, task)
|
||
return
|
||
}
|
||
if !errors.Is(err, sql.ErrNoRows) {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
// A target row lock serializes specifications against desktop maintenance.
|
||
var target string
|
||
if err = tx.GetContext(ctx, &target, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", c.Param("id")); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
d, _, _, err := s.hardwarePreflight(ctx, target, in, tx, "")
|
||
if err != nil {
|
||
s.fail(c, 409, "hardware_not_ready", err.Error())
|
||
return
|
||
}
|
||
id := store.ID("task_")
|
||
action := "configure-hardware"
|
||
if in.Kind == "disk" {
|
||
action = "expand-disk"
|
||
}
|
||
var task model.Task
|
||
err = tx.GetContext(ctx, &task, "INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,idempotency_key) VALUES($1,'hardware',$2,$3,$4,$5,'queued','已提交，执行前会再次预检',$6) RETURNING *", id, action, d.ID, d.Name, actor.Username, key)
|
||
if conflict(err) {
|
||
s.fail(c, 409, "task_conflict", "该桌面有其他任务或请求标识已使用")
|
||
return
|
||
}
|
||
if err == nil {
|
||
_, err = tx.ExecContext(ctx, "INSERT INTO hardware_jobs(task_id,request_json,request_hash) VALUES($1,$2,$3)", id, string(raw), hash)
|
||
}
|
||
if err == nil {
|
||
_, err = tx.ExecContext(ctx, "INSERT INTO audit_logs(id,actor,action,target,result,request_id) VALUES($1,$2,$3,$4,'submitted',$5)", store.ID("audit_"), actor.Username, action, d.ID, c.GetString("requestId"))
|
||
}
|
||
if err == nil {
|
||
err = tx.Commit()
|
||
}
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(202, task)
|
||
}
|
||
func hardwareMatches(h proxmox.VMHardware, in hardwareInput) bool {
|
||
if len(h.Blockers) > 0 {
|
||
return false
|
||
}
|
||
if in.Kind == "compute" {
|
||
return h.Cores == in.Cores && h.Sockets == in.Sockets && h.MemoryMiB == in.MemoryMiB
|
||
}
|
||
for _, d := range h.Disks {
|
||
if d.Key == in.Disk {
|
||
return math.Abs(d.SizeGiB-float64(in.SizeGiB)) < 0.001
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
func (s *Server) executeHardware(parent context.Context, task model.Task) {
|
||
ctx, cancel := context.WithTimeout(parent, 30*time.Minute)
|
||
defer cancel()
|
||
var job struct {
|
||
Raw string `db:"request_json"`
|
||
Stage string `db:"stage"`
|
||
}
|
||
err := s.DB.GetContext(ctx, &job, "SELECT request_json,stage FROM hardware_jobs WHERE task_id=$1", task.ID)
|
||
var in hardwareInput
|
||
if err == nil {
|
||
err = json.Unmarshal([]byte(job.Raw), &in)
|
||
}
|
||
submitted := job.Stage != "prepared"
|
||
pve := s.currentPVE()
|
||
if e := s.RefreshCompute(ctx); err == nil && e != nil {
|
||
err = e
|
||
}
|
||
pve = s.currentPVE()
|
||
var d model.Desktop
|
||
if err == nil {
|
||
err = s.DB.GetContext(ctx, &d, "SELECT * FROM desktops WHERE id=$1", task.TargetID)
|
||
}
|
||
if err == nil && pve == nil {
|
||
err = errors.New("计算平台未配置")
|
||
}
|
||
if err == nil && !submitted {
|
||
var config map[string]any
|
||
d, _, config, err = s.hardwarePreflight(ctx, d.ID, in, s.DB, task.ID)
|
||
if err == nil {
|
||
var claimed bool
|
||
claimed, err = s.claimHardware(ctx, task)
|
||
if err == nil && !claimed {
|
||
return
|
||
}
|
||
|
||
if err == nil {
|
||
submitted = true
|
||
if in.Kind == "compute" {
|
||
task.ProviderRef, err = pve.ConfigureHardware(ctx, d.Node, d.VMID, in.Cores, in.Sockets, in.MemoryMiB, in.Digest, config)
|
||
} else {
|
||
task.ProviderRef, err = pve.ResizeDisk(ctx, d.Node, d.VMID, in.Disk, in.SizeGiB, in.Digest)
|
||
}
|
||
var rejected *proxmox.HTTPError
|
||
if errors.As(err, &rejected) && rejected.Status >= 400 && rejected.Status < 500 {
|
||
submitted = false
|
||
}
|
||
if err == nil {
|
||
_, err = s.DB.ExecContext(ctx, "UPDATE tasks SET provider_ref=$1,updated_at=now() WHERE id=$2", task.ProviderRef, task.ID)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if err == nil && submitted && task.ProviderRef == "" {
|
||
err = errors.New("原 PVE 任务标识缺失，需人工核查")
|
||
}
|
||
if err == nil && task.ProviderRef != "" {
|
||
err = pve.Wait(ctx, d.Node, task.ProviderRef, time.Second)
|
||
}
|
||
if err == nil {
|
||
var h proxmox.VMHardware
|
||
h, _, err = pve.Hardware(ctx, d.Node, d.VMID)
|
||
if err == nil && !hardwareMatches(h, in) {
|
||
err = errors.New("实际规格尚未达到目标，请核对 PVE")
|
||
}
|
||
}
|
||
status, stage, message := "success", "completed", "PVE 任务与实际规格已核对完成；磁盘扩容后需在客体内扩展分区和文件系统"
|
||
if in.Kind == "compute" {
|
||
message = "PVE 任务与实际 CPU/内存规格已核对完成"
|
||
}
|
||
if err != nil {
|
||
status, stage, message = "failed", "failed", "规格变更未提交，请检查关机状态、权限、容量与配置是否变化"
|
||
if submitted {
|
||
status, stage, message = "timeout", "uncertain", "规格变更结果未确认，已阻止后续修改；请核对原任务，不会自动重发"
|
||
}
|
||
}
|
||
s.finishHardware(task, status, stage, message)
|
||
}
|
||
|
||
// Lock the desktop before changing job/task state, matching power submissions.
|
||
func (s *Server) claimHardware(ctx context.Context, task model.Task) (bool, error) {
|
||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
defer tx.Rollback()
|
||
var id string
|
||
if err = tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", task.TargetID); err != nil {
|
||
return false, err
|
||
}
|
||
result, err := tx.ExecContext(ctx, `UPDATE hardware_jobs SET stage='submitting' WHERE task_id=$1 AND stage='prepared' AND EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND status IN ('queued','running'))`, task.ID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
n, _ := result.RowsAffected()
|
||
return n == 1, tx.Commit()
|
||
}
|
||
func (s *Server) finishHardware(task model.Task, status, stage, message string) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer tx.Rollback()
|
||
var id, current, jobStage string
|
||
if err = tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", task.TargetID); err != nil {
|
||
return
|
||
}
|
||
if err = tx.GetContext(ctx, ¤t, "SELECT status FROM tasks WHERE id=$1 FOR UPDATE", task.ID); err != nil || (current != "queued" && current != "running") {
|
||
return
|
||
}
|
||
if err = tx.GetContext(ctx, &jobStage, "SELECT stage FROM hardware_jobs WHERE task_id=$1 FOR UPDATE", task.ID); err != nil {
|
||
return
|
||
}
|
||
if stage == "expired" {
|
||
status, stage, message = "failed", "failed", "规格任务在提交前过期，未发送修改"
|
||
if jobStage != "prepared" {
|
||
status, stage, message = "timeout", "uncertain", "规格变更结果未确认；请核对原任务，不会自动重发"
|
||
}
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "UPDATE hardware_jobs SET stage=$1 WHERE task_id=$2", stage, task.ID); err == nil {
|
||
err = s.completeTx(ctx, tx, task, status, message)
|
||
}
|
||
if err == nil {
|
||
_ = tx.Commit()
|
||
}
|
||
}
|
||
|
||
func (s *Server) reconcileHardware(c *gin.Context) {
|
||
ctx := c.Request.Context()
|
||
tx, actor := s.accountTransaction(c, true)
|
||
if tx == nil {
|
||
return
|
||
}
|
||
defer tx.Rollback()
|
||
var task model.Task
|
||
var in hardwareInput
|
||
var raw string
|
||
if err := tx.GetContext(ctx, &task, "SELECT * FROM tasks WHERE id=$1 AND kind='hardware'", c.Param("id")); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
var locked string
|
||
if err := tx.GetContext(ctx, &locked, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", task.TargetID); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if err := tx.GetContext(ctx, &task, "SELECT * FROM tasks WHERE id=$1 FOR UPDATE", task.ID); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if task.Status != "timeout" {
|
||
s.fail(c, 409, "reconcile_unavailable", "仅可核对结果未确认的规格任务")
|
||
return
|
||
}
|
||
if err := tx.GetContext(ctx, &raw, "SELECT request_json FROM hardware_jobs WHERE task_id=$1", task.ID); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if json.Unmarshal([]byte(raw), &in) != nil {
|
||
s.fail(c, 500, "invalid_job", "任务记录无法读取")
|
||
return
|
||
}
|
||
d, h, _, err := s.loadHardware(ctx, task.TargetID, tx)
|
||
if err != nil {
|
||
s.fail(c, 502, "hardware_unavailable", "无法读取实际规格")
|
||
return
|
||
}
|
||
if task.ProviderRef == "" {
|
||
s.fail(c, 409, "hardware_unconfirmed", "原任务标识缺失，请在 PVE 核查并由管理员离线恢复；未重新发送修改")
|
||
return
|
||
}
|
||
if task.ProviderRef != "" {
|
||
status, e := s.currentPVE().Task(ctx, d.Node, task.ProviderRef)
|
||
if e != nil || status.Status != "stopped" {
|
||
s.fail(c, 409, "hardware_unconfirmed", "原 PVE 任务仍在执行或无法查询")
|
||
return
|
||
}
|
||
}
|
||
if !hardwareMatches(h, in) {
|
||
s.fail(c, 409, "hardware_unconfirmed", "实际规格未达到目标，请在 PVE 核对；未重新提交修改")
|
||
return
|
||
}
|
||
_, err = tx.ExecContext(ctx, "UPDATE hardware_jobs SET stage='completed' WHERE task_id=$1 AND stage='uncertain'", task.ID)
|
||
if err == nil {
|
||
_, err = tx.ExecContext(ctx, "UPDATE tasks SET status='success',message='只读核对确认实际规格已达到目标，未重新提交修改',updated_at=now() WHERE id=$1 AND status='timeout'", task.ID)
|
||
}
|
||
if err == nil {
|
||
err = accountAudit(ctx, tx, actor.Username, "reconcile_hardware", task.ID, c.GetString("requestId"))
|
||
}
|
||
if err == nil {
|
||
err = tx.Commit()
|
||
}
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"accepted": true})
|
||
}
|
||
|
||
// Tickets cannot connect to a desktop while specifications are changing or unknown.
|
||
func (s *Server) hardwareAllowsConnection(c *gin.Context, id string) bool {
|
||
var blocked bool
|
||
err := deviceTx(c).GetContext(c.Request.Context(), &blocked, `SELECT EXISTS(SELECT 1 FROM tasks WHERE target_id=$1 AND kind IN ('hardware','snapshot','batch-desktop') AND status IN ('queued','running')) OR EXISTS(SELECT 1 FROM hardware_jobs h JOIN tasks t ON t.id=h.task_id WHERE t.target_id=$1 AND h.stage IN ('submitting','uncertain')) OR EXISTS(SELECT 1 FROM snapshot_jobs h JOIN tasks t ON t.id=h.task_id WHERE t.target_id=$1 AND h.stage IN ('submitting','uncertain')) OR EXISTS(SELECT 1 FROM power_batch_items h JOIN tasks t ON t.id=h.task_id WHERE t.target_id=$1 AND h.stage IN ('submitting','uncertain'))`, id)
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return false
|
||
}
|
||
if blocked {
|
||
s.fail(c, 409, "hardware_unconfirmed", "桌面维护中或结果未确认，请联系管理员核对")
|
||
return false
|
||
}
|
||
return true
|
||
}
|