feat: adding background jobs, Dockerfile

Dockerfile

@@ -0,0 +1,38 @@
+#################################
+# Builder: Go + Node in one
+#################################
+FROM golang:1.25.3-alpine AS builder
+
+RUN apk add --no-cache \
+      git ca-certificates tzdata \
+      build-base \
+      nodejs npm
+
+RUN npm i -g yarn pnpm
+
+WORKDIR /src
+
+WORKDIR /src
+
+COPY . .
+RUN make clean && make swagger && make -j3 sdk-all && make ui && make build
+
+#################################
+# Runtime
+#################################
+FROM alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412
+
+RUN apk add --no-cache ca-certificates tzdata postgresql17-client \
+ && addgroup -S app && adduser -S app -G app
+
+WORKDIR /app
+COPY --from=builder /src/autoglue /app/autoglue
+
+ENV PORT=8080
+EXPOSE 8080
+USER app
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+  CMD wget -qO- "http://127.0.0.1:${PORT}/api/v1/healthz" || exit 1
+
+ENTRYPOINT ["/app/autoglue"]

cmd/serve.go

@@ -12,10 +12,13 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dyaksa/archer"
 	"github.com/glueops/autoglue/internal/api"
 	"github.com/glueops/autoglue/internal/app"
 	"github.com/glueops/autoglue/internal/auth"
+	"github.com/glueops/autoglue/internal/bg"
 	"github.com/glueops/autoglue/internal/config"
+	"github.com/google/uuid"
 	"github.com/spf13/cobra"
 )
 
@@ -30,6 +33,83 @@ var serveCmd = &cobra.Command{
 			return err
 		}
 
+		jobs, err := bg.NewJobs(rt.DB, cfg.DbURL)
+		if err != nil {
+			log.Fatalf("failed to init background jobs: %v", err)
+		}
+
+		// Start workers in background ONCE
+		go func() {
+			if err := jobs.Start(); err != nil {
+				log.Fatalf("failed to start background jobs: %v", err)
+			}
+		}()
+		defer jobs.Stop()
+
+		// daily cleanups
+		{
+			// schedule next 03:30 local time
+			next := time.Now().Truncate(24 * time.Hour).Add(24 * time.Hour).Add(3*time.Hour + 30*time.Minute)
+			_, _ = jobs.Enqueue(
+				context.Background(),
+				uuid.NewString(),
+				"archer_cleanup",
+				bg.CleanupArgs{RetainDays: 7, Table: "jobs"},
+				archer.WithScheduleTime(next),
+				archer.WithMaxRetries(1),
+			)
+
+			// schedule next 03:45 local time
+			next2 := time.Now().Truncate(24 * time.Hour).Add(24 * time.Hour).Add(3*time.Hour + 45*time.Minute)
+			_, _ = jobs.Enqueue(
+				context.Background(),
+				uuid.NewString(),
+				"tokens_cleanup",
+				bg.TokensCleanupArgs{},
+				archer.WithScheduleTime(next2),
+				archer.WithMaxRetries(1),
+			)
+		}
+
+		// Periodic scheduler
+		schedCtx, schedCancel := context.WithCancel(context.Background())
+		defer schedCancel()
+
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+
+		go func() {
+			for {
+				select {
+				case <-ticker.C:
+					_, err := jobs.Enqueue(
+						context.Background(),
+						uuid.NewString(),
+						"bootstrap_bastion",
+						bg.BastionBootstrapArgs{},
+						archer.WithMaxRetries(3),
+						// while debugging, avoid extra schedule delay:
+						archer.WithScheduleTime(time.Now().Add(10*time.Second)),
+					)
+					if err != nil {
+						log.Printf("failed to enqueue bootstrap_bastion: %v", err)
+					}
+					/*
+						_, _ = jobs.Enqueue(
+							context.Background(),
+							uuid.NewString(),
+							"tokens_cleanup",
+							bg.TokensCleanupArgs{},
+							archer.WithMaxRetries(3),
+							archer.WithScheduleTime(time.Now().Add(10*time.Second)),
+						)
+					*/
+				case <-schedCtx.Done():
+					return
+				}
+			}
+		}()
+
 		_ = auth.Refresh(rt.DB, rt.Cfg.JWTPrivateEncKey)
 		go func() {
 			t := time.NewTicker(60 * time.Second)

docs/swagger.yaml

@@ -1,5 +1,20 @@
 basePath: /api/v1
 definitions:
+  dto.AnnotationResponse:
+    properties:
+      created_at:
+        type: string
+      id:
+        type: string
+      key:
+        type: string
+      organization_id:
+        type: string
+      updated_at:
+        type: string
+      value:
+        type: string
+    type: object
   dto.AuthStartResponse:
     properties:
       auth_url:
@@ -239,6 +254,12 @@ definitions:
       value:
         type: string
     type: object
+  handlers.HealthStatus:
+    properties:
+      status:
+        example: ok
+        type: string
+    type: object
   handlers.createUserKeyRequest:
     properties:
       expires_in_hours:
@@ -495,6 +516,114 @@ paths:
       summary: Get JWKS
       tags:
       - Auth
+  /annotations:
+    get:
+      consumes:
+      - application/json
+      description: 'Returns annotations for the organization in X-Org-ID. Filters:
+        `name`, `value`, and `q` (name contains). Add `include=node_pools` to include
+        linked node pools.'
+      operationId: ListAnnotations
+      parameters:
+      - description: Organization UUID
+        in: header
+        name: X-Org-ID
+        type: string
+      - description: Exact name
+        in: query
+        name: name
+        type: string
+      - description: Exact value
+        in: query
+        name: value
+        type: string
+      - description: name contains (case-insensitive)
+        in: query
+        name: q
+        type: string
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            items:
+              $ref: '#/definitions/dto.AnnotationResponse'
+            type: array
+        "401":
+          description: Unauthorized
+          schema:
+            type: string
+        "403":
+          description: organization required
+          schema:
+            type: string
+        "500":
+          description: failed to list annotations
+          schema:
+            type: string
+      security:
+      - BearerAuth: []
+      - OrgKeyAuth: []
+      - OrgSecretAuth: []
+      summary: List annotations (org scoped)
+      tags:
+      - Annotations
+  /annotations/{id}:
+    get:
+      consumes:
+      - application/json
+      description: Returns one annotation. Add `include=node_pools` to include node
+        pools.
+      operationId: GetAnnotation
+      parameters:
+      - description: Organization UUID
+        in: header
+        name: X-Org-ID
+        type: string
+      - description: Annotation ID (UUID)
+        in: path
+        name: id
+        required: true
+        type: string
+      - description: 'Optional: node_pools'
+        in: query
+        name: include
+        type: string
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/dto.AnnotationResponse'
+        "400":
+          description: invalid id
+          schema:
+            type: string
+        "401":
+          description: Unauthorized
+          schema:
+            type: string
+        "403":
+          description: organization required
+          schema:
+            type: string
+        "404":
+          description: not found
+          schema:
+            type: string
+        "500":
+          description: fetch failed
+          schema:
+            type: string
+      security:
+      - BearerAuth: []
+      - OrgKeyAuth: []
+      - OrgSecretAuth: []
+      summary: Get annotation by ID (org scoped)
+      tags:
+      - Annotations
   /auth/{provider}/callback:
     get:
       operationId: AuthCallback
@@ -576,6 +705,22 @@ paths:
       summary: Rotate refresh token
       tags:
       - Auth
+  /healthz:
+    get:
+      consumes:
+      - application/json
+      description: Returns 200 OK when the service is up
+      operationId: HealthCheck               // operationId
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/handlers.HealthStatus'
+      summary: Basic health check
+      tags:
+      - Health
   /labels:
     get:
       consumes:

go.mod

@@ -5,6 +5,7 @@ go 1.25.3
 require (
 	github.com/alexedwards/argon2id v1.0.0
 	github.com/coreos/go-oidc/v3 v3.16.0
+	github.com/dyaksa/archer v1.1.3
 	github.com/go-chi/chi/v5 v5.2.3
 	github.com/go-chi/cors v1.2.2
 	github.com/go-chi/httprate v0.15.0
@@ -36,6 +37,7 @@ require (
 	github.com/go-openapi/swag/jsonname v0.25.1 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -45,6 +47,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect

go.sum

@@ -1,5 +1,7 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/agiledragon/gomonkey/v2 v2.3.1 h1:k+UnUY0EMNYUFUAQVETGY9uUTxjMdnUkP0ARyJS1zzs=
@@ -14,6 +16,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dyaksa/archer v1.1.3 h1:jfe51tSNzzscFpu+Vilm4SKb0Lnv6FR1yaGspjab4x4=
+github.com/dyaksa/archer v1.1.3/go.mod h1:IYSp67u14JHTNuvvy6gG1eaX2TPywXvfk1FiyZwVEK4=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -46,6 +50,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
 github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
@@ -84,6 +90,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@@ -129,6 +137,8 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
 github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
 github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/api/routes.go

@@ -69,6 +69,8 @@ func NewRouter(db *gorm.DB) http.Handler {
 			// Also serving a versioned JWKS for swagger, which uses BasePath
 			v1.Get("/.well-known/jwks.json", handlers.JWKSHandler)
 
+			v1.Get("/healthz", handlers.HealthCheck)
+
 			v1.Route("/auth", func(a chi.Router) {
 				a.Post("/{provider}/start", handlers.AuthStart(db))
 				a.Get("/{provider}/callback", handlers.AuthCallback(db))

internal/app/runtime.go

@@ -22,6 +22,7 @@ func NewRuntime() *Runtime {
 	d := db.Open(cfg.DbURL)
 
 	err = db.Run(d,
+		&models.Job{},
 		&models.MasterKey{},
 		&models.SigningKey{},
 		&models.User{},

internal/bg/archer_cleanup.go

@@ -0,0 +1,53 @@
+package bg
+
+import (
+	"context"
+	"time"
+
+	"github.com/dyaksa/archer"
+	"github.com/dyaksa/archer/job"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
+type CleanupArgs struct {
+	RetainDays int    `json:"retain_days"`
+	Table      string `json:"table"`
+}
+
+type JobRow struct {
+	ID        string `gorm:"primaryKey"`
+	Status    string
+	UpdatedAt time.Time
+}
+
+func (JobRow) TableName() string { return "jobs" }
+
+func CleanupWorker(gdb *gorm.DB, jobs *Jobs, retainDays int) archer.WorkerFn {
+	return func(ctx context.Context, j job.Job) (any, error) {
+		if err := CleanupJobs(gdb, retainDays); err != nil {
+			return nil, err
+		}
+
+		// schedule tomorrow 03:30
+		next := time.Now().Truncate(24 * time.Hour).Add(24 * time.Hour).Add(3*time.Hour + 30*time.Minute)
+
+		_, _ = jobs.Enqueue(
+			ctx,
+			uuid.NewString(),
+			"archer_cleanup",
+			CleanupArgs{RetainDays: retainDays, Table: "jobs"},
+			archer.WithScheduleTime(next),
+			archer.WithMaxRetries(1),
+		)
+		return nil, nil
+	}
+}
+
+func CleanupJobs(db *gorm.DB, retainDays int) error {
+	cutoff := time.Now().AddDate(0, 0, -retainDays)
+	return db.
+		Where("status IN ?", []string{"success", "failed", "cancelled"}).
+		Where("updated_at < ?", cutoff).
+		Delete(&JobRow{}).Error
+}

internal/bg/bastion.go

@@ -0,0 +1,274 @@
+package bg
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/dyaksa/archer"
+	"github.com/dyaksa/archer/job"
+	"github.com/glueops/autoglue/internal/models"
+	"github.com/glueops/autoglue/internal/utils"
+	"github.com/google/uuid"
+	"golang.org/x/crypto/ssh"
+	"gorm.io/gorm"
+)
+
+// ----- Public types -----
+
+type BastionBootstrapArgs struct{}
+
+type BastionBootstrapFailure struct {
+	ID     uuid.UUID `json:"id"`
+	Step   string    `json:"step"`
+	Reason string    `json:"reason"`
+}
+
+type BastionBootstrapResult struct {
+	Status       string                    `json:"status"`
+	Processed    int                       `json:"processed"`
+	Ready        int                       `json:"ready"`
+	Failed       int                       `json:"failed"`
+	ElapsedMs    int                       `json:"elapsed_ms"`
+	FailedServer []uuid.UUID               `json:"failed_server_ids"`
+	Failures     []BastionBootstrapFailure `json:"failures"`
+}
+
+// ----- Worker -----
+
+func BastionBootstrapWorker(db *gorm.DB) archer.WorkerFn {
+	return func(ctx context.Context, j job.Job) (any, error) {
+		jobID := j.ID
+		start := time.Now()
+
+		var servers []models.Server
+		if err := db.
+			Preload("SshKey").
+			Where("role = ? AND status = ?", "bastion", "pending").
+			Find(&servers).Error; err != nil {
+			log.Printf("[bastion] level=ERROR job=%s step=query msg=%q", jobID, err)
+			return nil, err
+		}
+
+		log.Printf("[bastion] level=INFO job=%s step=start count=%d", jobID, len(servers))
+
+		proc, ok, fail := 0, 0, 0
+		var failedIDs []uuid.UUID
+		var failures []BastionBootstrapFailure
+
+		perHostTimeout := 8 * time.Minute
+
+		for i := range servers {
+			s := &servers[i]
+			hostStart := time.Now()
+			proc++
+
+			// 1) Defensive IP check
+			if s.PublicIPAddress == nil || *s.PublicIPAddress == "" {
+				fail++
+				failedIDs = append(failedIDs, s.ID)
+				failures = append(failures, BastionBootstrapFailure{ID: s.ID, Step: "ip_check", Reason: "missing public ip"})
+				logHostErr(jobID, s, "ip_check", fmt.Errorf("missing public ip"))
+				_ = setServerStatus(db, s.ID, "failed")
+				continue
+			}
+
+			// 2) Move to provisioning
+			if err := setServerStatus(db, s.ID, "provisioning"); err != nil {
+				fail++
+				failedIDs = append(failedIDs, s.ID)
+				failures = append(failures, BastionBootstrapFailure{ID: s.ID, Step: "set_provisioning", Reason: err.Error()})
+				logHostErr(jobID, s, "set_provisioning", err)
+				continue
+			}
+
+			// 3) Decrypt private key for org
+			privKey, err := utils.DecryptForOrg(
+				s.OrganizationID,
+				s.SshKey.EncryptedPrivateKey,
+				s.SshKey.PrivateIV,
+				s.SshKey.PrivateTag,
+				db,
+			)
+			if err != nil {
+				fail++
+				failedIDs = append(failedIDs, s.ID)
+				failures = append(failures, BastionBootstrapFailure{ID: s.ID, Step: "decrypt_key", Reason: err.Error()})
+				logHostErr(jobID, s, "decrypt_key", err)
+				_ = setServerStatus(db, s.ID, "failed")
+				continue
+			}
+
+			// 4) SSH + install docker
+			host := net.JoinHostPort(*s.PublicIPAddress, "22")
+			runCtx, cancel := context.WithTimeout(ctx, perHostTimeout)
+			out, err := sshInstallDockerWithOutput(runCtx, host, s.SSHUser, []byte(privKey))
+			cancel()
+
+			if err != nil {
+				fail++
+				failedIDs = append(failedIDs, s.ID)
+				failures = append(failures, BastionBootstrapFailure{ID: s.ID, Step: "ssh_install", Reason: err.Error()})
+				// include a short tail of output to speed debugging without flooding logs
+				tail := out
+				if len(tail) > 800 {
+					tail = tail[len(tail)-800:]
+				}
+				logHostErr(jobID, s, "ssh_install", fmt.Errorf("%v | tail=%q", err, tail))
+				_ = setServerStatus(db, s.ID, "failed")
+				continue
+			}
+
+			// 5) Mark ready
+			if err := setServerStatus(db, s.ID, "ready"); err != nil {
+				fail++
+				failedIDs = append(failedIDs, s.ID)
+				failures = append(failures, BastionBootstrapFailure{ID: s.ID, Step: "set_ready", Reason: err.Error()})
+				logHostErr(jobID, s, "set_ready", err)
+				_ = setServerStatus(db, s.ID, "failed")
+				continue
+			}
+
+			ok++
+			logHostInfo(jobID, s, "done", "host completed",
+				"elapsed_ms", time.Since(hostStart).Milliseconds())
+		}
+
+		res := BastionBootstrapResult{
+			Status:       "ok",
+			Processed:    proc,
+			Ready:        ok,
+			Failed:       fail,
+			ElapsedMs:    int(time.Since(start).Milliseconds()),
+			FailedServer: failedIDs,
+			Failures:     failures,
+		}
+
+		log.Printf("[bastion] level=INFO job=%s step=finish processed=%d ready=%d failed=%d elapsed_ms=%d",
+			jobID, proc, ok, fail, res.ElapsedMs)
+
+		return res, nil
+	}
+}
+
+// ----- Helpers -----
+
+func setServerStatus(db *gorm.DB, id uuid.UUID, status string) error {
+	return db.Model(&models.Server{}).
+		Where("id = ?", id).
+		Updates(map[string]any{
+			"status":     status,
+			"updated_at": time.Now(),
+		}).Error
+}
+
+// uniform log helpers for consistent, greppable output
+func logHostErr(jobID string, s *models.Server, step string, err error) {
+	ip := ""
+	if s.PublicIPAddress != nil {
+		ip = *s.PublicIPAddress
+	}
+	log.Printf("[bastion] level=ERROR job=%s server_id=%s host=%s step=%s msg=%q",
+		jobID, s.ID, ip, step, err)
+}
+
+func logHostInfo(jobID string, s *models.Server, step, msg string, kv ...any) {
+	ip := ""
+	if s.PublicIPAddress != nil {
+		ip = *s.PublicIPAddress
+	}
+	log.Printf("[bastion] level=INFO job=%s server_id=%s host=%s step=%s %s kv=%v",
+		jobID, s.ID, ip, step, msg, kv)
+}
+
+// ----- SSH & command execution -----
+
+// returns combined stdout/stderr so caller can log it on error
+// returns combined stdout/stderr so caller can log it on error
+func sshInstallDockerWithOutput(ctx context.Context, host, user string, privateKeyPEM []byte) (string, error) {
+	signer, err := ssh.ParsePrivateKey(privateKeyPEM)
+	if err != nil {
+		return "", fmt.Errorf("parse private key: %w", err)
+	}
+
+	config := &ssh.ClientConfig{
+		User:            user,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // TODO: known_hosts verification
+		Timeout:         30 * time.Second,
+	}
+
+	// context-aware dial
+	dialer := &net.Dialer{}
+	conn, err := dialer.DialContext(ctx, "tcp", host)
+	if err != nil {
+		return "", fmt.Errorf("dial: %w", err)
+	}
+	defer conn.Close()
+
+	c, chans, reqs, err := ssh.NewClientConn(conn, host, config)
+	if err != nil {
+		return "", fmt.Errorf("ssh handshake: %w", err)
+	}
+	client := ssh.NewClient(c, chans, reqs)
+	defer client.Close()
+
+	sess, err := client.NewSession()
+	if err != nil {
+		return "", fmt.Errorf("session: %w", err)
+	}
+	defer sess.Close()
+
+	// --- script to run remotely (no extra quoting) ---
+	script := `
+set -euxo pipefail
+
+if ! command -v docker >/dev/null 2>&1; then
+  curl -fsSL https://get.docker.com | sh
+fi
+
+# try to enable/start (handles distros with systemd)
+if command -v systemctl >/dev/null 2>&1; then
+  sudo systemctl enable --now docker || true
+fi
+
+# add current ssh user to docker group if exists
+if getent group docker >/dev/null 2>&1; then
+  sudo usermod -aG docker "$(id -un)" || true
+fi
+`
+
+	// Send script via stdin to avoid quoting/escaping issues
+	sess.Stdin = strings.NewReader(script)
+
+	// Capture combined stdout+stderr
+	out, runErr := sess.CombinedOutput("bash -s")
+	return string(out), wrapSSHError(runErr, string(out))
+}
+
+// annotate common SSH/remote failure modes to speed triage
+func wrapSSHError(err error, output string) error {
+	if err == nil {
+		return nil
+	}
+	switch {
+	case strings.Contains(output, "Could not resolve host"):
+		return fmt.Errorf("remote run: name resolution failed: %w", err)
+	case strings.Contains(output, "Permission denied"):
+		return fmt.Errorf("remote run: permission denied (check user/key/authorized_keys): %w", err)
+	case strings.Contains(output, "apt-get"):
+		return fmt.Errorf("remote run: apt failed: %w", err)
+	case strings.Contains(output, "yum"):
+		return fmt.Errorf("remote run: yum failed: %w", err)
+	default:
+		return fmt.Errorf("remote run: %w", err)
+	}
+}
+
+// super simple escaping for a here-string; avoids quoting hell
+func sshEscape(s string) string {
+	return fmt.Sprintf("%q", s)
+}

internal/bg/bg.go

@@ -0,0 +1,105 @@
+package bg
+
+import (
+	"context"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/dyaksa/archer"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+	"gorm.io/gorm"
+)
+
+type Jobs struct{ Client *archer.Client }
+
+func archerOptionsFromDSN(dsn string) (*archer.Options, error) {
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	var user, pass string
+	if u.User != nil {
+		user = u.User.Username()
+		pass, _ = u.User.Password()
+	}
+
+	host := u.Host
+	if !strings.Contains(host, ":") {
+		host = net.JoinHostPort(host, "5432")
+	}
+
+	return &archer.Options{
+		Addr:     host,
+		User:     user,
+		Password: pass,
+		DBName:   strings.TrimPrefix(u.Path, "/"),
+		SSL:      u.Query().Get("sslmode"), // forward sslmode
+	}, nil
+}
+
+func NewJobs(gdb *gorm.DB, dbUrl string) (*Jobs, error) {
+	opts, err := archerOptionsFromDSN(dbUrl)
+	if err != nil {
+		return nil, err
+	}
+
+	instances := viper.GetInt("archer.instances")
+	if instances <= 0 {
+		instances = 1
+	}
+
+	timeoutSec := viper.GetInt("archer.timeoutSec")
+	if timeoutSec <= 0 {
+		timeoutSec = 60
+	}
+
+	retainDays := viper.GetInt("archer.cleanup_retain_days")
+	if retainDays <= 0 {
+		retainDays = 7
+	}
+
+	c := archer.NewClient(
+		opts,
+		archer.WithSetTableName("jobs"), // <- ensure correct table
+		archer.WithSleepInterval(1*time.Second), // fast poll while debugging
+		archer.WithErrHandler(func(err error) { // bubble up worker SQL errors
+			log.Printf("[archer] ERROR: %v", err)
+		}),
+	)
+
+	jobs := &Jobs{Client: c}
+
+	c.Register(
+		"bootstrap_bastion",
+		BastionBootstrapWorker(gdb),
+		archer.WithInstances(instances),
+		archer.WithTimeout(time.Duration(timeoutSec)*time.Second),
+	)
+
+	c.Register(
+		"archer_cleanup",
+		CleanupWorker(gdb, jobs, retainDays),
+		archer.WithInstances(1),
+		archer.WithTimeout(5*time.Minute),
+	)
+
+	c.Register(
+		"tokens_cleanup",
+		TokensCleanupWorker(gdb, jobs),
+		archer.WithInstances(1),
+		archer.WithTimeout(5*time.Minute),
+	)
+
+	return jobs, nil
+}
+
+func (j *Jobs) Start() error { return j.Client.Start() }
+func (j *Jobs) Stop()        { j.Client.Stop() }
+
+func (j *Jobs) Enqueue(ctx context.Context, id, queue string, args any, opts ...archer.FnOptions) (any, error) {
+	return j.Client.Schedule(ctx, id, queue, args, opts...)
+}

internal/bg/tokens_cleanup.go

@@ -0,0 +1,51 @@
+package bg
+
+import (
+	"context"
+	"time"
+
+	"github.com/dyaksa/archer"
+	"github.com/dyaksa/archer/job"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
+type RefreshTokenRow struct {
+	ID        string `gorm:"primaryKey"`
+	RevokedAt *time.Time
+	ExpiresAt time.Time
+	UpdatedAt time.Time
+}
+
+func (RefreshTokenRow) TableName() string { return "refresh_tokens" }
+
+type TokensCleanupArgs struct {
+	// kept in case you want to change retention or add dry-run later
+}
+
+func TokensCleanupWorker(db *gorm.DB, jobs *Jobs) archer.WorkerFn {
+	return func(ctx context.Context, j job.Job) (any, error) {
+		if err := CleanupRefreshTokens(db); err != nil {
+			return nil, err
+		}
+
+		// schedule tomorrow 03:45
+		next := time.Now().Truncate(24 * time.Hour).Add(24 * time.Hour).Add(3*time.Hour + 45*time.Minute)
+		_, _ = jobs.Enqueue(
+			ctx,
+			uuid.NewString(),
+			"tokens_cleanup",
+			TokensCleanupArgs{},
+			archer.WithScheduleTime(next),
+			archer.WithMaxRetries(1),
+		)
+		return nil, nil
+	}
+}
+
+func CleanupRefreshTokens(db *gorm.DB) error {
+	now := time.Now()
+	return db.
+		Where("revoked_at IS NOT NULL OR expires_at < ?", now).
+		Delete(&RefreshTokenRow{}).Error
+}

internal/common/audit.go

@@ -7,8 +7,8 @@ import (
 )
 
 type AuditFields struct {
-	ID             uuid.UUID `json:"id"                   gorm:"type:uuid;default:gen_random_uuid()"`
+	ID             uuid.UUID `json:"id"                   gorm:"primaryKey;type:uuid;default:gen_random_uuid()"`
 	OrganizationID uuid.UUID `json:"organization_id"      gorm:"type:uuid;index"`
-	CreatedAt      time.Time `json:"created_at,omitempty" gorm:"column:created_at;not null;default:now()"`
+	CreatedAt      time.Time `json:"created_at,omitempty" gorm:"type:timestamptz;column:created_at;not null;default:now()"`
-	UpdatedAt      time.Time `json:"updated_at,omitempty" gorm:"autoUpdateTime;column:updated_at;not null;default:now()"`
+	UpdatedAt      time.Time `json:"updated_at,omitempty" gorm:"type:timestamptz;autoUpdateTime;column:updated_at;not null;default:now()"`
 }

internal/handlers/annotations.go

@@ -1,5 +1,19 @@
 package handlers
 
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/glueops/autoglue/internal/api/httpmiddleware"
+	"github.com/glueops/autoglue/internal/handlers/dto"
+	"github.com/glueops/autoglue/internal/models"
+	"github.com/glueops/autoglue/internal/utils"
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
 // ListAnnotations godoc
 // @ID           ListAnnotations
 // @Summary      List annotations (org scoped)
@@ -11,11 +25,86 @@ package handlers
 // @Param        name query string false "Exact name"
 // @Param        value query string false "Exact value"
 // @Param        q query string false "name contains (case-insensitive)"
-// @Success      200 {array}  annotationResponse
+// @Success      200 {array}  dto.AnnotationResponse
 // @Failure      401 {string} string "Unauthorized"
 // @Failure      403 {string} string "organization required"
 // @Failure      500 {string} string "failed to list annotations"
-// @Router       /api/v1/annotations [get]
+// @Router       /annotations [get]
 // @Security     BearerAuth
 // @Security     OrgKeyAuth
 // @Security     OrgSecretAuth
+func ListAnnotations(db *gorm.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		orgID, ok := httpmiddleware.OrgIDFrom(r.Context())
+		if !ok {
+			utils.WriteError(w, http.StatusForbidden, "org_required", "specify X-Org-ID")
+			return
+		}
+
+		q := db.Where("organization_id = ?", orgID)
+
+		if key := strings.TrimSpace(r.URL.Query().Get("key")); key != "" {
+			q = q.Where(`key = ?`, key)
+		}
+		if val := strings.TrimSpace(r.URL.Query().Get("value")); val != "" {
+			q = q.Where(`value = ?`, val)
+		}
+		if needle := strings.TrimSpace(r.URL.Query().Get("q")); needle != "" {
+			q = q.Where(`key ILIKE ?`, "%"+needle+"%")
+		}
+
+		var out []dto.AnnotationResponse
+		if err := q.Model(&models.Annotation{}).Order("created_at DESC").Scan(&out).Error; err != nil {
+			utils.WriteError(w, http.StatusInternalServerError, "db_error", "db error")
+			return
+		}
+		utils.WriteJSON(w, http.StatusOK, out)
+	}
+}
+
+// GetAnnotation godoc
+// @ID           GetAnnotation
+// @Summary      Get annotation by ID (org scoped)
+// @Description  Returns one annotation. Add `include=node_pools` to include node pools.
+// @Tags         Annotations
+// @Accept       json
+// @Produce      json
+// @Param        X-Org-ID header string false "Organization UUID"
+// @Param        id path string true "Annotation ID (UUID)"
+// @Param        include query string false "Optional: node_pools"
+// @Success      200 {object} dto.AnnotationResponse
+// @Failure      400 {string} string "invalid id"
+// @Failure      401 {string} string "Unauthorized"
+// @Failure      403 {string} string "organization required"
+// @Failure      404 {string} string "not found"
+// @Failure      500 {string} string "fetch failed"
+// @Router       /annotations/{id} [get]
+// @Security     BearerAuth
+// @Security     OrgKeyAuth
+// @Security     OrgSecretAuth
+func GetAnnotation(db *gorm.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		orgID, ok := httpmiddleware.OrgIDFrom(r.Context())
+		if !ok {
+			utils.WriteError(w, http.StatusForbidden, "org_required", "specify X-Org-ID")
+			return
+		}
+
+		id, err := uuid.Parse(chi.URLParam(r, "id"))
+		if err != nil {
+			utils.WriteError(w, http.StatusBadRequest, "bad_request", "bad request")
+			return
+		}
+
+		var out dto.AnnotationResponse
+		if err := db.Model(&models.Annotation{}).Where("id = ? AND organization_id = ?", id, orgID).First(&out).Error; err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				utils.WriteError(w, http.StatusNotFound, "not_found", "not_found")
+				return
+			}
+			utils.WriteError(w, http.StatusInternalServerError, "db_error", "db error")
+			return
+		}
+		utils.WriteJSON(w, http.StatusOK, out)
+	}
+}

internal/handlers/dto/ssh_keys.go

@@ -1,6 +1,8 @@
 package dto
 
-import "github.com/google/uuid"
+import (
+	"github.com/glueops/autoglue/internal/common"
+)
 
 type CreateSSHRequest struct {
 	Name    string  `json:"name"`
@@ -10,13 +12,13 @@ type CreateSSHRequest struct {
 }
 
 type SshResponse struct {
-	ID             uuid.UUID `json:"id"`
+	common.AuditFields
-	OrganizationID uuid.UUID `json:"organization_id"`
+	Name                string `json:"name"`
-	Name           string    `json:"name"`
+	PublicKey           string `json:"public_key"`
-	PublicKey      string    `json:"public_key"`
+	Fingerprint         string `json:"fingerprint"`
-	Fingerprint    string    `json:"fingerprint"`
+	EncryptedPrivateKey string `json:"-"`
-	CreatedAt      string    `json:"created_at,omitempty"`
+	PrivateIV           string `json:"-"`
-	UpdatedAt      string    `json:"updated_at,omitempty"`
+	PrivateTag          string `json:"-"`
 }
 
 type SshRevealResponse struct {

internal/handlers/health.go

@@ -0,0 +1,24 @@
+package handlers
+
+import (
+	"net/http"
+
+	"github.com/glueops/autoglue/internal/utils"
+)
+
+type HealthStatus struct {
+	Status string `json:"status" example:"ok"`
+}
+
+// HealthCheck godoc
+// @Summary      Basic health check
+// @Description  Returns 200 OK when the service is up
+// @Tags         Health
+// @ID           HealthCheck               // operationId
+// @Accept       json
+// @Produce      json
+// @Success      200 {object} HealthStatus
+// @Router       /healthz [get]
+func HealthCheck(w http.ResponseWriter, r *http.Request) {
+	utils.WriteJSON(w, http.StatusOK, HealthStatus{Status: "ok"})
+}

internal/handlers/ssh_keys.go

@@ -13,9 +13,9 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"time"
 
 	"github.com/glueops/autoglue/internal/api/httpmiddleware"
+	"github.com/glueops/autoglue/internal/common"
 	"github.com/glueops/autoglue/internal/handlers/dto"
 	"github.com/glueops/autoglue/internal/models"
 	"github.com/glueops/autoglue/internal/utils"
@@ -49,25 +49,18 @@ func ListPublicSshKeys(db *gorm.DB) http.HandlerFunc {
 			return
 		}
 
-		var rows []models.SshKey
+		var out []dto.SshResponse
-		if err := db.Where("organization_id = ?", orgID).Order("created_at DESC").Find(&rows).Error; err != nil {
+		if err := db.
+			Model(&models.SshKey{}).
+			Where("organization_id = ?", orgID).
+			// avoid selecting encrypted columns here
+			Select("id", "organization_id", "name", "public_key", "fingerprint", "created_at", "updated_at").
+			Order("created_at DESC").
+			Scan(&out).Error; err != nil {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "failed to list ssh keys")
 			return
 		}
 
-		out := make([]dto.SshResponse, 0, len(rows))
-		for _, row := range rows {
-			out = append(out, dto.SshResponse{
-				ID:             row.ID,
-				OrganizationID: row.OrganizationID,
-				Name:           row.Name,
-				PublicKey:      row.PublicKey,
-				Fingerprint:    row.Fingerprint,
-				CreatedAt:      row.CreatedAt.UTC().Format(time.RFC3339),
-				UpdatedAt:      row.UpdatedAt.UTC().Format(time.RFC3339),
-			})
-		}
-
 		utils.WriteJSON(w, http.StatusOK, out)
 	}
 }
@@ -160,7 +153,9 @@ func CreateSSHKey(db *gorm.DB) http.HandlerFunc {
 		fp := ssh.FingerprintSHA256(parsed)
 
 		key := models.SshKey{
-			OrganizationID:      orgID,
+			AuditFields: common.AuditFields{
+				OrganizationID: orgID,
+			},
 			Name:                req.Name,
 			PublicKey:           pubAuth,
 			EncryptedPrivateKey: cipher,
@@ -175,13 +170,10 @@ func CreateSSHKey(db *gorm.DB) http.HandlerFunc {
 		}
 
 		utils.WriteJSON(w, http.StatusCreated, dto.SshResponse{
-			ID:             key.ID,
+			AuditFields: key.AuditFields,
-			OrganizationID: key.OrganizationID,
+			Name:        key.Name,
-			Name:           key.Name,
+			PublicKey:   key.PublicKey,
-			PublicKey:      key.PublicKey,
+			Fingerprint: key.Fingerprint,
-			Fingerprint:    key.Fingerprint,
-			CreatedAt:      key.CreatedAt.UTC().Format(time.RFC3339),
-			UpdatedAt:      key.UpdatedAt.UTC().Format(time.RFC3339),
 		})
 	}
 }
@@ -221,30 +213,47 @@ func GetSSHKey(db *gorm.DB) http.HandlerFunc {
 			return
 		}
 
-		var key models.SshKey
+		reveal := strings.EqualFold(r.URL.Query().Get("reveal"), "true")
-		if err := db.Where("id = ? AND organization_id = ?", id, orgID).First(&key).Error; err != nil {
+
-			if errors.Is(err, gorm.ErrRecordNotFound) {
+		if !reveal {
+			var out dto.SshResponse
+			if err := db.
+				Model(&models.SshKey{}).
+				Where("id = ? AND organization_id = ?", id, orgID).
+				Select("id", "organization_id", "name", "public_key", "fingerprint", "created_at", "updated_at").
+				Limit(1).
+				Scan(&out).Error; err != nil {
+				utils.WriteError(w, http.StatusInternalServerError, "db_error", "failed to get ssh key")
+				return
+			}
+			if out.ID == uuid.Nil {
 				utils.WriteError(w, http.StatusNotFound, "ssh_key_not_found", "ssh key not found")
 				return
 			}
+			utils.WriteJSON(w, http.StatusOK, out)
+			return
+		}
+
+		var secret dto.SshResponse
+		if err := db.
+			Model(&models.SshKey{}).
+			Where("id = ? AND organization_id = ?", id, orgID).
+			// include the encrypted bits too
+			Select("id", "organization_id", "name", "public_key", "fingerprint",
+				"encrypted_private_key", "private_iv", "private_tag",
+				"created_at", "updated_at").
+			Limit(1).
+			Scan(&secret).Error; err != nil {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "failed to get ssh key")
 			return
 		}
 
-		if r.URL.Query().Get("reveal") != "true" {
+		if secret.ID == uuid.Nil {
-			utils.WriteJSON(w, http.StatusOK, dto.SshResponse{
+			utils.WriteError(w, http.StatusNotFound, "ssh_key_not_found", "ssh key not found")
-				ID:             key.ID,
-				OrganizationID: key.OrganizationID,
-				Name:           key.Name,
-				PublicKey:      key.PublicKey,
-				Fingerprint:    key.Fingerprint,
-				CreatedAt:      key.CreatedAt.UTC().Format(time.RFC3339),
-				UpdatedAt:      key.UpdatedAt.UTC().Format(time.RFC3339),
-			})
 			return
 		}
 
-		plain, err := utils.DecryptForOrg(orgID, key.EncryptedPrivateKey, key.PrivateIV, key.PrivateTag, db)
+		plain, err := utils.DecryptForOrg(orgID, secret.EncryptedPrivateKey, secret.PrivateIV, secret.PrivateTag, db)
 		if err != nil {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "failed to decrypt ssh key")
 			return
@@ -252,13 +261,10 @@ func GetSSHKey(db *gorm.DB) http.HandlerFunc {
 
 		utils.WriteJSON(w, http.StatusOK, dto.SshRevealResponse{
 			SshResponse: dto.SshResponse{
-				ID:             key.ID,
+				AuditFields: secret.AuditFields,
-				OrganizationID: key.OrganizationID,
+				Name:        secret.Name,
-				Name:           key.Name,
+				PublicKey:   secret.PublicKey,
-				PublicKey:      key.PublicKey,
+				Fingerprint: secret.Fingerprint,
-				Fingerprint:    key.Fingerprint,
-				CreatedAt:      key.CreatedAt.UTC().Format(time.RFC3339),
-				UpdatedAt:      key.UpdatedAt.UTC().Format(time.RFC3339),
 			},
 			PrivateKey: plain,
 		})
@@ -297,11 +303,16 @@ func DeleteSSHKey(db *gorm.DB) http.HandlerFunc {
 			return
 		}
 
-		if err := db.Where("id = ? AND organization_id = ?", id, orgID).
+		res := db.Where("id = ? AND organization_id = ?", id, orgID).
-			Delete(&models.SshKey{}).Error; err != nil {
+			Delete(&models.SshKey{})
+		if res.Error != nil {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "failed to delete ssh key")
 			return
 		}
+		if res.RowsAffected == 0 {
+			utils.WriteError(w, http.StatusNotFound, "ssh_key_not_found", "ssh key not found")
+			return
+		}
 		w.WriteHeader(http.StatusNoContent)
 	}
 }

internal/handlers/taints.go

@@ -55,24 +55,11 @@ func ListTaints(db *gorm.DB) http.HandlerFunc {
 			q = q.Where(`key ILIKE ?`, "%"+needle+"%")
 		}
 
-		var rows []models.Taint
+		var out []dto.TaintResponse
-		if err := q.Order("created_at DESC").Find(&rows).Error; err != nil {
+		if err := q.Model(&models.Taint{}).Order("created_at DESC").Find(&out).Error; err != nil {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "db error")
 			return
 		}
-
-		out := make([]dto.TaintResponse, 0, len(rows))
-		for _, row := range rows {
-			out = append(out, dto.TaintResponse{
-				ID:             row.ID,
-				Key:            row.Key,
-				Value:          row.Value,
-				Effect:         row.Effect,
-				OrganizationID: row.OrganizationID,
-				CreatedAt:      row.CreatedAt.UTC().Format(time.RFC3339),
-				UpdatedAt:      row.UpdatedAt.UTC().Format(time.RFC3339),
-			})
-		}
 		utils.WriteJSON(w, http.StatusOK, out)
 	}
 }
@@ -109,8 +96,8 @@ func GetTaint(db *gorm.DB) http.HandlerFunc {
 			return
 		}
 
-		var row models.Taint
+		var out dto.TaintResponse
-		if err := db.Where("id = ? AND organization_id = ?", id, orgID).First(&row).Error; err != nil {
+		if err := db.Model(&models.Taint{}).Where("id = ? AND organization_id = ?", id, orgID).First(&out).Error; err != nil {
 			if errors.Is(err, gorm.ErrRecordNotFound) {
 				utils.WriteError(w, http.StatusNotFound, "not_found", "not_found")
 				return
@@ -118,15 +105,7 @@ func GetTaint(db *gorm.DB) http.HandlerFunc {
 			utils.WriteError(w, http.StatusInternalServerError, "db_error", "db error")
 			return
 		}
-		out := dto.TaintResponse{
+
-			ID:             row.ID,
-			Key:            row.Key,
-			Value:          row.Value,
-			Effect:         row.Effect,
-			OrganizationID: row.OrganizationID,
-			CreatedAt:      row.CreatedAt.UTC().Format(time.RFC3339),
-			UpdatedAt:      row.UpdatedAt.UTC().Format(time.RFC3339),
-		}
 		utils.WriteJSON(w, http.StatusOK, out)
 	}
 }

internal/models/api_key.go

@@ -18,6 +18,6 @@ type APIKey struct {
 	Revoked    bool       `gorm:"not null;default:false" json:"revoked"`
 	Prefix     *string    `json:"prefix,omitempty"`
 	LastUsedAt *time.Time `json:"last_used_at,omitempty" format:"date-time"`
-	CreatedAt  time.Time  `gorm:"not null;default:now()" json:"created_at" format:"date-time"`
+	CreatedAt  time.Time  `json:"created_at,omitempty" gorm:"type:timestamptz;column:created_at;not null;default:now()" format:"date-time"`
-	UpdatedAt  time.Time  `gorm:"not null;default:now()" json:"updated_at" format:"date-time"`
+	UpdatedAt  time.Time  `json:"updated_at,omitempty" gorm:"type:timestamptz;autoUpdateTime;column:updated_at;not null;default:now()" format:"date-time"`
 }

internal/models/job.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/datatypes"
+)
+
+type Job struct {
+	ID            string         `gorm:"type:varchar;primaryKey" json:"id"` // no default; supply from app
+	QueueName     string         `gorm:"type:varchar;not null" json:"queue_name"`
+	Status        string         `gorm:"type:varchar;not null" json:"status"`
+	Arguments     datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'"`
+	Result        datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'"`
+	LastError     *string        `gorm:"type:varchar"`
+	RetryCount    int            `gorm:"not null;default:0"`
+	MaxRetry      int            `gorm:"not null;default:0"`
+	RetryInterval int            `gorm:"not null;default:0"`
+	ScheduledAt   time.Time      `gorm:"type:timestamptz;default:now();index"`
+	StartedAt     *time.Time     `gorm:"type:timestamptz;index"`
+	CreatedAt     time.Time      `gorm:"type:timestamptz;column:created_at;not null;default:now()"`
+	UpdatedAt     time.Time      `gorm:"type:timestamptz;autoUpdateTime;column:updated_at;not null;default:now()"`
+}

internal/models/ssh-key.go

@@ -1,14 +1,11 @@
 package models
 
 import (
-	"time"
+	"github.com/glueops/autoglue/internal/common"
-
-	"github.com/google/uuid"
 )
 
 type SshKey struct {
-	ID                  uuid.UUID    `gorm:"type:uuid;primaryKey;default:gen_random_uuid()"`
+	common.AuditFields
-	OrganizationID      uuid.UUID    `gorm:"type:uuid;not null" json:"organization_id"`
 	Organization        Organization `gorm:"foreignKey:OrganizationID;constraint:OnDelete:CASCADE" json:"organization"`
 	Name                string       `gorm:"not null" json:"name"`
 	PublicKey           string       `gorm:"not null"`
@@ -16,6 +13,4 @@ type SshKey struct {
 	PrivateIV           string       `gorm:"not null"`
 	PrivateTag          string       `gorm:"not null"`
 	Fingerprint         string       `gorm:"not null;index" json:"fingerprint"`
-	CreatedAt           time.Time    `gorm:"not null;default:now()" json:"created_at" format:"date-time"`
-	UpdatedAt           time.Time    `gorm:"not null;default:now()" json:"updated_at" format:"date-time"`
 }

sdk/go/.openapi-generator/FILES

@@ -3,7 +3,9 @@
 .travis.yml
 README.md
 api/openapi.yaml
+api_annotations.go
 api_auth.go
+api_health.go
 api_labels.go
 api_me.go
 api_me_api_keys.go
@@ -13,7 +15,9 @@ api_ssh.go
 api_taints.go
 client.go
 configuration.go
+docs/AnnotationsAPI.md
 docs/AuthAPI.md
+docs/DtoAnnotationResponse.md
 docs/DtoAuthStartResponse.md
 docs/DtoCreateLabelRequest.md
 docs/DtoCreateSSHRequest.md
@@ -33,6 +37,7 @@ docs/DtoUpdateLabelRequest.md
 docs/DtoUpdateServerRequest.md
 docs/DtoUpdateTaintRequest.md
 docs/HandlersCreateUserKeyRequest.md
+docs/HandlersHealthStatus.md
 docs/HandlersMeResponse.md
 docs/HandlersMemberOut.md
 docs/HandlersMemberUpsertReq.md
@@ -42,6 +47,7 @@ docs/HandlersOrgKeyCreateResp.md
 docs/HandlersOrgUpdateReq.md
 docs/HandlersUpdateMeRequest.md
 docs/HandlersUserAPIKeyOut.md
+docs/HealthAPI.md
 docs/LabelsAPI.md
 docs/MeAPI.md
 docs/MeAPIKeysAPI.md
@@ -57,6 +63,7 @@ docs/UtilsErrorResponse.md
 git_push.sh
 go.mod
 go.sum
+model_dto_annotation_response.go
 model_dto_auth_start_response.go
 model_dto_create_label_request.go
 model_dto_create_server_request.go
@@ -76,6 +83,7 @@ model_dto_update_label_request.go
 model_dto_update_server_request.go
 model_dto_update_taint_request.go
 model_handlers_create_user_key_request.go
+model_handlers_health_status.go
 model_handlers_me_response.go
 model_handlers_member_out.go
 model_handlers_member_upsert_req.go
@@ -91,7 +99,9 @@ model_models_user.go
 model_models_user_email.go
 model_utils_error_response.go
 response.go
+test/api_annotations_test.go
 test/api_auth_test.go
+test/api_health_test.go
 test/api_labels_test.go
 test/api_me_api_keys_test.go
 test/api_me_test.go

sdk/go/README.md

@@ -78,11 +78,14 @@ All URIs are relative to *http://localhost:8080/api/v1*
 
 Class | Method | HTTP request | Description
 ------------ | ------------- | ------------- | -------------
+*AnnotationsAPI* | [**GetAnnotation**](docs/AnnotationsAPI.md#getannotation) | **Get** /annotations/{id} | Get annotation by ID (org scoped)
+*AnnotationsAPI* | [**ListAnnotations**](docs/AnnotationsAPI.md#listannotations) | **Get** /annotations | List annotations (org scoped)
 *AuthAPI* | [**AuthCallback**](docs/AuthAPI.md#authcallback) | **Get** /auth/{provider}/callback | Handle social login callback
 *AuthAPI* | [**AuthStart**](docs/AuthAPI.md#authstart) | **Post** /auth/{provider}/start | Begin social login
 *AuthAPI* | [**GetJWKS**](docs/AuthAPI.md#getjwks) | **Get** /.well-known/jwks.json | Get JWKS
 *AuthAPI* | [**Logout**](docs/AuthAPI.md#logout) | **Post** /auth/logout | Revoke refresh token family (logout everywhere)
 *AuthAPI* | [**Refresh**](docs/AuthAPI.md#refresh) | **Post** /auth/refresh | Rotate refresh token
+*HealthAPI* | [**HealthCheckOperationId**](docs/HealthAPI.md#healthcheckoperationid) | **Get** /healthz | Basic health check
 *LabelsAPI* | [**CreateLabel**](docs/LabelsAPI.md#createlabel) | **Post** /labels | Create label (org scoped)
 *LabelsAPI* | [**DeleteLabel**](docs/LabelsAPI.md#deletelabel) | **Delete** /labels/{id} | Delete label (org scoped)
 *LabelsAPI* | [**GetLabel**](docs/LabelsAPI.md#getlabel) | **Get** /labels/{id} | Get label by ID (org scoped)
@@ -123,6 +126,7 @@ Class | Method | HTTP request | Description
 
 ## Documentation For Models
 
+ - [DtoAnnotationResponse](docs/DtoAnnotationResponse.md)
  - [DtoAuthStartResponse](docs/DtoAuthStartResponse.md)
  - [DtoCreateLabelRequest](docs/DtoCreateLabelRequest.md)
  - [DtoCreateSSHRequest](docs/DtoCreateSSHRequest.md)
@@ -142,6 +146,7 @@ Class | Method | HTTP request | Description
  - [DtoUpdateServerRequest](docs/DtoUpdateServerRequest.md)
  - [DtoUpdateTaintRequest](docs/DtoUpdateTaintRequest.md)
  - [HandlersCreateUserKeyRequest](docs/HandlersCreateUserKeyRequest.md)
+ - [HandlersHealthStatus](docs/HandlersHealthStatus.md)
  - [HandlersMeResponse](docs/HandlersMeResponse.md)
  - [HandlersMemberOut](docs/HandlersMemberOut.md)
  - [HandlersMemberUpsertReq](docs/HandlersMemberUpsertReq.md)

sdk/go/api/openapi.yaml

@@ -23,6 +23,133 @@ paths:
       summary: Get JWKS
       tags:
       - Auth
+  /annotations:
+    get:
+      description: "Returns annotations for the organization in X-Org-ID. Filters:\
+        \ `name`, `value`, and `q` (name contains). Add `include=node_pools` to include\
+        \ linked node pools."
+      operationId: ListAnnotations
+      parameters:
+      - description: Organization UUID
+        in: header
+        name: X-Org-ID
+        schema:
+          type: string
+      - description: Exact name
+        in: query
+        name: name
+        schema:
+          type: string
+      - description: Exact value
+        in: query
+        name: value
+        schema:
+          type: string
+      - description: name contains (case-insensitive)
+        in: query
+        name: q
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                items:
+                  $ref: "#/components/schemas/dto.AnnotationResponse"
+                type: array
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: organization required
+        "500":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: failed to list annotations
+      security:
+      - BearerAuth: []
+      - OrgKeyAuth: []
+      - OrgSecretAuth: []
+      summary: List annotations (org scoped)
+      tags:
+      - Annotations
+  /annotations/{id}:
+    get:
+      description: Returns one annotation. Add `include=node_pools` to include node
+        pools.
+      operationId: GetAnnotation
+      parameters:
+      - description: Organization UUID
+        in: header
+        name: X-Org-ID
+        schema:
+          type: string
+      - description: Annotation ID (UUID)
+        in: path
+        name: id
+        required: true
+        schema:
+          type: string
+      - description: "Optional: node_pools"
+        in: query
+        name: include
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/dto.AnnotationResponse"
+          description: OK
+        "400":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: invalid id
+        "401":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: organization required
+        "404":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: not found
+        "500":
+          content:
+            application/json:
+              schema:
+                type: string
+          description: fetch failed
+      security:
+      - BearerAuth: []
+      - OrgKeyAuth: []
+      - OrgSecretAuth: []
+      summary: Get annotation by ID (org scoped)
+      tags:
+      - Annotations
   /auth/logout:
     post:
       operationId: Logout
@@ -103,6 +230,20 @@ paths:
       summary: Begin social login
       tags:
       - Auth
+  /healthz:
+    get:
+      description: Returns 200 OK when the service is up
+      operationId: HealthCheck               // operationId
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/handlers.HealthStatus"
+          description: OK
+      summary: Basic health check
+      tags:
+      - Health
   /labels:
     get:
       description: "Returns node labels for the organization in X-Org-ID. Filters:\
@@ -1740,6 +1881,28 @@ paths:
       x-codegen-request-body-name: body
 components:
   schemas:
+    dto.AnnotationResponse:
+      example:
+        updated_at: updated_at
+        organization_id: organization_id
+        created_at: created_at
+        id: id
+        value: value
+        key: key
+      properties:
+        created_at:
+          type: string
+        id:
+          type: string
+        key:
+          type: string
+        organization_id:
+          type: string
+        updated_at:
+          type: string
+        value:
+          type: string
+      type: object
     dto.AuthStartResponse:
       example:
         auth_url: https://accounts.google.com/o/oauth2/v2/auth?client_id=...
@@ -2054,6 +2217,14 @@ components:
         value:
           type: string
       type: object
+    handlers.HealthStatus:
+      example:
+        status: ok
+      properties:
+        status:
+          example: ok
+          type: string
+      type: object
     handlers.createUserKeyRequest:
       properties:
         expires_in_hours:

sdk/go/client.go

@@ -48,8 +48,12 @@ type APIClient struct {
 
 	// API Services
 
+	AnnotationsAPI *AnnotationsAPIService
+
 	AuthAPI *AuthAPIService
 
+	HealthAPI *HealthAPIService
+
 	LabelsAPI *LabelsAPIService
 
 	MeAPI *MeAPIService
@@ -81,7 +85,9 @@ func NewAPIClient(cfg *Configuration) *APIClient {
 	c.common.client = c
 
 	// API Services
+	c.AnnotationsAPI = (*AnnotationsAPIService)(&c.common)
 	c.AuthAPI = (*AuthAPIService)(&c.common)
+	c.HealthAPI = (*HealthAPIService)(&c.common)
 	c.LabelsAPI = (*LabelsAPIService)(&c.common)
 	c.MeAPI = (*MeAPIService)(&c.common)
 	c.MeAPIKeysAPI = (*MeAPIKeysAPIService)(&c.common)

sdk/ts/.openapi-generator/FILES

@@ -2,7 +2,9 @@
 .npmignore
 .openapi-generator-ignore
 README.md
+docs/AnnotationsApi.md
 docs/AuthApi.md
+docs/DtoAnnotationResponse.md
 docs/DtoAuthStartResponse.md
 docs/DtoCreateLabelRequest.md
 docs/DtoCreateSSHRequest.md
@@ -22,6 +24,7 @@ docs/DtoUpdateLabelRequest.md
 docs/DtoUpdateServerRequest.md
 docs/DtoUpdateTaintRequest.md
 docs/HandlersCreateUserKeyRequest.md
+docs/HandlersHealthStatus.md
 docs/HandlersMeResponse.md
 docs/HandlersMemberOut.md
 docs/HandlersMemberUpsertReq.md
@@ -31,6 +34,7 @@ docs/HandlersOrgKeyCreateResp.md
 docs/HandlersOrgUpdateReq.md
 docs/HandlersUpdateMeRequest.md
 docs/HandlersUserAPIKeyOut.md
+docs/HealthApi.md
 docs/LabelsApi.md
 docs/MeAPIKeysApi.md
 docs/MeApi.md
@@ -44,7 +48,9 @@ docs/SshApi.md
 docs/TaintsApi.md
 docs/UtilsErrorResponse.md
 package.json
+src/apis/AnnotationsApi.ts
 src/apis/AuthApi.ts
+src/apis/HealthApi.ts
 src/apis/LabelsApi.ts
 src/apis/MeAPIKeysApi.ts
 src/apis/MeApi.ts
@@ -54,6 +60,7 @@ src/apis/SshApi.ts
 src/apis/TaintsApi.ts
 src/apis/index.ts
 src/index.ts
+src/models/DtoAnnotationResponse.ts
 src/models/DtoAuthStartResponse.ts
 src/models/DtoCreateLabelRequest.ts
 src/models/DtoCreateSSHRequest.ts
@@ -73,6 +80,7 @@ src/models/DtoUpdateLabelRequest.ts
 src/models/DtoUpdateServerRequest.ts
 src/models/DtoUpdateTaintRequest.ts
 src/models/HandlersCreateUserKeyRequest.ts
+src/models/HandlersHealthStatus.ts
 src/models/HandlersMeResponse.ts
 src/models/HandlersMemberOut.ts
 src/models/HandlersMemberUpsertReq.ts

sdk/ts/README.md

@@ -13,20 +13,32 @@ npm install @glueops/autoglue-sdk-go --save
 Next, try it out.
 
 ```ts
-import { Configuration, AuthApi } from "@glueops/autoglue-sdk-go";
+import { Configuration, AnnotationsApi } from "@glueops/autoglue-sdk-go";
-import type { AuthCallbackRequest } from "@glueops/autoglue-sdk-go";
+import type { GetAnnotationRequest } from "@glueops/autoglue-sdk-go";
 
 async function example() {
   console.log("🚀 Testing @glueops/autoglue-sdk-go SDK...");
-  const api = new AuthApi();
+  const config = new Configuration({
+    // To configure API key authorization: OrgKeyAuth
+    apiKey: "YOUR API KEY",
+    // To configure API key authorization: OrgSecretAuth
+    apiKey: "YOUR API KEY",
+    // To configure API key authorization: BearerAuth
+    apiKey: "YOUR API KEY",
+  });
+  const api = new AnnotationsApi(config);
 
   const body = {
-    // string | google|github
+    // string | Annotation ID (UUID)
-    provider: provider_example,
+    id: id_example,
-  } satisfies AuthCallbackRequest;
+    // string | Organization UUID (optional)
+    xOrgID: xOrgID_example,
+    // string | Optional: node_pools (optional)
+    include: include_example,
+  } satisfies GetAnnotationRequest;
 
   try {
-    const data = await api.authCallback(body);
+    const data = await api.getAnnotation(body);
     console.log(data);
   } catch (error) {
     console.error(error);
@@ -43,52 +55,56 @@ example().catch(console.error);
 
 All URIs are relative to _http://localhost:8080/api/v1_
 
-| Class          | Method                                                        | HTTP request                            | Description                                     |
+| Class            | Method                                                                 | HTTP request                            | Description                                     |
-| -------------- | ------------------------------------------------------------- | --------------------------------------- | ----------------------------------------------- |
+| ---------------- | ---------------------------------------------------------------------- | --------------------------------------- | ----------------------------------------------- |
-| _AuthApi_      | [**authCallback**](docs/AuthApi.md#authcallback)              | **GET** /auth/{provider}/callback       | Handle social login callback                    |
+| _AnnotationsApi_ | [**getAnnotation**](docs/AnnotationsApi.md#getannotation)              | **GET** /annotations/{id}               | Get annotation by ID (org scoped)               |
-| _AuthApi_      | [**authStart**](docs/AuthApi.md#authstart)                    | **POST** /auth/{provider}/start         | Begin social login                              |
+| _AnnotationsApi_ | [**listAnnotations**](docs/AnnotationsApi.md#listannotations)          | **GET** /annotations                    | List annotations (org scoped)                   |
-| _AuthApi_      | [**getJWKS**](docs/AuthApi.md#getjwks)                        | **GET** /.well-known/jwks.json          | Get JWKS                                        |
+| _AuthApi_        | [**authCallback**](docs/AuthApi.md#authcallback)                       | **GET** /auth/{provider}/callback       | Handle social login callback                    |
-| _AuthApi_      | [**logout**](docs/AuthApi.md#logout)                          | **POST** /auth/logout                   | Revoke refresh token family (logout everywhere) |
+| _AuthApi_        | [**authStart**](docs/AuthApi.md#authstart)                             | **POST** /auth/{provider}/start         | Begin social login                              |
-| _AuthApi_      | [**refresh**](docs/AuthApi.md#refresh)                        | **POST** /auth/refresh                  | Rotate refresh token                            |
+| _AuthApi_        | [**getJWKS**](docs/AuthApi.md#getjwks)                                 | **GET** /.well-known/jwks.json          | Get JWKS                                        |
-| _LabelsApi_    | [**createLabel**](docs/LabelsApi.md#createlabel)              | **POST** /labels                        | Create label (org scoped)                       |
+| _AuthApi_        | [**logout**](docs/AuthApi.md#logout)                                   | **POST** /auth/logout                   | Revoke refresh token family (logout everywhere) |
-| _LabelsApi_    | [**deleteLabel**](docs/LabelsApi.md#deletelabel)              | **DELETE** /labels/{id}                 | Delete label (org scoped)                       |
+| _AuthApi_        | [**refresh**](docs/AuthApi.md#refresh)                                 | **POST** /auth/refresh                  | Rotate refresh token                            |
-| _LabelsApi_    | [**getLabel**](docs/LabelsApi.md#getlabel)                    | **GET** /labels/{id}                    | Get label by ID (org scoped)                    |
+| _HealthApi_      | [**healthCheckOperationId**](docs/HealthApi.md#healthcheckoperationid) | **GET** /healthz                        | Basic health check                              |
-| _LabelsApi_    | [**listLabels**](docs/LabelsApi.md#listlabels)                | **GET** /labels                         | List node labels (org scoped)                   |
+| _LabelsApi_      | [**createLabel**](docs/LabelsApi.md#createlabel)                       | **POST** /labels                        | Create label (org scoped)                       |
-| _LabelsApi_    | [**updateLabel**](docs/LabelsApi.md#updatelabel)              | **PATCH** /labels/{id}                  | Update label (org scoped)                       |
+| _LabelsApi_      | [**deleteLabel**](docs/LabelsApi.md#deletelabel)                       | **DELETE** /labels/{id}                 | Delete label (org scoped)                       |
-| _MeApi_        | [**getMe**](docs/MeApi.md#getme)                              | **GET** /me                             | Get current user profile                        |
+| _LabelsApi_      | [**getLabel**](docs/LabelsApi.md#getlabel)                             | **GET** /labels/{id}                    | Get label by ID (org scoped)                    |
-| _MeApi_        | [**updateMe**](docs/MeApi.md#updateme)                        | **PATCH** /me                           | Update current user profile                     |
+| _LabelsApi_      | [**listLabels**](docs/LabelsApi.md#listlabels)                         | **GET** /labels                         | List node labels (org scoped)                   |
-| _MeAPIKeysApi_ | [**createUserAPIKey**](docs/MeAPIKeysApi.md#createuserapikey) | **POST** /me/api-keys                   | Create a new user API key                       |
+| _LabelsApi_      | [**updateLabel**](docs/LabelsApi.md#updatelabel)                       | **PATCH** /labels/{id}                  | Update label (org scoped)                       |
-| _MeAPIKeysApi_ | [**deleteUserAPIKey**](docs/MeAPIKeysApi.md#deleteuserapikey) | **DELETE** /me/api-keys/{id}            | Delete a user API key                           |
+| _MeApi_          | [**getMe**](docs/MeApi.md#getme)                                       | **GET** /me                             | Get current user profile                        |
-| _MeAPIKeysApi_ | [**listUserAPIKeys**](docs/MeAPIKeysApi.md#listuserapikeys)   | **GET** /me/api-keys                    | List my API keys                                |
+| _MeApi_          | [**updateMe**](docs/MeApi.md#updateme)                                 | **PATCH** /me                           | Update current user profile                     |
-| _OrgsApi_      | [**addOrUpdateMember**](docs/OrgsApi.md#addorupdatemember)    | **POST** /orgs/{id}/members             | Add or update a member (owner/admin)            |
+| _MeAPIKeysApi_   | [**createUserAPIKey**](docs/MeAPIKeysApi.md#createuserapikey)          | **POST** /me/api-keys                   | Create a new user API key                       |
-| _OrgsApi_      | [**createOrg**](docs/OrgsApi.md#createorg)                    | **POST** /orgs                          | Create organization                             |
+| _MeAPIKeysApi_   | [**deleteUserAPIKey**](docs/MeAPIKeysApi.md#deleteuserapikey)          | **DELETE** /me/api-keys/{id}            | Delete a user API key                           |
-| _OrgsApi_      | [**createOrgKey**](docs/OrgsApi.md#createorgkey)              | **POST** /orgs/{id}/api-keys            | Create org key/secret pair (owner/admin)        |
+| _MeAPIKeysApi_   | [**listUserAPIKeys**](docs/MeAPIKeysApi.md#listuserapikeys)            | **GET** /me/api-keys                    | List my API keys                                |
-| _OrgsApi_      | [**deleteOrg**](docs/OrgsApi.md#deleteorg)                    | **DELETE** /orgs/{id}                   | Delete organization (owner)                     |
+| _OrgsApi_        | [**addOrUpdateMember**](docs/OrgsApi.md#addorupdatemember)             | **POST** /orgs/{id}/members             | Add or update a member (owner/admin)            |
-| _OrgsApi_      | [**deleteOrgKey**](docs/OrgsApi.md#deleteorgkey)              | **DELETE** /orgs/{id}/api-keys/{key_id} | Delete org key (owner/admin)                    |
+| _OrgsApi_        | [**createOrg**](docs/OrgsApi.md#createorg)                             | **POST** /orgs                          | Create organization                             |
-| _OrgsApi_      | [**getOrg**](docs/OrgsApi.md#getorg)                          | **GET** /orgs/{id}                      | Get organization                                |
+| _OrgsApi_        | [**createOrgKey**](docs/OrgsApi.md#createorgkey)                       | **POST** /orgs/{id}/api-keys            | Create org key/secret pair (owner/admin)        |
-| _OrgsApi_      | [**listMembers**](docs/OrgsApi.md#listmembers)                | **GET** /orgs/{id}/members              | List members in org                             |
+| _OrgsApi_        | [**deleteOrg**](docs/OrgsApi.md#deleteorg)                             | **DELETE** /orgs/{id}                   | Delete organization (owner)                     |
-| _OrgsApi_      | [**listMyOrgs**](docs/OrgsApi.md#listmyorgs)                  | **GET** /orgs                           | List organizations I belong to                  |
+| _OrgsApi_        | [**deleteOrgKey**](docs/OrgsApi.md#deleteorgkey)                       | **DELETE** /orgs/{id}/api-keys/{key_id} | Delete org key (owner/admin)                    |
-| _OrgsApi_      | [**listOrgKeys**](docs/OrgsApi.md#listorgkeys)                | **GET** /orgs/{id}/api-keys             | List org-scoped API keys (no secrets)           |
+| _OrgsApi_        | [**getOrg**](docs/OrgsApi.md#getorg)                                   | **GET** /orgs/{id}                      | Get organization                                |
-| _OrgsApi_      | [**removeMember**](docs/OrgsApi.md#removemember)              | **DELETE** /orgs/{id}/members/{user_id} | Remove a member (owner/admin)                   |
+| _OrgsApi_        | [**listMembers**](docs/OrgsApi.md#listmembers)                         | **GET** /orgs/{id}/members              | List members in org                             |
-| _OrgsApi_      | [**updateOrg**](docs/OrgsApi.md#updateorg)                    | **PATCH** /orgs/{id}                    | Update organization (owner/admin)               |
+| _OrgsApi_        | [**listMyOrgs**](docs/OrgsApi.md#listmyorgs)                           | **GET** /orgs                           | List organizations I belong to                  |
-| _ServersApi_   | [**createServer**](docs/ServersApi.md#createserver)           | **POST** /servers                       | Create server (org scoped)                      |
+| _OrgsApi_        | [**listOrgKeys**](docs/OrgsApi.md#listorgkeys)                         | **GET** /orgs/{id}/api-keys             | List org-scoped API keys (no secrets)           |
-| _ServersApi_   | [**deleteServer**](docs/ServersApi.md#deleteserver)           | **DELETE** /servers/{id}                | Delete server (org scoped)                      |
+| _OrgsApi_        | [**removeMember**](docs/OrgsApi.md#removemember)                       | **DELETE** /orgs/{id}/members/{user_id} | Remove a member (owner/admin)                   |
-| _ServersApi_   | [**getServer**](docs/ServersApi.md#getserver)                 | **GET** /servers/{id}                   | Get server by ID (org scoped)                   |
+| _OrgsApi_        | [**updateOrg**](docs/OrgsApi.md#updateorg)                             | **PATCH** /orgs/{id}                    | Update organization (owner/admin)               |
-| _ServersApi_   | [**listServers**](docs/ServersApi.md#listservers)             | **GET** /servers                        | List servers (org scoped)                       |
+| _ServersApi_     | [**createServer**](docs/ServersApi.md#createserver)                    | **POST** /servers                       | Create server (org scoped)                      |
-| _ServersApi_   | [**updateServer**](docs/ServersApi.md#updateserver)           | **PATCH** /servers/{id}                 | Update server (org scoped)                      |
+| _ServersApi_     | [**deleteServer**](docs/ServersApi.md#deleteserver)                    | **DELETE** /servers/{id}                | Delete server (org scoped)                      |
-| _SshApi_       | [**createSSHKey**](docs/SshApi.md#createsshkey)               | **POST** /ssh                           | Create ssh keypair (org scoped)                 |
+| _ServersApi_     | [**getServer**](docs/ServersApi.md#getserver)                          | **GET** /servers/{id}                   | Get server by ID (org scoped)                   |
-| _SshApi_       | [**deleteSSHKey**](docs/SshApi.md#deletesshkey)               | **DELETE** /ssh/{id}                    | Delete ssh keypair (org scoped)                 |
+| _ServersApi_     | [**listServers**](docs/ServersApi.md#listservers)                      | **GET** /servers                        | List servers (org scoped)                       |
-| _SshApi_       | [**downloadSSHKey**](docs/SshApi.md#downloadsshkey)           | **GET** /ssh/{id}/download              | Download ssh key files by ID (org scoped)       |
+| _ServersApi_     | [**updateServer**](docs/ServersApi.md#updateserver)                    | **PATCH** /servers/{id}                 | Update server (org scoped)                      |
-| _SshApi_       | [**getSSHKey**](docs/SshApi.md#getsshkey)                     | **GET** /ssh/{id}                       | Get ssh key by ID (org scoped)                  |
+| _SshApi_         | [**createSSHKey**](docs/SshApi.md#createsshkey)                        | **POST** /ssh                           | Create ssh keypair (org scoped)                 |
-| _SshApi_       | [**listPublicSshKeys**](docs/SshApi.md#listpublicsshkeys)     | **GET** /ssh                            | List ssh keys (org scoped)                      |
+| _SshApi_         | [**deleteSSHKey**](docs/SshApi.md#deletesshkey)                        | **DELETE** /ssh/{id}                    | Delete ssh keypair (org scoped)                 |
-| _TaintsApi_    | [**createTaint**](docs/TaintsApi.md#createtaint)              | **POST** /taints                        | Create node taint (org scoped)                  |
+| _SshApi_         | [**downloadSSHKey**](docs/SshApi.md#downloadsshkey)                    | **GET** /ssh/{id}/download              | Download ssh key files by ID (org scoped)       |
-| _TaintsApi_    | [**deleteTaint**](docs/TaintsApi.md#deletetaint)              | **DELETE** /taints/{id}                 | Delete taint (org scoped)                       |
+| _SshApi_         | [**getSSHKey**](docs/SshApi.md#getsshkey)                              | **GET** /ssh/{id}                       | Get ssh key by ID (org scoped)                  |
-| _TaintsApi_    | [**getTaint**](docs/TaintsApi.md#gettaint)                    | **GET** /taints/{id}                    | Get node taint by ID (org scoped)               |
+| _SshApi_         | [**listPublicSshKeys**](docs/SshApi.md#listpublicsshkeys)              | **GET** /ssh                            | List ssh keys (org scoped)                      |
-| _TaintsApi_    | [**listTaints**](docs/TaintsApi.md#listtaints)                | **GET** /taints                         | List node pool taints (org scoped)              |
+| _TaintsApi_      | [**createTaint**](docs/TaintsApi.md#createtaint)                       | **POST** /taints                        | Create node taint (org scoped)                  |
-| _TaintsApi_    | [**updateTaint**](docs/TaintsApi.md#updatetaint)              | **PATCH** /taints/{id}                  | Update node taint (org scoped)                  |
+| _TaintsApi_      | [**deleteTaint**](docs/TaintsApi.md#deletetaint)                       | **DELETE** /taints/{id}                 | Delete taint (org scoped)                       |
+| _TaintsApi_      | [**getTaint**](docs/TaintsApi.md#gettaint)                             | **GET** /taints/{id}                    | Get node taint by ID (org scoped)               |
+| _TaintsApi_      | [**listTaints**](docs/TaintsApi.md#listtaints)                         | **GET** /taints                         | List node pool taints (org scoped)              |
+| _TaintsApi_      | [**updateTaint**](docs/TaintsApi.md#updatetaint)                       | **PATCH** /taints/{id}                  | Update node taint (org scoped)                  |
 
 ### Models
 
+- [DtoAnnotationResponse](docs/DtoAnnotationResponse.md)
 - [DtoAuthStartResponse](docs/DtoAuthStartResponse.md)
 - [DtoCreateLabelRequest](docs/DtoCreateLabelRequest.md)
 - [DtoCreateSSHRequest](docs/DtoCreateSSHRequest.md)
@@ -108,6 +124,7 @@ All URIs are relative to _http://localhost:8080/api/v1_
 - [DtoUpdateServerRequest](docs/DtoUpdateServerRequest.md)
 - [DtoUpdateTaintRequest](docs/DtoUpdateTaintRequest.md)
 - [HandlersCreateUserKeyRequest](docs/HandlersCreateUserKeyRequest.md)
+- [HandlersHealthStatus](docs/HandlersHealthStatus.md)
 - [HandlersMeResponse](docs/HandlersMeResponse.md)
 - [HandlersMemberOut](docs/HandlersMemberOut.md)
 - [HandlersMemberUpsertReq](docs/HandlersMemberUpsertReq.md)

sdk/ts/src/apis/index.ts

@@ -1,6 +1,8 @@
 /* tslint:disable */
 /* eslint-disable */
+export * from "./AnnotationsApi";
 export * from "./AuthApi";
+export * from "./HealthApi";
 export * from "./LabelsApi";
 export * from "./MeApi";
 export * from "./MeAPIKeysApi";

sdk/ts/src/models/index.ts

@@ -1,5 +1,6 @@
 /* tslint:disable */
 /* eslint-disable */
+export * from "./DtoAnnotationResponse";
 export * from "./DtoAuthStartResponse";
 export * from "./DtoCreateLabelRequest";
 export * from "./DtoCreateSSHRequest";
@@ -19,6 +20,7 @@ export * from "./DtoUpdateLabelRequest";
 export * from "./DtoUpdateServerRequest";
 export * from "./DtoUpdateTaintRequest";
 export * from "./HandlersCreateUserKeyRequest";
+export * from "./HandlersHealthStatus";
 export * from "./HandlersMeResponse";
 export * from "./HandlersMemberOut";
 export * from "./HandlersMemberUpsertReq";

ui/src/pages/servers/server-page.tsx

@@ -554,7 +554,10 @@ export const ServerPage = () => {
                       </TableCell>
                       <TableCell className="capitalize">
                         <span
-                          className={cn(k.role === "bastion" && "rounded bg-amber-50 px-2 py-0.5")}
+                          className={cn(
+                            k.role === "bastion" &&
+                              "rounded bg-amber-50 px-2 py-0.5 dark:bg-amber-900"
+                          )}
                         >
                           {k.role}
                         </span>

ui/src/sdk/.openapi-generator/FILES

@@ -2,7 +2,9 @@
 .npmignore
 .openapi-generator-ignore
 README.md
+docs/AnnotationsApi.md
 docs/AuthApi.md
+docs/DtoAnnotationResponse.md
 docs/DtoAuthStartResponse.md
 docs/DtoCreateLabelRequest.md
 docs/DtoCreateSSHRequest.md
@@ -22,6 +24,7 @@ docs/DtoUpdateLabelRequest.md
 docs/DtoUpdateServerRequest.md
 docs/DtoUpdateTaintRequest.md
 docs/HandlersCreateUserKeyRequest.md
+docs/HandlersHealthStatus.md
 docs/HandlersMeResponse.md
 docs/HandlersMemberOut.md
 docs/HandlersMemberUpsertReq.md
@@ -31,6 +34,7 @@ docs/HandlersOrgKeyCreateResp.md
 docs/HandlersOrgUpdateReq.md
 docs/HandlersUpdateMeRequest.md
 docs/HandlersUserAPIKeyOut.md
+docs/HealthApi.md
 docs/LabelsApi.md
 docs/MeAPIKeysApi.md
 docs/MeApi.md
@@ -44,7 +48,9 @@ docs/SshApi.md
 docs/TaintsApi.md
 docs/UtilsErrorResponse.md
 package.json
+src/apis/AnnotationsApi.ts
 src/apis/AuthApi.ts
+src/apis/HealthApi.ts
 src/apis/LabelsApi.ts
 src/apis/MeAPIKeysApi.ts
 src/apis/MeApi.ts
@@ -54,6 +60,7 @@ src/apis/SshApi.ts
 src/apis/TaintsApi.ts
 src/apis/index.ts
 src/index.ts
+src/models/DtoAnnotationResponse.ts
 src/models/DtoAuthStartResponse.ts
 src/models/DtoCreateLabelRequest.ts
 src/models/DtoCreateSSHRequest.ts
@@ -73,6 +80,7 @@ src/models/DtoUpdateLabelRequest.ts
 src/models/DtoUpdateServerRequest.ts
 src/models/DtoUpdateTaintRequest.ts
 src/models/HandlersCreateUserKeyRequest.ts
+src/models/HandlersHealthStatus.ts
 src/models/HandlersMeResponse.ts
 src/models/HandlersMemberOut.ts
 src/models/HandlersMemberUpsertReq.ts

ui/src/sdk/apis/index.ts

@@ -1,6 +1,8 @@
 /* tslint:disable */
 /* eslint-disable */
+export * from './AnnotationsApi';
 export * from './AuthApi';
+export * from './HealthApi';
 export * from './LabelsApi';
 export * from './MeApi';
 export * from './MeAPIKeysApi';

ui/src/sdk/models/index.ts

@@ -1,5 +1,6 @@
 /* tslint:disable */
 /* eslint-disable */
+export * from './DtoAnnotationResponse';
 export * from './DtoAuthStartResponse';
 export * from './DtoCreateLabelRequest';
 export * from './DtoCreateSSHRequest';
@@ -19,6 +20,7 @@ export * from './DtoUpdateLabelRequest';
 export * from './DtoUpdateServerRequest';
 export * from './DtoUpdateTaintRequest';
 export * from './HandlersCreateUserKeyRequest';
+export * from './HandlersHealthStatus';
 export * from './HandlersMeResponse';
 export * from './HandlersMemberOut';
 export * from './HandlersMemberUpsertReq';