initial jobs to bootstrap bastion with docker

cmd/cli/init-config.go

@@ -32,6 +32,11 @@ var initConfigCmd = &cobra.Command{
 			"authentication": map[string]string{
 				"secret": defaultSecret,
 			},
+			"archer": map[string]interface{}{
+				"instances":     2,
+				"timeoutSec":    60,
+				"enqueue_every": 500,
+			},
 			"smtp": map[string]interface{}{
 				"enabled":  false,
 				"host":     "smtp.example.com",

cmd/cli/serve.go

@@ -10,8 +10,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dyaksa/archer"
 	"github.com/glueops/autoglue/internal/api"
+	"github.com/glueops/autoglue/internal/bg"
 	"github.com/glueops/autoglue/internal/db"
+	"github.com/google/uuid"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -28,13 +31,69 @@ var serveCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		db.Connect()
 
-		// Resolve bind address/port from viper (flags/env/config/defaults)
-		addr := fmt.Sprintf("%s:%s", viper.GetString("bind_address"), viper.GetString("bind_port"))
+		jobs, err := bg.NewJobs()
+		if err != nil {
+			log.Fatalf("failed to init background jobs: %v", err)
+		}
 
-		// Build server (uses Chi router inside)
+		// Start workers in background ONCE
+		go func() {
+			if err := jobs.Start(); err != nil {
+				log.Fatalf("failed to start background workers: %v", err)
+			}
+		}()
+		defer jobs.Stop()
+
+		// Enqueue one job immediately
+		/*
+			id := uuid.NewString()
+			if _, err := jobs.Enqueue(context.Background(), id, "bootstrap_bastion", bg.BastionBootstrapArgs{}); err != nil {
+				log.Fatalf("[enqueue] failed (job_id=%s): %v", id, err)
+			}
+			log.Printf("[enqueue] queued (job_id=%s)", id)
+
+			// Verify the row exists
+			if got, err := jobs.Client.Get(context.Background(), id); err != nil {
+				log.Fatalf("[verify] Get failed (job_id=%s): %v", id, err)
+			} else if j, ok := got.(*job.Job); ok {
+				log.Printf("[verify] Get ok (job_id=%s, status=%s)", j.ID, j.Status)
+			} else {
+				log.Printf("[verify] Get ok (job_id=%s) but unexpected type %T", id, got)
+			}
+		*/
+		// Periodic scheduler
+		schedCtx, schedCancel := context.WithCancel(context.Background())
+		defer schedCancel()
+
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+
+		go func() {
+			for {
+				select {
+				case <-ticker.C:
+					_, err := jobs.Enqueue(
+						context.Background(),
+						uuid.NewString(),
+						"bootstrap_bastion",
+						bg.BastionBootstrapArgs{},
+						archer.WithMaxRetries(3),
+						// while debugging, avoid extra schedule delay:
+						archer.WithScheduleTime(time.Now().Add(10*time.Second)),
+					)
+					if err != nil {
+						log.Printf("failed to enqueue bootstrap_bastion: %v", err)
+					}
+				case <-schedCtx.Done():
+					return
+				}
+			}
+		}()
+
+		// HTTP server
+		addr := fmt.Sprintf("%s:%s", viper.GetString("bind_address"), viper.GetString("bind_port"))
 		srv := api.NewServer(addr)
 
-		// Start server
 		errCh := make(chan error, 1)
 		go func() {
 			log.Printf("HTTP server listening on http://%s (ui.dev=%v)", addr, viper.GetBool("ui.dev"))
@@ -44,7 +103,6 @@ var serveCmd = &cobra.Command{
 			close(errCh)
 		}()
 
-		// Handle OS signals for graceful shutdown
 		stop := make(chan os.Signal, 1)
 		signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
 
@@ -57,9 +115,10 @@ var serveCmd = &cobra.Command{
 			}
 		}
 
+		schedCancel()
+
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
-
 		if err := srv.Shutdown(ctx); err != nil {
 			log.Printf("Graceful shutdown failed: %v; forcing close", err)
 			_ = srv.Close()
@@ -70,14 +129,9 @@ var serveCmd = &cobra.Command{
 }
 
 func init() {
-	// Flags to override bind address/port
 	serveCmd.Flags().StringVar(&bindAddress, "bind-address", "", "Address to bind the HTTP server (default 127.0.0.1)")
 	serveCmd.Flags().StringVar(&bindPort, "bind-port", "", "Port to bind the HTTP server (default 8080)")
-
-	// Bind flags to viper keys
 	_ = viper.BindPFlag("bind_address", serveCmd.Flags().Lookup("bind-address"))
 	_ = viper.BindPFlag("bind_port", serveCmd.Flags().Lookup("bind-port"))
-
-	// Register command
 	rootCmd.AddCommand(serveCmd)
 }

docker-compose.yml

@@ -15,17 +15,6 @@ services:
   #  depends_on:
   #    - postgres
 
-  redis:
-    image: redis:latest
-    healthcheck:
-      test: ["CMD-SHELL", "redis-cli ping | grep pong"]
-      interval: 1s
-      timeout: 3s
-      retries: 5
-    command: "redis-server"
-    ports:
-      - "6379:6379"
-
   postgres:
     build:
       context: postgres

go.mod

@@ -3,6 +3,7 @@ module github.com/glueops/autoglue
 go 1.25.0
 
 require (
+	github.com/dyaksa/archer v1.1.3
 	github.com/earthboundkid/versioninfo/v2 v2.24.1
 	github.com/go-chi/chi/v5 v5.2.3
 	github.com/go-chi/cors v1.2.2
@@ -17,11 +18,13 @@ require (
 	golang.org/x/crypto v0.41.0
 	golang.org/x/text v0.28.0
 	gopkg.in/yaml.v3 v3.0.1
+	gorm.io/datatypes v1.2.6
 	gorm.io/driver/postgres v1.6.0
 	gorm.io/gorm v1.30.2
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
@@ -29,7 +32,9 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -39,6 +44,7 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -57,4 +63,5 @@ require (
 	golang.org/x/sys v0.35.0 // indirect
 	golang.org/x/tools v0.36.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gorm.io/driver/mysql v1.5.6 // indirect
 )

go.sum

@@ -1,3 +1,7 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -6,6 +10,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dyaksa/archer v1.1.3 h1:jfe51tSNzzscFpu+Vilm4SKb0Lnv6FR1yaGspjab4x4=
+github.com/dyaksa/archer v1.1.3/go.mod h1:IYSp67u14JHTNuvvy6gG1eaX2TPywXvfk1FiyZwVEK4=
 github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg=
 github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -26,10 +32,19 @@ github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
 github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
+github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
@@ -60,10 +75,16 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
+github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
@@ -88,6 +109,8 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
 github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -196,7 +219,16 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/datatypes v1.2.6 h1:KafLdXvFUhzNeL2ncm03Gl3eTLONQfNKZ+wJ+9Y4Nck=
+gorm.io/datatypes v1.2.6/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY=
+gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8=
+gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
 gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
 gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
+gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU=
+gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI=
+gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc=
+gorm.io/driver/sqlserver v1.6.0/go.mod h1:WQzt4IJo/WHKnckU9jXBLMJIVNMVeTu25dnOzehntWw=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gorm.io/gorm v1.30.2 h1:f7bevlVoVe4Byu3pmbWPVHnPsLoWaMjEb7/clyr9Ivs=
 gorm.io/gorm v1.30.2/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=

internal/bg/bastion.go

@@ -0,0 +1,230 @@
+package bg
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/dyaksa/archer/job"
+	"github.com/glueops/autoglue/internal/db"
+	"github.com/glueops/autoglue/internal/db/models"
+	"github.com/glueops/autoglue/internal/utils"
+	"github.com/google/uuid"
+	"golang.org/x/crypto/ssh"
+)
+
+type BastionBootstrapArgs struct {
+}
+
+type BastionBootstrapResult struct {
+	Status       string      `json:"status"`
+	Processed    int         `json:"processed"`
+	Ready        int         `json:"ready"`
+	Failed       int         `json:"failed"`
+	ElapsedMs    int         `json:"elapsed_ms"`
+	FailedServer []uuid.UUID `json:"failed_server_ids"`
+}
+
+func BastionBootstrap(ctx context.Context, j job.Job) (any, error) {
+	start := time.Now()
+	log.Printf("[bastion] scan for pending bastions...")
+
+	var bastions []models.Server
+	if err := db.DB.
+		Preload("SshKey").
+		Where("role = ? AND status = ?", "bastion", "pending").
+		Find(&bastions).Error; err != nil {
+		return nil, err
+	}
+
+	if len(bastions) == 0 {
+		log.Printf("[bastion] nothing to do")
+		return BastionBootstrapResult{
+			Status:    "ok",
+			Processed: 0,
+			Ready:     0,
+			Failed:    0,
+			ElapsedMs: int(time.Since(start) / time.Millisecond),
+		}, nil
+	}
+
+	res := BastionBootstrapResult{Status: "ok", Processed: 0}
+	for _, s := range bastions {
+		select {
+		case <-ctx.Done():
+			return res, ctx.Err()
+		default:
+		}
+
+		// 2) Claim atomically so only one worker does it
+		claimed, err := claimPendingBastion(s.ID)
+		if err != nil {
+			log.Printf("[bastion] claim %s error: %v", s.ID, err)
+			res.Failed++
+			res.FailedServer = append(res.FailedServer, s.ID)
+			continue
+		}
+		if !claimed {
+			continue // someone else took it
+		}
+
+		// 3) Decrypt the private key
+		privPEM, err := utils.DecryptForOrg(s.OrganizationID, s.SshKey.EncryptedPrivateKey, s.SshKey.PrivateIV, s.SshKey.PrivateTag)
+		if err != nil {
+			log.Printf("[bastion] %s decrypt key: %v", s.ID, err)
+			_ = markFailed(s.ID, fmt.Errorf("decrypt key: %w", err))
+			res.Failed++
+			res.FailedServer = append(res.FailedServer, s.ID)
+			continue
+		}
+
+		// 4) Provision over SSH
+		addr := s.IPAddress
+		if !strings.Contains(addr, ":") {
+			addr = addr + ":22"
+		}
+		err = provisionDocker(ctx, addr, s.SSHUser, []byte(privPEM))
+		if err != nil {
+			log.Printf("[bastion] %s provision failed: %v", s.ID, err)
+			_ = markFailed(s.ID, err)
+			res.Failed++
+			res.FailedServer = append(res.FailedServer, s.ID)
+			continue
+		}
+
+		// 5) Mark ready
+		if err := markReady(s.ID); err != nil {
+			log.Printf("[bastion] %s mark ready err: %v", s.ID, err)
+			res.Failed++
+			res.FailedServer = append(res.FailedServer, s.ID)
+			continue
+		}
+
+		res.Ready++
+		res.Processed++
+	}
+
+	res.ElapsedMs = int(time.Since(start) / time.Millisecond)
+	log.Printf("[bastion] done: processed=%d ready=%d failed=%d", res.Processed, res.Ready, res.Failed)
+	return res, nil
+}
+
+func claimPendingBastion(id uuid.UUID) (bool, error) {
+	tx := db.DB.Model(&models.Server{}).
+		Where("id = ? AND status = ?", id, "pending").
+		Update("status", "provisioning")
+	if tx.Error != nil {
+		return false, tx.Error
+	}
+	return tx.RowsAffected == 1, nil
+}
+
+func markReady(id uuid.UUID) error {
+	return db.DB.Model(&models.Server{}).
+		Where("id = ?", id).
+		Update("status", "ready").Error
+}
+
+func markFailed(id uuid.UUID, cause error) error {
+	msg := cause.Error()
+	if len(msg) > 1000 {
+		msg = msg[:1000]
+	}
+	// You can also add a separate column for error details if you want to preserve more text.
+	return db.DB.Model(&models.Server{}).
+		Where("id = ?", id).
+		Updates(map[string]any{
+			"status": "failed",
+		}).Error
+}
+
+func provisionDocker(ctx context.Context, addr, user string, privKeyPEM []byte) error {
+	signer, err := ssh.ParsePrivateKey(privKeyPEM)
+	if err != nil {
+		return fmt.Errorf("parse private key: %w", err)
+	}
+
+	cfg := &ssh.ClientConfig{
+		User:            user,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		Timeout:         20 * time.Second,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // TODO: replace with known_hosts verification
+	}
+
+	client, err := ssh.Dial("tcp", addr, cfg)
+	if err != nil {
+		return fmt.Errorf("dial ssh %s: %w", addr, err)
+	}
+	defer client.Close()
+
+	script := `
+set -euo pipefail
+
+if command -v docker >/dev/null 2>&1; then
+  echo "docker already installed"
+else
+  if command -v apt-get >/dev/null 2>&1; then
+    export DEBIAN_FRONTEND=noninteractive
+    sudo apt-get update -y
+    sudo apt-get install -y ca-certificates curl gnupg lsb-release
+    curl -fsSL https://get.docker.com | sh
+  elif command -v dnf >/dev/null 2>&1; then
+    sudo dnf -y install dnf-plugins-core || true
+    curl -fsSL https://get.docker.com | sh
+  elif command -v yum >/dev/null 2>&1; then
+    sudo yum -y install yum-utils || true
+    curl -fsSL https://get.docker.com | sh
+  else
+    curl -fsSL https://get.docker.com | sh
+  fi
+fi
+
+# Ensure service is enabled and running
+if command -v systemctl >/dev/null 2>&1; then
+  sudo systemctl enable docker || true
+  sudo systemctl restart docker || sudo systemctl start docker || true
+fi
+
+docker --version
+`
+
+	return runRemote(ctx, client, "bash -lc "+quoteForShell(script))
+}
+
+func runRemote(ctx context.Context, client *ssh.Client, cmd string) error {
+	sess, err := client.NewSession()
+	if err != nil {
+		return fmt.Errorf("new session: %w", err)
+	}
+	defer sess.Close()
+
+	var out, stderr bytes.Buffer
+	sess.Stdout = &out
+	sess.Stderr = &stderr
+
+	done := make(chan error, 1)
+	go func() { done <- sess.Run(cmd) }()
+
+	select {
+	case <-ctx.Done():
+		_ = sess.Signal(ssh.SIGKILL) // best-effort
+		return ctx.Err()
+	case err := <-done:
+		if err != nil {
+			if stderr.Len() > 0 {
+				return errors.New(strings.TrimSpace(stderr.String()))
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func quoteForShell(s string) string {
+	// naive single-quote wrapper; escape single quotes for bash
+	return fmt.Sprintf("'%s'", strings.ReplaceAll(s, "'", `'\''`))
+}

internal/bg/bg.go

@@ -0,0 +1,85 @@
+// internal/bg/bg.go
+package bg
+
+import (
+	"context"
+	"log"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/dyaksa/archer"
+	"github.com/spf13/viper"
+)
+
+type Jobs struct{ Client *archer.Client }
+
+func archerOptionsFromDSN(dsn string) (*archer.Options, error) {
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	var user, pass string
+	if u.User != nil {
+		user = u.User.Username()
+		pass, _ = u.User.Password()
+	}
+	host := u.Host
+	if !strings.Contains(host, ":") {
+		host = net.JoinHostPort(host, "5432")
+	}
+
+	return &archer.Options{
+		Addr:     host,
+		User:     user,
+		Password: pass,
+		DBName:   strings.TrimPrefix(u.Path, "/"),
+		SSL:      u.Query().Get("sslmode"), // forward sslmode
+	}, nil
+}
+
+func NewJobs() (*Jobs, error) {
+	opts, err := archerOptionsFromDSN(viper.GetString("database.dsn"))
+	if err != nil {
+		return nil, err
+	}
+
+	instances := viper.GetInt("archer.instances")
+	if instances <= 0 {
+		instances = 1
+	}
+	timeoutSec := viper.GetInt("archer.timeoutSec")
+	if timeoutSec <= 0 {
+		timeoutSec = 60
+	}
+
+	// LOG what we’re connecting to (sanitized) so you can confirm DB/host
+	log.Printf("[archer] addr=%s db=%s user=%s ssl=%s", opts.Addr, opts.DBName, opts.User, opts.SSL)
+
+	c := archer.NewClient(
+		opts,
+		archer.WithSetTableName("jobs"), // <- ensure correct table
+		archer.WithSleepInterval(1*time.Second), // fast poll while debugging
+		archer.WithErrHandler(func(err error) { // bubble up worker SQL errors
+			log.Printf("[archer] ERROR: %v", err)
+		}),
+	)
+
+	c.Register(
+		"bootstrap_bastion",
+		BastionBootstrap,
+		archer.WithInstances(instances),
+		archer.WithTimeout(time.Duration(timeoutSec)*time.Second),
+	)
+
+	return &Jobs{Client: c}, nil
+}
+
+func (j *Jobs) Start() error { return j.Client.Start() }
+func (j *Jobs) Stop()        { j.Client.Stop() }
+
+func (j *Jobs) Enqueue(ctx context.Context, id, queue string, args any, opts ...archer.FnOptions) (any, error) {
+	return j.Client.Schedule(ctx, id, queue, args, opts...)
+}

internal/config/config.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/joho/godotenv"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v3"
 )
 
 var File = "config.yaml"
@@ -35,6 +36,9 @@ func Load() {
 
 	viper.SetDefault("frontend.base_url", "http://localhost:5173")
 
+	viper.SetDefault("archer.instances", 2)
+	viper.SetDefault("archer.timeoutSec", 60)
+
 	viper.SetEnvPrefix("AUTOGLUE")
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
@@ -70,10 +74,14 @@ func GetAuthSecret() string {
 
 func DebugPrintConfig() {
 	all := viper.AllSettings()
-	fmt.Println("Loaded configuration:")
-	for k, v := range all {
-		fmt.Printf("%s: %#v\n", k, v)
+
+	b, err := yaml.Marshal(all)
+	if err != nil {
+		fmt.Println("error marshalling config:", err)
+		return
 	}
+	fmt.Println("Loaded configuration:")
+	fmt.Println(string(b))
 }
 
 func IsUIDev() bool {

internal/db/database.go

@@ -31,6 +31,7 @@ func Connect() {
 		&models.Credential{},
 		&models.EmailVerification{},
 		&models.Invitation{},
+		&models.Job{},
 		&models.Label{},
 		&models.MasterKey{},
 		&models.Member{},

internal/db/models/jobs.go

@@ -0,0 +1,22 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/datatypes"
+)
+
+type Job struct {
+	ID            string         `gorm:"type:varchar;primaryKey" json:"id"` // no default; supply from app
+	QueueName     string         `gorm:"type:varchar;not null" json:"queue_name"`
+	Status        string         `gorm:"type:varchar;not null" json:"status"`
+	Arguments     datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'"`
+	Result        datatypes.JSON `gorm:"type:jsonb;not null;default:'{}'"`
+	LastError     *string        `gorm:"type:varchar"`
+	RetryCount    int            `gorm:"not null;default:0"`
+	MaxRetry      int            `gorm:"not null;default:0"`
+	RetryInterval int            `gorm:"not null;default:0"`
+	ScheduledAt   time.Time      `gorm:"type:timestamptz;default:now();index"`
+	StartedAt     *time.Time     `gorm:"type:timestamptz;index"`
+	Timestamped
+}

ui/vite.config.ts

@@ -28,6 +28,7 @@ export default defineConfig({
       "/swagger": "http://localhost:8080",
       "/debug/pprof": "http://localhost:8080",
     },
+      allowedHosts: ['.getexposed.io']
   },
   build: {
     chunkSizeWarningLimit: 1000,