mirror of
https://github.com/GlueOps/autoglue.git
synced 2026-02-13 12:50:05 +01:00
feat: adding hourly backups to s3
Signed-off-by: allanice001 <allanice001@gmail.com>
This commit is contained in:
19
cmd/serve.go
19
cmd/serve.go
@@ -72,6 +72,15 @@ var serveCmd = &cobra.Command{
|
|||||||
archer.WithScheduleTime(next2),
|
archer.WithScheduleTime(next2),
|
||||||
archer.WithMaxRetries(1),
|
archer.WithMaxRetries(1),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_, _ = jobs.Enqueue(
|
||||||
|
context.Background(),
|
||||||
|
uuid.NewString(),
|
||||||
|
"db_backup_s3",
|
||||||
|
bg.DbBackupArgs{},
|
||||||
|
archer.WithMaxRetries(1),
|
||||||
|
archer.WithScheduleTime(time.Now().Add(1*time.Hour)),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Periodic scheduler
|
// Periodic scheduler
|
||||||
@@ -97,16 +106,6 @@ var serveCmd = &cobra.Command{
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("failed to enqueue bootstrap_bastion: %v", err)
|
log.Printf("failed to enqueue bootstrap_bastion: %v", err)
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
_, _ = jobs.Enqueue(
|
|
||||||
context.Background(),
|
|
||||||
uuid.NewString(),
|
|
||||||
"tokens_cleanup",
|
|
||||||
bg.TokensCleanupArgs{},
|
|
||||||
archer.WithMaxRetries(3),
|
|
||||||
archer.WithScheduleTime(time.Now().Add(10*time.Second)),
|
|
||||||
)
|
|
||||||
*/
|
|
||||||
case <-schedCtx.Done():
|
case <-schedCtx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
18
go.mod
18
go.mod
@@ -4,6 +4,10 @@ go 1.25.4
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/alexedwards/argon2id v1.0.0
|
github.com/alexedwards/argon2id v1.0.0
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.39.6
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.31.18
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.18.22
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
|
||||||
github.com/coreos/go-oidc/v3 v3.16.0
|
github.com/coreos/go-oidc/v3 v3.16.0
|
||||||
github.com/dyaksa/archer v1.1.3
|
github.com/dyaksa/archer v1.1.3
|
||||||
github.com/go-chi/chi/v5 v5.2.3
|
github.com/go-chi/chi/v5 v5.2.3
|
||||||
@@ -29,6 +33,20 @@ require (
|
|||||||
require (
|
require (
|
||||||
filippo.io/edwards25519 v1.1.0 // indirect
|
filippo.io/edwards25519 v1.1.0 // indirect
|
||||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0 // indirect
|
||||||
|
github.com/aws/smithy-go v1.23.2 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/gabriel-vasile/mimetype v1.4.10 // indirect
|
github.com/gabriel-vasile/mimetype v1.4.10 // indirect
|
||||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||||
|
|||||||
36
go.sum
36
go.sum
@@ -8,6 +8,42 @@ github.com/agiledragon/gomonkey/v2 v2.3.1 h1:k+UnUY0EMNYUFUAQVETGY9uUTxjMdnUkP0A
|
|||||||
github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
|
github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
|
||||||
github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w=
|
github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w=
|
||||||
github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw=
|
github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.31.18 h1:RouG3AcF2fLFhw+Z0qbnuIl9HZ0Kh4E/U9sKwTMRpMI=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.31.18/go.mod h1:aXZ13mSQC8S2VEHwGfL1COMuJ1Zty6pX5xU7hyqjvCg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.18.22 h1:hyIVGBHhQPaNP9D4BaVRwpjLMCwMMdAkHqB3gGMiykU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.18.22/go.mod h1:B9E2qHs3/YGfeQZ4jrIE/nPvqxtyafZrJ5EQiZBG6pk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0 h1:ZGDJVmlpPFiNFCb/I42nYVKUanJAdFUiSmUo/32AqPQ=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
|
||||||
|
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
|
||||||
|
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||||
github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow=
|
github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow=
|
||||||
github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
|
github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||||
|
|||||||
258
internal/bg/backup_s3.go
Normal file
258
internal/bg/backup_s3.go
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
package bg
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"mime"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/dyaksa/archer"
|
||||||
|
"github.com/dyaksa/archer/job"
|
||||||
|
"github.com/glueops/autoglue/internal/config"
|
||||||
|
"github.com/glueops/autoglue/internal/models"
|
||||||
|
"github.com/glueops/autoglue/internal/utils"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DbBackupArgs struct {
|
||||||
|
// kept in case you want to change retention or add dry-run later
|
||||||
|
}
|
||||||
|
|
||||||
|
type s3Scope struct {
|
||||||
|
Service string `json:"service"`
|
||||||
|
Region string `json:"region"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type encAWS struct {
|
||||||
|
AccessKeyID string `json:"access_key_id"`
|
||||||
|
SecretAccessKey string `json:"secret_access_key"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func DbBackupWorker(db *gorm.DB, jobs *Jobs) archer.WorkerFn {
|
||||||
|
return func(ctx context.Context, j job.Job) (any, error) {
|
||||||
|
if err := DbBackup(ctx, db); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
queue := j.QueueName
|
||||||
|
if strings.TrimSpace(queue) == "" {
|
||||||
|
queue = "db_backup_s3"
|
||||||
|
}
|
||||||
|
|
||||||
|
next := time.Now().UTC().Add(1 * time.Hour)
|
||||||
|
|
||||||
|
payload := DbBackupArgs{}
|
||||||
|
|
||||||
|
opts := []archer.FnOptions{
|
||||||
|
archer.WithScheduleTime(next),
|
||||||
|
archer.WithMaxRetries(1),
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := jobs.Enqueue(ctx, uuid.NewString(), queue, payload, opts...); err != nil {
|
||||||
|
log.Error().Err(err).Str("queue", queue).Time("next", next).Msg("failed to enqueue next db backup")
|
||||||
|
} else {
|
||||||
|
log.Info().Str("queue", queue).Time("next", next).Msg("scheduled next db backup")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func DbBackup(ctx context.Context, db *gorm.DB) error {
|
||||||
|
cfg, err := config.Load()
|
||||||
|
log.Info().Err(err).Msg("loading config")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("load config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cred, sc, err := loadS3Credential(ctx, db)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("load credential: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ak, sk, err := decryptAwsAccessKeys(ctx, db, cred)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("decrypt aws keys: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
region := sc.Region
|
||||||
|
|
||||||
|
if strings.TrimSpace(region) == "" {
|
||||||
|
region = cred.Region
|
||||||
|
if strings.TrimSpace(region) == "" {
|
||||||
|
region = "us-west-1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bucket := strings.ToLower(fmt.Sprintf("%s-autoglue-backups-%s", cred.OrganizationID, region))
|
||||||
|
|
||||||
|
s3cli, err := makeS3Client(ctx, ak, sk, region)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ensureBucket(ctx, s3cli, bucket, region); err != nil {
|
||||||
|
return fmt.Errorf("ensure bucket: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpDir := os.TempDir()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
key := fmt.Sprintf("%04d/%02d/%02d/backup-%02d.sql", now.Year(), now.Month(), now.Day(), now.Hour())
|
||||||
|
outPath := filepath.Join(tmpDir, "autoglue-backup-"+now.Format("20060102T150405Z")+".sql")
|
||||||
|
|
||||||
|
if err := runPgDump(ctx, cfg.DbURL, outPath); err != nil {
|
||||||
|
return fmt.Errorf("pg_dump: %w", err)
|
||||||
|
}
|
||||||
|
defer os.Remove(outPath)
|
||||||
|
|
||||||
|
if err := uploadFileToS3(ctx, s3cli, bucket, key, outPath); err != nil {
|
||||||
|
return fmt.Errorf("s3 upload: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Str("bucket", bucket).Str("key", key).Msg("backup uploaded")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Helpers
|
||||||
|
|
||||||
|
func loadS3Credential(ctx context.Context, db *gorm.DB) (models.Credential, s3Scope, error) {
|
||||||
|
var c models.Credential
|
||||||
|
err := db.
|
||||||
|
WithContext(ctx).
|
||||||
|
Where("provider = ? AND kind = ? AND scope_kind = ?", "aws", "aws_access_key", "service").
|
||||||
|
Where("scope ->> 'service' = ?", "s3").
|
||||||
|
Order("created_at DESC").
|
||||||
|
First(&c).Error
|
||||||
|
if err != nil {
|
||||||
|
return models.Credential{}, s3Scope{}, fmt.Errorf("load credential: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sc s3Scope
|
||||||
|
_ = json.Unmarshal(c.Scope, &sc)
|
||||||
|
return c, sc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decryptAwsAccessKeys(ctx context.Context, db *gorm.DB, c models.Credential) (string, string, error) {
|
||||||
|
plain, err := utils.DecryptForOrg(c.OrganizationID, c.EncryptedData, c.IV, c.Tag, db)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
var payload encAWS
|
||||||
|
if err := json.Unmarshal([]byte(plain), &payload); err != nil {
|
||||||
|
return "", "", fmt.Errorf("parse decrypted payload: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if payload.AccessKeyID == "" || payload.SecretAccessKey == "" {
|
||||||
|
return "", "", errors.New("decrypted payload missing keys")
|
||||||
|
}
|
||||||
|
return payload.AccessKeyID, payload.SecretAccessKey, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeS3Client(ctx context.Context, accessKey, secret, region string) (*s3.Client, error) {
|
||||||
|
staticCredentialsProvider := credentials.NewStaticCredentialsProvider(accessKey, secret, "")
|
||||||
|
cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithCredentialsProvider(staticCredentialsProvider), awsconfig.WithRegion(region))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("aws config: %w", err)
|
||||||
|
}
|
||||||
|
return s3.NewFromConfig(cfg), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureBucket(ctx context.Context, s3cli *s3.Client, bucket, region string) error {
|
||||||
|
_, err := s3cli.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if out, err := s3cli.GetBucketLocation(ctx, &s3.GetBucketLocationInput{Bucket: aws.String(bucket)}); err == nil {
|
||||||
|
existing := string(out.LocationConstraint)
|
||||||
|
if existing == "" {
|
||||||
|
existing = "us-east-1"
|
||||||
|
}
|
||||||
|
if existing != region {
|
||||||
|
return fmt.Errorf("bucket %q already exists in region %q (requested %q)", bucket, existing, region)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create; LocationConstraint except us-east-1
|
||||||
|
in := &s3.CreateBucketInput{Bucket: aws.String(bucket)}
|
||||||
|
if region != "us-east-1" {
|
||||||
|
in.CreateBucketConfiguration = &s3types.CreateBucketConfiguration{
|
||||||
|
LocationConstraint: s3types.BucketLocationConstraint(region),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := s3cli.CreateBucket(ctx, in); err != nil {
|
||||||
|
return fmt.Errorf("create bucket: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// default SSE (best-effort)
|
||||||
|
_, _ = s3cli.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
ServerSideEncryptionConfiguration: &s3types.ServerSideEncryptionConfiguration{
|
||||||
|
Rules: []s3types.ServerSideEncryptionRule{
|
||||||
|
{ApplyServerSideEncryptionByDefault: &s3types.ServerSideEncryptionByDefault{
|
||||||
|
SSEAlgorithm: s3types.ServerSideEncryptionAes256,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runPgDump(ctx context.Context, dbURL, outPath string) error {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"--no-owner",
|
||||||
|
"--no-privileges",
|
||||||
|
"--format=plain",
|
||||||
|
"--file", outPath,
|
||||||
|
dbURL,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "pg_dump", args...)
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return fmt.Errorf("pg_dump failed: %v | %s", err, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadFileToS3(ctx context.Context, s3cli *s3.Client, bucket, key, path string) error {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
info, _ := f.Stat()
|
||||||
|
_, err = s3cli.PutObject(ctx, &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
Body: f,
|
||||||
|
ContentLength: aws.Int64(info.Size()),
|
||||||
|
ContentType: aws.String(mime.TypeByExtension(filepath.Ext(path))),
|
||||||
|
ServerSideEncryption: s3types.ServerSideEncryptionAes256,
|
||||||
|
})
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -67,7 +67,7 @@ func NewJobs(gdb *gorm.DB, dbUrl string) (*Jobs, error) {
|
|||||||
archer.WithSetTableName("jobs"), // <- ensure correct table
|
archer.WithSetTableName("jobs"), // <- ensure correct table
|
||||||
archer.WithSleepInterval(1*time.Second), // fast poll while debugging
|
archer.WithSleepInterval(1*time.Second), // fast poll while debugging
|
||||||
archer.WithErrHandler(func(err error) { // bubble up worker SQL errors
|
archer.WithErrHandler(func(err error) { // bubble up worker SQL errors
|
||||||
log.Printf("[archer] ERROR: %v", err)
|
log.Error().Err(err).Msg("[archer] worker error")
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -94,6 +94,12 @@ func NewJobs(gdb *gorm.DB, dbUrl string) (*Jobs, error) {
|
|||||||
archer.WithTimeout(5*time.Minute),
|
archer.WithTimeout(5*time.Minute),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
c.Register(
|
||||||
|
"db_backup_s3",
|
||||||
|
DbBackupWorker(gdb, jobs),
|
||||||
|
archer.WithInstances(1),
|
||||||
|
archer.WithTimeout(15*time.Minute),
|
||||||
|
)
|
||||||
return jobs, nil
|
return jobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
|
|
||||||
type Credential struct {
|
type Credential struct {
|
||||||
ID uuid.UUID `gorm:"type:uuid;primaryKey;default:gen_random_uuid()"`
|
ID uuid.UUID `gorm:"type:uuid;primaryKey;default:gen_random_uuid()"`
|
||||||
OrganizationID uuid.UUID `gorm:"type:uuid;not null;index" json:"organization_id"`
|
OrganizationID uuid.UUID `gorm:"type:uuid;not null;index;uniqueIndex:uniq_org_provider_scopekind_scope,priority:1" json:"organization_id"`
|
||||||
Organization Organization `gorm:"foreignKey:OrganizationID;constraint:OnDelete:CASCADE" json:"organization"`
|
Organization Organization `gorm:"foreignKey:OrganizationID;constraint:OnDelete:CASCADE" json:"organization"`
|
||||||
Provider string `gorm:"type:varchar(50);not null;uniqueIndex:uniq_org_provider_scopekind_scope,priority:2;index:idx_provider_kind"`
|
Provider string `gorm:"type:varchar(50);not null;uniqueIndex:uniq_org_provider_scopekind_scope,priority:2;index:idx_provider_kind"`
|
||||||
Kind string `gorm:"type:varchar(50);not null;index:idx_provider_kind;index:idx_kind_scope"`
|
Kind string `gorm:"type:varchar(50);not null;index:idx_provider_kind;index:idx_kind_scope"`
|
||||||
|
|||||||
@@ -2,16 +2,7 @@ import { useMemo, useState } from "react"
|
|||||||
import { credentialsApi } from "@/api/credentials"
|
import { credentialsApi } from "@/api/credentials"
|
||||||
import { zodResolver } from "@hookform/resolvers/zod"
|
import { zodResolver } from "@hookform/resolvers/zod"
|
||||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
|
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
|
||||||
import {
|
import { AlertTriangle, Eye, Loader2, MoreHorizontal, Pencil, Plus, Search, Trash2, } from "lucide-react"
|
||||||
AlertTriangle,
|
|
||||||
Eye,
|
|
||||||
Loader2,
|
|
||||||
MoreHorizontal,
|
|
||||||
Pencil,
|
|
||||||
Plus,
|
|
||||||
Search,
|
|
||||||
Trash2,
|
|
||||||
} from "lucide-react"
|
|
||||||
import { Controller, useForm } from "react-hook-form"
|
import { Controller, useForm } from "react-hook-form"
|
||||||
import { toast } from "sonner"
|
import { toast } from "sonner"
|
||||||
import { z } from "zod"
|
import { z } from "zod"
|
||||||
@@ -29,36 +20,16 @@ import {
|
|||||||
} from "@/components/ui/alert-dialog"
|
} from "@/components/ui/alert-dialog"
|
||||||
import { Badge } from "@/components/ui/badge"
|
import { Badge } from "@/components/ui/badge"
|
||||||
import { Button } from "@/components/ui/button"
|
import { Button } from "@/components/ui/button"
|
||||||
import {
|
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, } from "@/components/ui/dialog"
|
||||||
Dialog,
|
|
||||||
DialogContent,
|
|
||||||
DialogFooter,
|
|
||||||
DialogHeader,
|
|
||||||
DialogTitle,
|
|
||||||
DialogTrigger,
|
|
||||||
} from "@/components/ui/dialog"
|
|
||||||
import {
|
import {
|
||||||
DropdownMenu,
|
DropdownMenu,
|
||||||
DropdownMenuContent,
|
DropdownMenuContent,
|
||||||
DropdownMenuItem,
|
DropdownMenuItem,
|
||||||
DropdownMenuTrigger,
|
DropdownMenuTrigger,
|
||||||
} from "@/components/ui/dropdown-menu"
|
} from "@/components/ui/dropdown-menu"
|
||||||
import {
|
import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage, } from "@/components/ui/form"
|
||||||
Form,
|
|
||||||
FormControl,
|
|
||||||
FormField,
|
|
||||||
FormItem,
|
|
||||||
FormLabel,
|
|
||||||
FormMessage,
|
|
||||||
} from "@/components/ui/form"
|
|
||||||
import { Input } from "@/components/ui/input"
|
import { Input } from "@/components/ui/input"
|
||||||
import {
|
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"
|
||||||
Select,
|
|
||||||
SelectContent,
|
|
||||||
SelectItem,
|
|
||||||
SelectTrigger,
|
|
||||||
SelectValue,
|
|
||||||
} from "@/components/ui/select"
|
|
||||||
import { Switch } from "@/components/ui/switch"
|
import { Switch } from "@/components/ui/switch"
|
||||||
import { Textarea } from "@/components/ui/textarea"
|
import { Textarea } from "@/components/ui/textarea"
|
||||||
|
|
||||||
@@ -1381,6 +1352,7 @@ export const CredentialPage = () => {
|
|||||||
</DialogFooter>
|
</DialogFooter>
|
||||||
</DialogContent>
|
</DialogContent>
|
||||||
</Dialog>
|
</Dialog>
|
||||||
|
<pre>{JSON.stringify(credentialQ.data, null, 2)}</pre>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user