GlueOps/autoglue
1
0
Fork 0
mirror of https://github.com/GlueOps/autoglue.git synced 2026-02-13 04:40:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: adding hourly backups to s3

Browse Source 
Signed-off-by: allanice001 <allanice001@gmail.com>
This commit is contained in:
allanice001
2025-11-12 05:33:09 +00:00
parent 586e51b8cc
commit ad8141a497
7 changed files with 334 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/serve.go
View File

@@ -72,6 +72,15 @@ var serveCmd = &cobra.Command{
archer.WithScheduleTime(next2),
archer.WithMaxRetries(1),
)
_, _ = jobs.Enqueue(
context.Background(),
uuid.NewString(),
"db_backup_s3",
bg.DbBackupArgs{},
archer.WithMaxRetries(1),
archer.WithScheduleTime(time.Now().Add(1*time.Hour)),
)
}
// Periodic scheduler
@@ -97,16 +106,6 @@ var serveCmd = &cobra.Command{
if err != nil {
log.Printf("failed to enqueue bootstrap_bastion: %v", err)
}
/*
_, _ = jobs.Enqueue(
context.Background(),
uuid.NewString(),
"tokens_cleanup",
bg.TokensCleanupArgs{},
archer.WithMaxRetries(3),
archer.WithScheduleTime(time.Now().Add(10*time.Second)),
)
*/
case <-schedCtx.Done():
return
}

18
go.mod
View File

@@ -4,6 +4,10 @@ go 1.25.4
require (
github.com/alexedwards/argon2id v1.0.0
github.com/aws/aws-sdk-go-v2 v1.39.6
github.com/aws/aws-sdk-go-v2/config v1.31.18
github.com/aws/aws-sdk-go-v2/credentials v1.18.22
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
github.com/coreos/go-oidc/v3 v3.16.0
github.com/dyaksa/archer v1.1.3
github.com/go-chi/chi/v5 v5.2.3
@@ -29,6 +33,20 @@ require (
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0 // indirect
github.com/aws/smithy-go v1.23.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.10 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect

36
go.sum
View File

@@ -8,6 +8,42 @@ github.com/agiledragon/gomonkey/v2 v2.3.1 h1:k+UnUY0EMNYUFUAQVETGY9uUTxjMdnUkP0A
github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w=
github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw=
github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
github.com/aws/aws-sdk-go-v2/config v1.31.18 h1:RouG3AcF2fLFhw+Z0qbnuIl9HZ0Kh4E/U9sKwTMRpMI=
github.com/aws/aws-sdk-go-v2/config v1.31.18/go.mod h1:aXZ13mSQC8S2VEHwGfL1COMuJ1Zty6pX5xU7hyqjvCg=
github.com/aws/aws-sdk-go-v2/credentials v1.18.22 h1:hyIVGBHhQPaNP9D4BaVRwpjLMCwMMdAkHqB3gGMiykU=
github.com/aws/aws-sdk-go-v2/credentials v1.18.22/go.mod h1:B9E2qHs3/YGfeQZ4jrIE/nPvqxtyafZrJ5EQiZBG6pk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0 h1:ZGDJVmlpPFiNFCb/I42nYVKUanJAdFUiSmUo/32AqPQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.40.0/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow=
github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

258
internal/bg/backup_s3.go Normal file
View File

@@ -0,0 +1,258 @@
package bg
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"mime"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/dyaksa/archer"
"github.com/dyaksa/archer/job"
"github.com/glueops/autoglue/internal/config"
"github.com/glueops/autoglue/internal/models"
"github.com/glueops/autoglue/internal/utils"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"gorm.io/gorm"
)
type DbBackupArgs struct {
// kept in case you want to change retention or add dry-run later
}
type s3Scope struct {
Service string `json:"service"`
Region string `json:"region"`
}
type encAWS struct {
AccessKeyID string `json:"access_key_id"`
SecretAccessKey string `json:"secret_access_key"`
}
func DbBackupWorker(db *gorm.DB, jobs *Jobs) archer.WorkerFn {
return func(ctx context.Context, j job.Job) (any, error) {
if err := DbBackup(ctx, db); err != nil {
return nil, err
}
queue := j.QueueName
if strings.TrimSpace(queue) == "" {
queue = "db_backup_s3"
}
next := time.Now().UTC().Add(1 * time.Hour)
payload := DbBackupArgs{}
opts := []archer.FnOptions{
archer.WithScheduleTime(next),
archer.WithMaxRetries(1),
}
if _, err := jobs.Enqueue(ctx, uuid.NewString(), queue, payload, opts...); err != nil {
log.Error().Err(err).Str("queue", queue).Time("next", next).Msg("failed to enqueue next db backup")
} else {
log.Info().Str("queue", queue).Time("next", next).Msg("scheduled next db backup")
}
return nil, nil
}
}
func DbBackup(ctx context.Context, db *gorm.DB) error {
cfg, err := config.Load()
log.Info().Err(err).Msg("loading config")
if err != nil {
return fmt.Errorf("load config: %w", err)
}
cred, sc, err := loadS3Credential(ctx, db)
if err != nil {
return fmt.Errorf("load credential: %w", err)
}
ak, sk, err := decryptAwsAccessKeys(ctx, db, cred)
if err != nil {
return fmt.Errorf("decrypt aws keys: %w", err)
}
region := sc.Region
if strings.TrimSpace(region) == "" {
region = cred.Region
if strings.TrimSpace(region) == "" {
region = "us-west-1"
}
}
bucket := strings.ToLower(fmt.Sprintf("%s-autoglue-backups-%s", cred.OrganizationID, region))
s3cli, err := makeS3Client(ctx, ak, sk, region)
if err != nil {
return err
}
if err := ensureBucket(ctx, s3cli, bucket, region); err != nil {
return fmt.Errorf("ensure bucket: %w", err)
}
tmpDir := os.TempDir()
now := time.Now().UTC()
key := fmt.Sprintf("%04d/%02d/%02d/backup-%02d.sql", now.Year(), now.Month(), now.Day(), now.Hour())
outPath := filepath.Join(tmpDir, "autoglue-backup-"+now.Format("20060102T150405Z")+".sql")
if err := runPgDump(ctx, cfg.DbURL, outPath); err != nil {
return fmt.Errorf("pg_dump: %w", err)
}
defer os.Remove(outPath)
if err := uploadFileToS3(ctx, s3cli, bucket, key, outPath); err != nil {
return fmt.Errorf("s3 upload: %w", err)
}
log.Info().Str("bucket", bucket).Str("key", key).Msg("backup uploaded")
return nil
}
// --- Helpers
func loadS3Credential(ctx context.Context, db *gorm.DB) (models.Credential, s3Scope, error) {
var c models.Credential
err := db.
WithContext(ctx).
Where("provider = ? AND kind = ? AND scope_kind = ?", "aws", "aws_access_key", "service").
Where("scope ->> 'service' = ?", "s3").
Order("created_at DESC").
First(&c).Error
if err != nil {
return models.Credential{}, s3Scope{}, fmt.Errorf("load credential: %w", err)
}
var sc s3Scope
_ = json.Unmarshal(c.Scope, &sc)
return c, sc, nil
}
func decryptAwsAccessKeys(ctx context.Context, db *gorm.DB, c models.Credential) (string, string, error) {
plain, err := utils.DecryptForOrg(c.OrganizationID, c.EncryptedData, c.IV, c.Tag, db)
if err != nil {
return "", "", err
}
var payload encAWS
if err := json.Unmarshal([]byte(plain), &payload); err != nil {
return "", "", fmt.Errorf("parse decrypted payload: %w", err)
}
if payload.AccessKeyID == "" || payload.SecretAccessKey == "" {
return "", "", errors.New("decrypted payload missing keys")
}
return payload.AccessKeyID, payload.SecretAccessKey, nil
}
func makeS3Client(ctx context.Context, accessKey, secret, region string) (*s3.Client, error) {
staticCredentialsProvider := credentials.NewStaticCredentialsProvider(accessKey, secret, "")
cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithCredentialsProvider(staticCredentialsProvider), awsconfig.WithRegion(region))
if err != nil {
return nil, fmt.Errorf("aws config: %w", err)
}
return s3.NewFromConfig(cfg), nil
}
func ensureBucket(ctx context.Context, s3cli *s3.Client, bucket, region string) error {
_, err := s3cli.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
if err == nil {
return nil
}
if out, err := s3cli.GetBucketLocation(ctx, &s3.GetBucketLocationInput{Bucket: aws.String(bucket)}); err == nil {
existing := string(out.LocationConstraint)
if existing == "" {
existing = "us-east-1"
}
if existing != region {
return fmt.Errorf("bucket %q already exists in region %q (requested %q)", bucket, existing, region)
}
}
// Create; LocationConstraint except us-east-1
in := &s3.CreateBucketInput{Bucket: aws.String(bucket)}
if region != "us-east-1" {
in.CreateBucketConfiguration = &s3types.CreateBucketConfiguration{
LocationConstraint: s3types.BucketLocationConstraint(region),
}
}
if _, err := s3cli.CreateBucket(ctx, in); err != nil {
return fmt.Errorf("create bucket: %w", err)
}
// default SSE (best-effort)
_, _ = s3cli.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
Bucket: aws.String(bucket),
ServerSideEncryptionConfiguration: &s3types.ServerSideEncryptionConfiguration{
Rules: []s3types.ServerSideEncryptionRule{
{ApplyServerSideEncryptionByDefault: &s3types.ServerSideEncryptionByDefault{
SSEAlgorithm: s3types.ServerSideEncryptionAes256,
}},
},
},
})
return nil
}
func runPgDump(ctx context.Context, dbURL, outPath string) error {
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
return err
}
args := []string{
"--no-owner",
"--no-privileges",
"--format=plain",
"--file", outPath,
dbURL,
}
cmd := exec.CommandContext(ctx, "pg_dump", args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("pg_dump failed: %v | %s", err, stderr.String())
}
return nil
}
func uploadFileToS3(ctx context.Context, s3cli *s3.Client, bucket, key, path string) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
info, _ := f.Stat()
_, err = s3cli.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: f,
ContentLength: aws.Int64(info.Size()),
ContentType: aws.String(mime.TypeByExtension(filepath.Ext(path))),
ServerSideEncryption: s3types.ServerSideEncryptionAes256,
})
return err
}

8
internal/bg/bg.go
View File

@@ -67,7 +67,7 @@ func NewJobs(gdb *gorm.DB, dbUrl string) (*Jobs, error) {
archer.WithSetTableName("jobs"), // <- ensure correct table
archer.WithSleepInterval(1*time.Second), // fast poll while debugging
archer.WithErrHandler(func(err error) { // bubble up worker SQL errors
log.Printf("[archer] ERROR: %v", err)
log.Error().Err(err).Msg("[archer] worker error")
}),
)
@@ -94,6 +94,12 @@ func NewJobs(gdb *gorm.DB, dbUrl string) (*Jobs, error) {
archer.WithTimeout(5*time.Minute),
)
c.Register(
"db_backup_s3",
DbBackupWorker(gdb, jobs),
archer.WithInstances(1),
archer.WithTimeout(15*time.Minute),
)
return jobs, nil
}

2
internal/models/credential.go
View File

@@ -9,7 +9,7 @@ import (
type Credential struct {
ID uuid.UUID `gorm:"type:uuid;primaryKey;default:gen_random_uuid()"`
OrganizationID uuid.UUID `gorm:"type:uuid;not null;index" json:"organization_id"`
OrganizationID uuid.UUID `gorm:"type:uuid;not null;index;uniqueIndex:uniq_org_provider_scopekind_scope,priority:1" json:"organization_id"`
Organization Organization `gorm:"foreignKey:OrganizationID;constraint:OnDelete:CASCADE" json:"organization"`
Provider string `gorm:"type:varchar(50);not null;uniqueIndex:uniq_org_provider_scopekind_scope,priority:2;index:idx_provider_kind"`
Kind string `gorm:"type:varchar(50);not null;index:idx_provider_kind;index:idx_kind_scope"`

38
ui/src/pages/credentials/credential-page.tsx
View File

@@ -2,16 +2,7 @@ import { useMemo, useState } from "react"
import { credentialsApi } from "@/api/credentials"
import { zodResolver } from "@hookform/resolvers/zod"
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
import {
AlertTriangle,
Eye,
Loader2,
MoreHorizontal,
Pencil,
Plus,
Search,
Trash2,
} from "lucide-react"
import { AlertTriangle, Eye, Loader2, MoreHorizontal, Pencil, Plus, Search, Trash2, } from "lucide-react"
import { Controller, useForm } from "react-hook-form"
import { toast } from "sonner"
import { z } from "zod"
@@ -29,36 +20,16 @@ import {
} from "@/components/ui/alert-dialog"
import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog"
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, } from "@/components/ui/dialog"
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form"
import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage, } from "@/components/ui/form"
import { Input } from "@/components/ui/input"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"
import { Switch } from "@/components/ui/switch"
import { Textarea } from "@/components/ui/textarea"
@@ -1381,6 +1352,7 @@ export const CredentialPage = () => {
</DialogFooter>
</DialogContent>
</Dialog>
<pre>{JSON.stringify(credentialQ.data, null, 2)}</pre>
</div>
)
}