Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Remove multipart upload cleanup tool

Manav Rathi49a33757a7d6
diff --git a/server/tools/abort-unfinished-multipart-uploads/README.md b/server/tools/abort-unfinished-multipart-uploads/README.md
deleted file mode 100644
index f2d45a0fc1..0000000000
--- a/server/tools/abort-unfinished-multipart-uploads/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-Find all pending multipart uploads, and abort any of them that are older than x days.
-
-This shouldn't be needed in practice since we already track and clear temp_objects. However, in rare cases it might happen that museum gets restarted in the middle of a multipart replication. This tool can be used to list and clean up such stale replication attempts.
-
-## Running
-
-```sh
-go run tools/abort-unfinished-multipart-uploads/main.go \
- --profile my-profile --endpoint-url https://s3.example.org --bucket my-bucket
-```
-
-For more details, see `ParseAndCreateSession`.
-
-To see all the uploads which'll get aborted, you can
-
-```sh
-go run tools/abort-unfinished-multipart-uploads/main.go \
- --profile p --endpoint-url e --bucket b | grep 'Dry run:'
-```
diff --git a/server/tools/abort-unfinished-multipart-uploads/main.go b/server/tools/abort-unfinished-multipart-uploads/main.go
deleted file mode 100644
index 81b97b268b..0000000000
--- a/server/tools/abort-unfinished-multipart-uploads/main.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "time"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/ente/museum/tools/pkg/cli"
-)
-
-func main() {
- bucket := ""
- confirm := false
-
- flag.StringVar(&bucket, "bucket", "",
- "Bucket to delete from")
-
- flag.BoolVar(&confirm, "confirm", false,
- "By default, the tool does a dry run. Set this to true to do the actual abort")
-
- sess, err := cli.ParseAndCreateSession("", false)
- if err != nil {
- return
- }
-
- if bucket == "" {
- fmt.Printf("Error: no bucket specified (hint: use `--bucket`)\n")
- return
- }
-
- s3Client := s3.New(sess)
-
- listOut, err := s3Client.ListMultipartUploads(&s3.ListMultipartUploadsInput{
- Bucket: aws.String(bucket),
- })
- if err != nil {
- fmt.Printf("ListMultipartUploads %s error: %s\n", bucket, err)
- return
- }
-
- fmt.Printf("ListMultipartUploads: %v\n", listOut)
-
- if listOut.IsTruncated != nil && *listOut.IsTruncated {
- fmt.Printf("Warning: Found more than 1000 pending multipart uploads. We were not expecting this many.")
- }
-
- cutoff := time.Now().AddDate(0, 0, -20)
- fmt.Printf("Cutoff: %v\n", cutoff)
-
- for _, upload := range listOut.Uploads {
- fmt.Printf("Processing multipart upload key %v id %v initiated %v\n",
- *upload.Key, *upload.UploadId, *upload.Initiated)
- if upload.Initiated.After(cutoff) {
- fmt.Printf("Skipping multipart upload since it was initated (%v) after cutoff (%v)\n",
- *upload.Initiated, cutoff)
- continue
- }
-
- if confirm {
- abortMultipartUpload(s3Client, bucket, *upload.Key, *upload.UploadId)
- } else {
- fmt.Printf("Dry run: AbortMultipartUpload: %v/%v/%v\n", bucket,
- *upload.Key,
- *upload.UploadId)
- }
- }
-}
-
-func abortMultipartUpload(s3Client *s3.S3, bucket string, key string, uploadId string) error {
- _, err := s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
- Bucket: &bucket,
- Key: &key,
- UploadId: &uploadId,
- })
- if err != nil {
- fmt.Printf("AbortMultipartUpload failed: key %v id %v: %v\n", key, uploadId, err)
- return err
- }
-
- fmt.Printf("AbortMultipartUpload success key %v id %v\n",
- key, uploadId)
- return nil
-}
diff --git a/server/tools/pkg/cli/aws.go b/server/tools/pkg/cli/aws.go
deleted file mode 100644
index 9007591c2e..0000000000
--- a/server/tools/pkg/cli/aws.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package cli
-
-import (
- "flag"
- "fmt"
- "os"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/session"
-)
-
-func ParseAndCreateSession(endpointURL string, S3ForcePathStyle bool) (*session.Session, error) {
- logLevel := aws.LogDebugWithHTTPBody
- cliProfile := flag.String("profile", "AWS_PROFILE",
- "The profile to use from the S3 config file")
- cliEndpointURL := flag.String("endpoint-url", "",
- "The root URL of the S3 compatible API (excluding the bucket)")
- flag.Parse()
-
- profile := *cliProfile
- if profile == "" {
- profile = os.Getenv("AWS_PROFILE")
- }
-
- if *cliEndpointURL != "" {
- endpointURL = *cliEndpointURL
- }
-
- fmt.Printf("Using profile %s, endpoint %s\n", profile, endpointURL)
-
- sess, err := session.NewSessionWithOptions(session.Options{
- Profile: profile,
- // Needed to read region from .aws/profile
- SharedConfigState: session.SharedConfigEnable,
- Config: aws.Config{
- Endpoint: aws.String(endpointURL),
- S3ForcePathStyle: aws.Bool(S3ForcePathStyle),
- LogLevel: &logLevel,
- },
- })
- if err != nil {
- fmt.Printf("NewSessionWithOptions error: %s\n", err)
- return sess, err
- }
-
- return sess, err
-}