π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Remove multipart upload cleanup tool
Manav Rathi49a33757a7d6
diff --git a/server/tools/abort-unfinished-multipart-uploads/README.md b/server/tools/abort-unfinished-multipart-uploads/README.mddeleted file mode 100644index f2d45a0fc1..0000000000--- a/server/tools/abort-unfinished-multipart-uploads/README.md+++ /dev/null@@ -1,19 +0,0 @@-Find all pending multipart uploads, and abort any of them that are older than x days.--This shouldn't be needed in practice since we already track and clear temp_objects. However, in rare cases it might happen that museum gets restarted in the middle of a multipart replication. This tool can be used to list and clean up such stale replication attempts.--## Running--```sh-go run tools/abort-unfinished-multipart-uploads/main.go \- --profile my-profile --endpoint-url https://s3.example.org --bucket my-bucket-```--For more details, see `ParseAndCreateSession`.--To see all the uploads which'll get aborted, you can--```sh-go run tools/abort-unfinished-multipart-uploads/main.go \- --profile p --endpoint-url e --bucket b | grep 'Dry run:'-```diff --git a/server/tools/abort-unfinished-multipart-uploads/main.go b/server/tools/abort-unfinished-multipart-uploads/main.godeleted file mode 100644index 81b97b268b..0000000000--- a/server/tools/abort-unfinished-multipart-uploads/main.go+++ /dev/null@@ -1,85 +0,0 @@-package main--import (- "flag"- "fmt"- "time"-- "github.com/aws/aws-sdk-go/aws"- "github.com/aws/aws-sdk-go/service/s3"- "github.com/ente/museum/tools/pkg/cli"-)--func main() {- bucket := ""- confirm := false-- flag.StringVar(&bucket, "bucket", "",- "Bucket to delete from")-- flag.BoolVar(&confirm, "confirm", false,- "By default, the tool does a dry run. Set this to true to do the actual abort")-- sess, err := cli.ParseAndCreateSession("", false)- if err != nil {- return- }-- if bucket == "" {- fmt.Printf("Error: no bucket specified (hint: use `--bucket`)\n")- return- }-- s3Client := s3.New(sess)-- listOut, err := s3Client.ListMultipartUploads(&s3.ListMultipartUploadsInput{- Bucket: aws.String(bucket),- })- if err != nil {- fmt.Printf("ListMultipartUploads %s error: %s\n", bucket, err)- return- }-- fmt.Printf("ListMultipartUploads: %v\n", listOut)-- if listOut.IsTruncated != nil && *listOut.IsTruncated {- fmt.Printf("Warning: Found more than 1000 pending multipart uploads. We were not expecting this many.")- }-- cutoff := time.Now().AddDate(0, 0, -20)- fmt.Printf("Cutoff: %v\n", cutoff)-- for _, upload := range listOut.Uploads {- fmt.Printf("Processing multipart upload key %v id %v initiated %v\n",- *upload.Key, *upload.UploadId, *upload.Initiated)- if upload.Initiated.After(cutoff) {- fmt.Printf("Skipping multipart upload since it was initated (%v) after cutoff (%v)\n",- *upload.Initiated, cutoff)- continue- }-- if confirm {- abortMultipartUpload(s3Client, bucket, *upload.Key, *upload.UploadId)- } else {- fmt.Printf("Dry run: AbortMultipartUpload: %v/%v/%v\n", bucket,- *upload.Key,- *upload.UploadId)- }- }-}--func abortMultipartUpload(s3Client *s3.S3, bucket string, key string, uploadId string) error {- _, err := s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{- Bucket: &bucket,- Key: &key,- UploadId: &uploadId,- })- if err != nil {- fmt.Printf("AbortMultipartUpload failed: key %v id %v: %v\n", key, uploadId, err)- return err- }-- fmt.Printf("AbortMultipartUpload success key %v id %v\n",- key, uploadId)- return nil-}diff --git a/server/tools/pkg/cli/aws.go b/server/tools/pkg/cli/aws.godeleted file mode 100644index 9007591c2e..0000000000--- a/server/tools/pkg/cli/aws.go+++ /dev/null@@ -1,47 +0,0 @@-package cli--import (- "flag"- "fmt"- "os"-- "github.com/aws/aws-sdk-go/aws"- "github.com/aws/aws-sdk-go/aws/session"-)--func ParseAndCreateSession(endpointURL string, S3ForcePathStyle bool) (*session.Session, error) {- logLevel := aws.LogDebugWithHTTPBody- cliProfile := flag.String("profile", "AWS_PROFILE",- "The profile to use from the S3 config file")- cliEndpointURL := flag.String("endpoint-url", "",- "The root URL of the S3 compatible API (excluding the bucket)")- flag.Parse()-- profile := *cliProfile- if profile == "" {- profile = os.Getenv("AWS_PROFILE")- }-- if *cliEndpointURL != "" {- endpointURL = *cliEndpointURL- }-- fmt.Printf("Using profile %s, endpoint %s\n", profile, endpointURL)-- sess, err := session.NewSessionWithOptions(session.Options{- Profile: profile,- // Needed to read region from .aws/profile- SharedConfigState: session.SharedConfigEnable,- Config: aws.Config{- Endpoint: aws.String(endpointURL),- S3ForcePathStyle: aws.Bool(S3ForcePathStyle),- LogLevel: &logLevel,- },- })- if err != nil {- fmt.Printf("NewSessionWithOptions error: %s\n", err)- return sess, err- }-- return sess, err-}