Skip to content

Commit 23199ac

Browse files
harshavardhanaharshavardhana
authored
Support GCS bulk delete with serialized single DELETE requests (#2185)
1 parent 89b666c commit 23199ac

4 files changed

Lines changed: 84 additions & 7 deletions

File tree

‎.github/workflows/go-windows.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
runs-on: ${{ matrix.os }}
1818
strategy:
1919
matrix:
20-
go-version: [1.23.x, 1.24.x]
20+
go-version: [1.24.x, 1.25.x]
2121
os: [windows-latest]
2222
steps:
2323
- name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

‎.github/workflows/go.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
runs-on: ${{ matrix.os }}
1818
strategy:
1919
matrix:
20-
go-version: [1.23.x, 1.24.x]
20+
go-version: [1.24.x, 1.25.x]
2121
os: [ubuntu-latest]
2222
steps:
2323
- name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

‎api-remove.go‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ import (
3030
"github.com/minio/minio-go/v7/pkg/s3utils"
3131
)
3232

33+
// useMultiDeleteForBulkDelete returns true if the client should use
34+
// multi-object delete API for bulk delete operations. Returns false
35+
// for endpoints that do not support multi-object delete (e.g., GCS).
36+
func (c *Client) useMultiDeleteForBulkDelete() bool {
37+
// NOTE: GCS does not support multi-object delete API.
38+
return !s3utils.IsGoogleEndpoint(*c.endpointURL)
39+
}
40+
3341
//revive:disable
3442

3543
// Deprecated: BucketOptions will be renamed to RemoveBucketOptions in future versions.
@@ -411,6 +419,12 @@ func hasInvalidXMLChar(str string) bool {
411419

412420
// Generate and call MultiDelete S3 requests based on entries received from the iterator.
413421
func (c *Client) removeObjectsIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], yield func(RemoveObjectResult) bool, opts RemoveObjectsOptions) {
422+
// NOTE: GCS does not support multi-object delete, use single DELETE requests.
423+
if !c.useMultiDeleteForBulkDelete() {
424+
c.removeObjectsSingleIter(ctx, bucketName, objectsIter, yield, opts)
425+
return
426+
}
427+
414428
maxEntries := 1000
415429
urlValues := make(url.Values)
416430
urlValues.Set("delete", "")
@@ -549,14 +563,20 @@ func (c *Client) removeObjectsIter(ctx context.Context, bucketName string, objec
549563

550564
// Generate and call MultiDelete S3 requests based on entries received from objectsCh
551565
func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) {
566+
// Close result channel when delete finishes.
567+
defer close(resultCh)
568+
569+
// NOTE: GCS does not support multi-object delete, use single DELETE requests.
570+
if !c.useMultiDeleteForBulkDelete() {
571+
c.removeObjectsSingle(ctx, bucketName, objectsCh, resultCh, opts)
572+
return
573+
}
574+
552575
maxEntries := 1000
553576
finish := false
554577
urlValues := make(url.Values)
555578
urlValues.Set("delete", "")
556579

557-
// Close result channel when Multi delete finishes.
558-
defer close(resultCh)
559-
560580
// Loop over entries by 1000 and call MultiDelete requests
561581
for !finish {
562582
count := 0
@@ -640,6 +660,63 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
640660
}
641661
}
642662

663+
// removeObjectsSingle deletes objects one by one using single DELETE requests.
664+
// This is used for endpoints that do not support multi-object delete (e.g., GCS).
665+
func (c *Client) removeObjectsSingle(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) {
666+
for {
667+
select {
668+
case <-ctx.Done():
669+
return
670+
case object, ok := <-objectsCh:
671+
if !ok {
672+
return
673+
}
674+
removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
675+
VersionID: object.VersionID,
676+
GovernanceBypass: opts.GovernanceBypass,
677+
})
678+
if err := removeResult.Err; err != nil {
679+
// Version/object does not exist is not an error, ignore and continue.
680+
switch ToErrorResponse(err).Code {
681+
case NoSuchVersion, NoSuchKey:
682+
continue
683+
}
684+
}
685+
select {
686+
case <-ctx.Done():
687+
return
688+
case resultCh <- removeResult:
689+
}
690+
}
691+
}
692+
}
693+
694+
// removeObjectsSingleIter deletes objects one by one using single DELETE requests.
695+
// This is used for endpoints that do not support multi-object delete (e.g., GCS).
696+
func (c *Client) removeObjectsSingleIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], yield func(RemoveObjectResult) bool, opts RemoveObjectsOptions) {
697+
for object := range objectsIter {
698+
select {
699+
case <-ctx.Done():
700+
return
701+
default:
702+
}
703+
removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
704+
VersionID: object.VersionID,
705+
GovernanceBypass: opts.GovernanceBypass,
706+
})
707+
if err := removeResult.Err; err != nil {
708+
// Version/object does not exist is not an error, ignore and continue.
709+
switch ToErrorResponse(err).Code {
710+
case NoSuchVersion, NoSuchKey:
711+
continue
712+
}
713+
}
714+
if !yield(removeResult) {
715+
return
716+
}
717+
}
718+
}
719+
643720
// RemoveIncompleteUpload aborts an partially uploaded object.
644721
func (c *Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error {
645722
// Input validation.

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
module github.com/minio/minio-go/v7
22

3-
go 1.23.0
3+
go 1.24.0
44

5-
toolchain go1.24.3
5+
toolchain go1.25.4
66

77
require (
88
github.com/dustin/go-humanize v1.0.1

0 commit comments

Comments
 (0)