Skip to content

Commit d75e965

Browse files
david-wagihdavid-wagih
authored
add S3 Outposts support: Implement request signing and endpoint validation for S3 on Outposts (#2201)
1 parent d00bd2f commit d75e965

11 files changed

Lines changed: 335 additions & 37 deletions

‎api.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
198198
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
199199
// for backwards compatibility.
200200
clnt.s3DualstackEnabled = true
201+
} else if s3utils.IsAmazonOutpostsEndpoint(*clnt.endpointURL) {
202+
// S3 on Outposts uses signature v4 with service name s3-outposts.
203+
clnt.overrideSignerType = credentials.SignatureV4
201204
}
202205

203206
return clnt, nil
@@ -912,7 +915,11 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
912915
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
913916
} else if signerType.IsV4() {
914917
// Presign URL with signature v4.
915-
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
918+
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
919+
req = signer.PreSignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
920+
} else {
921+
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
922+
}
916923
}
917924
return req, nil
918925
}
@@ -971,6 +978,9 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
971978
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
972979
req = signer.StreamingSignV4Express(req, accessKeyID,
973980
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
981+
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
982+
req = signer.StreamingSignV4Outposts(req, accessKeyID,
983+
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
974984
} else {
975985
req = signer.StreamingSignV4(req, accessKeyID,
976986
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
@@ -991,6 +1001,8 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
9911001

9921002
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
9931003
req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
1004+
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
1005+
req = signer.SignV4TrailerOutposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
9941006
} else {
9951007
// Add signature version '4' authorization header.
9961008
req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)

‎bucket-cache.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
209209
}
210210

211211
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
212-
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
212+
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
213+
region := getDefaultLocation(*c.endpointURL, c.region)
214+
req = signer.SignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, region)
215+
} else {
216+
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
217+
}
213218
return req, nil
214219
}

‎examples/s3/s3-outposts-put-get.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
//go:build example
2+
// +build example
3+
4+
/*
5+
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
6+
* Copyright 2015-2024 MinIO, Inc.
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
// This example uses the minio-go client against AWS S3 on Outposts.
22+
// It performs a PutObject and GetObject to verify the client works with
23+
// Outposts endpoints (s3-outposts signing and path-style requests).
24+
//
25+
// Set environment variables (do not commit real credentials):
26+
//
27+
// S3_OUTPOSTS_ENDPOINT - Outposts access point endpoint (e.g. myap-123.op-xxx.s3-outposts.region.amazonaws.com)
28+
// S3_OUTPOSTS_BUCKET - Access point alias / bucket name (e.g. mybucket--op-xxx--op-s3)
29+
// S3_OUTPOSTS_REGION - AWS region (e.g. eu-central-1). Optional if AWS_REGION is set.
30+
// S3_OUTPOSTS_PROFILE - Optional. AWS credentials profile. If unset, uses AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY.
31+
// AWS_ACCESS_KEY_ID - Required if S3_OUTPOSTS_PROFILE is unset
32+
// AWS_SECRET_ACCESS_KEY - Required if S3_OUTPOSTS_PROFILE is unset
33+
//
34+
// Run from repo root (uses local minio-go):
35+
//
36+
// go run -tags example ./examples/s3/s3-outposts-put-get.go
37+
//
38+
// Or from examples/s3 (replace in go.mod points to parent minio-go):
39+
//
40+
// cd examples/s3 && go run -tags example s3-outposts-put-get.go
41+
package main
42+
43+
import (
44+
"context"
45+
"fmt"
46+
"io"
47+
"log"
48+
"os"
49+
"strings"
50+
51+
"github.com/minio/minio-go/v7"
52+
"github.com/minio/minio-go/v7/pkg/credentials"
53+
)
54+
55+
func main() {
56+
endpoint := os.Getenv("S3_OUTPOSTS_ENDPOINT")
57+
bucket := os.Getenv("S3_OUTPOSTS_BUCKET")
58+
region := os.Getenv("S3_OUTPOSTS_REGION")
59+
if region == "" {
60+
region = os.Getenv("AWS_REGION")
61+
}
62+
profile := os.Getenv("S3_OUTPOSTS_PROFILE")
63+
64+
if endpoint == "" || bucket == "" || region == "" {
65+
log.Fatalf("Missing required env: set S3_OUTPOSTS_ENDPOINT, S3_OUTPOSTS_BUCKET, and S3_OUTPOSTS_REGION (or AWS_REGION)")
66+
}
67+
68+
var creds *credentials.Credentials
69+
if profile != "" {
70+
creds = credentials.NewFileAWSCredentials("", profile)
71+
} else {
72+
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
73+
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
74+
if accessKey == "" || secretKey == "" {
75+
log.Fatalf("Set S3_OUTPOSTS_PROFILE or both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY")
76+
}
77+
creds = credentials.NewStaticV4(accessKey, secretKey, "")
78+
}
79+
80+
client, err := minio.New(endpoint, &minio.Options{
81+
Creds: creds,
82+
Secure: true,
83+
Region: region,
84+
})
85+
if err != nil {
86+
log.Fatalf("New client: %v", err)
87+
}
88+
89+
objectKey := "outposts-test/hello-minio-go.txt"
90+
objectBody := "Hello from minio-go S3 on Outposts\n"
91+
92+
ctx := context.Background()
93+
94+
fmt.Println("PutObject...")
95+
_, err = client.PutObject(ctx, bucket, objectKey, strings.NewReader(objectBody), int64(len(objectBody)), minio.PutObjectOptions{
96+
ContentType: "text/plain",
97+
})
98+
if err != nil {
99+
log.Fatalf("PutObject: %v", err)
100+
}
101+
fmt.Println("PutObject OK")
102+
103+
fmt.Println("GetObject...")
104+
obj, err := client.GetObject(ctx, bucket, objectKey, minio.GetObjectOptions{})
105+
if err != nil {
106+
log.Fatalf("GetObject: %v", err)
107+
}
108+
defer obj.Close()
109+
110+
data, err := io.ReadAll(obj)
111+
if err != nil {
112+
log.Fatalf("Read body: %v", err)
113+
}
114+
fmt.Println("GetObject OK")
115+
fmt.Println("Content:", string(data))
116+
fmt.Println("Done. MinIO client works with S3 on Outposts.")
117+
}

‎pkg/s3utils/utils.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
119119
// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style
120120
var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`)
121121

122+
// amazonS3HostOutposts - regular expression used to determine if an arg is S3 on Outposts endpoint.
123+
// Pattern: <something>.s3-outposts.<region>.amazonaws.com
124+
var amazonS3HostOutposts = regexp.MustCompile(`^(.+)\.s3-outposts\.([a-z0-9-]+)\.amazonaws\.com$`)
125+
122126
// GetRegionFromURL - returns a region from url host.
123127
func GetRegionFromURL(endpointURL url.URL) string {
124128
if endpointURL == sentinelURL {
@@ -181,6 +185,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
181185
return parts[1]
182186
}
183187

188+
parts = amazonS3HostOutposts.FindStringSubmatch(endpointURL.Hostname())
189+
if len(parts) > 2 {
190+
return parts[2]
191+
}
192+
184193
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
185194
if len(parts) > 1 {
186195
if strings.HasPrefix(parts[1], "xpress-") {
@@ -210,8 +219,20 @@ func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool {
210219
return amazonS3HostExpress.MatchString(endpointURL.Hostname())
211220
}
212221

222+
// IsAmazonOutpostsEndpoint - Match if the endpoint is S3 on Outposts endpoint.
223+
func IsAmazonOutpostsEndpoint(endpointURL url.URL) bool {
224+
if endpointURL == sentinelURL {
225+
return false
226+
}
227+
return amazonS3HostOutposts.MatchString(endpointURL.Hostname())
228+
}
229+
213230
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
231+
// S3 on Outposts is not treated as Amazon S3 here so that the client keeps path-style and does not replace the host.
214232
func IsAmazonEndpoint(endpointURL url.URL) bool {
233+
if IsAmazonOutpostsEndpoint(endpointURL) {
234+
return false
235+
}
215236
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" {
216237
return true
217238
}

‎pkg/s3utils/utils_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ func TestGetRegionFromURL(t *testing.T) {
5757
{u: "s3express-euc1-az2.eu-central-1.amazonaws.com", expectedRegion: "eu-central-1"},
5858
{u: "s3express-usgw1-az3.us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"},
5959
{u: "s3express-control.us-west-2.amazonaws.com", expectedRegion: "us-west-2"},
60+
// S3 on Outposts.
61+
{u: "test-access-point-000000000000.op-00000000000000000.s3-outposts.eu-central-1.amazonaws.com", expectedRegion: "eu-central-1"},
62+
{u: "myap-123456789012.op-0ab1c2d3e4f5.s3-outposts.us-west-2.amazonaws.com", expectedRegion: "us-west-2"},
6063

6164
// Test cases with port numbers.
6265
{u: "storage.googleapis.com:80", expectedRegion: ""},
@@ -176,6 +179,32 @@ func TestIsVirtualHostSupported(t *testing.T) {
176179
}
177180
}
178181

182+
// Tests validate S3 Outposts endpoint detector.
183+
func TestIsAmazonOutpostsEndpoint(t *testing.T) {
184+
testCases := []struct {
185+
url string
186+
result bool
187+
}{
188+
{"https://s3.amazonaws.com", false},
189+
{"https://s3.eu-west-1.amazonaws.com", false},
190+
{"https://storage.googleapis.com", false},
191+
{"https://test-access-point-000000000000.op-00000000000000000.s3-outposts.eu-central-1.amazonaws.com", true},
192+
{"https://myap-123456789012.op-0ab1c2d3e4f5.s3-outposts.us-west-2.amazonaws.com", true},
193+
{"https://accesspoint-account.op-outpostid.s3-outposts.eu-north-1.amazonaws.com", true},
194+
}
195+
for i, testCase := range testCases {
196+
u, err := url.Parse(testCase.url)
197+
if err != nil {
198+
t.Errorf("Test %d: url.Parse failed: %v", i+1, err)
199+
continue
200+
}
201+
got := IsAmazonOutpostsEndpoint(*u)
202+
if got != testCase.result {
203+
t.Errorf("Test %d: IsAmazonOutpostsEndpoint(%q) = %v, want %v", i+1, testCase.url, got, testCase.result)
204+
}
205+
}
206+
}
207+
179208
// Tests validate Amazon endpoint validator.
180209
func TestIsAmazonEndpoint(t *testing.T) {
181210
testCases := []struct {

0 commit comments

Comments
 (0)