mirror of
				https://github.com/go-gitea/gitea.git
				synced 2025-10-31 21:28:11 +09:00 
			
		
		
		
	* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
		
			
				
	
	
		
			318 lines
		
	
	
		
			9.0 KiB
		
	
	
	
		
			Go
		
	
	
	
		
			Vendored
		
	
	
	
			
		
		
	
	
			318 lines
		
	
	
		
			9.0 KiB
		
	
	
	
		
			Go
		
	
	
	
		
			Vendored
		
	
	
	
| /*
 | |
|  * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 | |
|  * Copyright 2015-2017 MinIO, Inc.
 | |
|  *
 | |
|  * Licensed under the Apache License, Version 2.0 (the "License");
 | |
|  * you may not use this file except in compliance with the License.
 | |
|  * You may obtain a copy of the License at
 | |
|  *
 | |
|  *     http://www.apache.org/licenses/LICENSE-2.0
 | |
|  *
 | |
|  * Unless required by applicable law or agreed to in writing, software
 | |
|  * distributed under the License is distributed on an "AS IS" BASIS,
 | |
|  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | |
|  * See the License for the specific language governing permissions and
 | |
|  * limitations under the License.
 | |
|  */
 | |
| 
 | |
| package signer
 | |
| 
 | |
| import (
 | |
| 	"bytes"
 | |
| 	"crypto/hmac"
 | |
| 	"crypto/sha1"
 | |
| 	"encoding/base64"
 | |
| 	"fmt"
 | |
| 	"net/http"
 | |
| 	"net/url"
 | |
| 	"sort"
 | |
| 	"strconv"
 | |
| 	"strings"
 | |
| 	"time"
 | |
| 
 | |
| 	"github.com/minio/minio-go/v7/pkg/s3utils"
 | |
| )
 | |
| 
 | |
| // Signature and API related constants.
 | |
| const (
 | |
| 	signV2Algorithm = "AWS"
 | |
| )
 | |
| 
 | |
| // Encode input URL path to URL encoded path.
 | |
| func encodeURL2Path(req *http.Request, virtualHost bool) (path string) {
 | |
| 	if virtualHost {
 | |
| 		reqHost := getHostAddr(req)
 | |
| 		dotPos := strings.Index(reqHost, ".")
 | |
| 		if dotPos > -1 {
 | |
| 			bucketName := reqHost[:dotPos]
 | |
| 			path = "/" + bucketName
 | |
| 			path += req.URL.Path
 | |
| 			path = s3utils.EncodePath(path)
 | |
| 			return
 | |
| 		}
 | |
| 	}
 | |
| 	path = s3utils.EncodePath(req.URL.Path)
 | |
| 	return
 | |
| }
 | |
| 
 | |
| // PreSignV2 - presign the request in following style.
 | |
| // https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
 | |
| func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64, virtualHost bool) *http.Request {
 | |
| 	// Presign is not needed for anonymous credentials.
 | |
| 	if accessKeyID == "" || secretAccessKey == "" {
 | |
| 		return &req
 | |
| 	}
 | |
| 
 | |
| 	d := time.Now().UTC()
 | |
| 	// Find epoch expires when the request will expire.
 | |
| 	epochExpires := d.Unix() + expires
 | |
| 
 | |
| 	// Add expires header if not present.
 | |
| 	if expiresStr := req.Header.Get("Expires"); expiresStr == "" {
 | |
| 		req.Header.Set("Expires", strconv.FormatInt(epochExpires, 10))
 | |
| 	}
 | |
| 
 | |
| 	// Get presigned string to sign.
 | |
| 	stringToSign := preStringToSignV2(req, virtualHost)
 | |
| 	hm := hmac.New(sha1.New, []byte(secretAccessKey))
 | |
| 	hm.Write([]byte(stringToSign))
 | |
| 
 | |
| 	// Calculate signature.
 | |
| 	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
 | |
| 
 | |
| 	query := req.URL.Query()
 | |
| 	// Handle specially for Google Cloud Storage.
 | |
| 	if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") {
 | |
| 		query.Set("GoogleAccessId", accessKeyID)
 | |
| 	} else {
 | |
| 		query.Set("AWSAccessKeyId", accessKeyID)
 | |
| 	}
 | |
| 
 | |
| 	// Fill in Expires for presigned query.
 | |
| 	query.Set("Expires", strconv.FormatInt(epochExpires, 10))
 | |
| 
 | |
| 	// Encode query and save.
 | |
| 	req.URL.RawQuery = s3utils.QueryEncode(query)
 | |
| 
 | |
| 	// Save signature finally.
 | |
| 	req.URL.RawQuery += "&Signature=" + s3utils.EncodePath(signature)
 | |
| 
 | |
| 	// Return.
 | |
| 	return &req
 | |
| }
 | |
| 
 | |
| // PostPresignSignatureV2 - presigned signature for PostPolicy
 | |
| // request.
 | |
| func PostPresignSignatureV2(policyBase64, secretAccessKey string) string {
 | |
| 	hm := hmac.New(sha1.New, []byte(secretAccessKey))
 | |
| 	hm.Write([]byte(policyBase64))
 | |
| 	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
 | |
| 	return signature
 | |
| }
 | |
| 
 | |
| // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
 | |
| // Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
 | |
| //
 | |
| // StringToSign = HTTP-Verb + "\n" +
 | |
| //  	Content-Md5 + "\n" +
 | |
| //  	Content-Type + "\n" +
 | |
| //  	Date + "\n" +
 | |
| //  	CanonicalizedProtocolHeaders +
 | |
| //  	CanonicalizedResource;
 | |
| //
 | |
| // CanonicalizedResource = [ "/" + Bucket ] +
 | |
| //  	<HTTP-Request-URI, from the protocol name up to the query string> +
 | |
| //  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
 | |
| //
 | |
| // CanonicalizedProtocolHeaders = <described below>
 | |
| 
 | |
| // SignV2 sign the request before Do() (AWS Signature Version 2).
 | |
| func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost bool) *http.Request {
 | |
| 	// Signature calculation is not needed for anonymous credentials.
 | |
| 	if accessKeyID == "" || secretAccessKey == "" {
 | |
| 		return &req
 | |
| 	}
 | |
| 
 | |
| 	// Initial time.
 | |
| 	d := time.Now().UTC()
 | |
| 
 | |
| 	// Add date if not present.
 | |
| 	if date := req.Header.Get("Date"); date == "" {
 | |
| 		req.Header.Set("Date", d.Format(http.TimeFormat))
 | |
| 	}
 | |
| 
 | |
| 	// Calculate HMAC for secretAccessKey.
 | |
| 	stringToSign := stringToSignV2(req, virtualHost)
 | |
| 	hm := hmac.New(sha1.New, []byte(secretAccessKey))
 | |
| 	hm.Write([]byte(stringToSign))
 | |
| 
 | |
| 	// Prepare auth header.
 | |
| 	authHeader := new(bytes.Buffer)
 | |
| 	authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID))
 | |
| 	encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
 | |
| 	encoder.Write(hm.Sum(nil))
 | |
| 	encoder.Close()
 | |
| 
 | |
| 	// Set Authorization header.
 | |
| 	req.Header.Set("Authorization", authHeader.String())
 | |
| 
 | |
| 	return &req
 | |
| }
 | |
| 
 | |
| // From the Amazon docs:
 | |
| //
 | |
| // StringToSign = HTTP-Verb + "\n" +
 | |
| // 	 Content-Md5 + "\n" +
 | |
| //	 Content-Type + "\n" +
 | |
| //	 Expires + "\n" +
 | |
| //	 CanonicalizedProtocolHeaders +
 | |
| //	 CanonicalizedResource;
 | |
| func preStringToSignV2(req http.Request, virtualHost bool) string {
 | |
| 	buf := new(bytes.Buffer)
 | |
| 	// Write standard headers.
 | |
| 	writePreSignV2Headers(buf, req)
 | |
| 	// Write canonicalized protocol headers if any.
 | |
| 	writeCanonicalizedHeaders(buf, req)
 | |
| 	// Write canonicalized Query resources if any.
 | |
| 	writeCanonicalizedResource(buf, req, virtualHost)
 | |
| 	return buf.String()
 | |
| }
 | |
| 
 | |
| // writePreSignV2Headers - write preSign v2 required headers.
 | |
| func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
 | |
| 	buf.WriteString(req.Method + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Content-Type") + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Expires") + "\n")
 | |
| }
 | |
| 
 | |
| // From the Amazon docs:
 | |
| //
 | |
| // StringToSign = HTTP-Verb + "\n" +
 | |
| // 	 Content-Md5 + "\n" +
 | |
| //	 Content-Type + "\n" +
 | |
| //	 Date + "\n" +
 | |
| //	 CanonicalizedProtocolHeaders +
 | |
| //	 CanonicalizedResource;
 | |
| func stringToSignV2(req http.Request, virtualHost bool) string {
 | |
| 	buf := new(bytes.Buffer)
 | |
| 	// Write standard headers.
 | |
| 	writeSignV2Headers(buf, req)
 | |
| 	// Write canonicalized protocol headers if any.
 | |
| 	writeCanonicalizedHeaders(buf, req)
 | |
| 	// Write canonicalized Query resources if any.
 | |
| 	writeCanonicalizedResource(buf, req, virtualHost)
 | |
| 	return buf.String()
 | |
| }
 | |
| 
 | |
| // writeSignV2Headers - write signV2 required headers.
 | |
| func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
 | |
| 	buf.WriteString(req.Method + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Content-Type") + "\n")
 | |
| 	buf.WriteString(req.Header.Get("Date") + "\n")
 | |
| }
 | |
| 
 | |
| // writeCanonicalizedHeaders - write canonicalized headers.
 | |
| func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
 | |
| 	var protoHeaders []string
 | |
| 	vals := make(map[string][]string)
 | |
| 	for k, vv := range req.Header {
 | |
| 		// All the AMZ headers should be lowercase
 | |
| 		lk := strings.ToLower(k)
 | |
| 		if strings.HasPrefix(lk, "x-amz") {
 | |
| 			protoHeaders = append(protoHeaders, lk)
 | |
| 			vals[lk] = vv
 | |
| 		}
 | |
| 	}
 | |
| 	sort.Strings(protoHeaders)
 | |
| 	for _, k := range protoHeaders {
 | |
| 		buf.WriteString(k)
 | |
| 		buf.WriteByte(':')
 | |
| 		for idx, v := range vals[k] {
 | |
| 			if idx > 0 {
 | |
| 				buf.WriteByte(',')
 | |
| 			}
 | |
| 			if strings.Contains(v, "\n") {
 | |
| 				// TODO: "Unfold" long headers that
 | |
| 				// span multiple lines (as allowed by
 | |
| 				// RFC 2616, section 4.2) by replacing
 | |
| 				// the folding white-space (including
 | |
| 				// new-line) by a single space.
 | |
| 				buf.WriteString(v)
 | |
| 			} else {
 | |
| 				buf.WriteString(v)
 | |
| 			}
 | |
| 		}
 | |
| 		buf.WriteByte('\n')
 | |
| 	}
 | |
| }
 | |
| 
 | |
| // AWS S3 Signature V2 calculation rule is give here:
 | |
| // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign
 | |
| 
 | |
| // Whitelist resource list that will be used in query string for signature-V2 calculation.
 | |
| // The list should be alphabetically sorted
 | |
| var resourceList = []string{
 | |
| 	"acl",
 | |
| 	"delete",
 | |
| 	"lifecycle",
 | |
| 	"location",
 | |
| 	"logging",
 | |
| 	"notification",
 | |
| 	"partNumber",
 | |
| 	"policy",
 | |
| 	"replication",
 | |
| 	"requestPayment",
 | |
| 	"response-cache-control",
 | |
| 	"response-content-disposition",
 | |
| 	"response-content-encoding",
 | |
| 	"response-content-language",
 | |
| 	"response-content-type",
 | |
| 	"response-expires",
 | |
| 	"torrent",
 | |
| 	"uploadId",
 | |
| 	"uploads",
 | |
| 	"versionId",
 | |
| 	"versioning",
 | |
| 	"versions",
 | |
| 	"website",
 | |
| }
 | |
| 
 | |
| // From the Amazon docs:
 | |
| //
 | |
| // CanonicalizedResource = [ "/" + Bucket ] +
 | |
| // 	  <HTTP-Request-URI, from the protocol name up to the query string> +
 | |
| // 	  [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
 | |
| func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) {
 | |
| 	// Save request URL.
 | |
| 	requestURL := req.URL
 | |
| 	// Get encoded URL path.
 | |
| 	buf.WriteString(encodeURL2Path(&req, virtualHost))
 | |
| 	if requestURL.RawQuery != "" {
 | |
| 		var n int
 | |
| 		vals, _ := url.ParseQuery(requestURL.RawQuery)
 | |
| 		// Verify if any sub resource queries are present, if yes
 | |
| 		// canonicallize them.
 | |
| 		for _, resource := range resourceList {
 | |
| 			if vv, ok := vals[resource]; ok && len(vv) > 0 {
 | |
| 				n++
 | |
| 				// First element
 | |
| 				switch n {
 | |
| 				case 1:
 | |
| 					buf.WriteByte('?')
 | |
| 				// The rest
 | |
| 				default:
 | |
| 					buf.WriteByte('&')
 | |
| 				}
 | |
| 				buf.WriteString(resource)
 | |
| 				// Request parameters
 | |
| 				if len(vv[0]) > 0 {
 | |
| 					buf.WriteByte('=')
 | |
| 					buf.WriteString(vv[0])
 | |
| 				}
 | |
| 			}
 | |
| 		}
 | |
| 	}
 | |
| }
 |