mirror of
https://github.com/go-gitea/gitea.git
synced 2026-09-17 05:52:13 +09:00
refactor: replace Azure Blob SDK with a REST client (#39315)
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
silverwind
GitHub
Generated
-15
File diff suppressed because one or more lines are too long
@@ -16,8 +16,6 @@ require (
|
||||
gitea.dev/sdk v1.2.0
|
||||
github.com/42wim/httpsig v1.2.4
|
||||
github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
|
||||
github.com/Azure/go-ntlmssp v0.1.1
|
||||
github.com/Necoro/html2text v0.0.0-20250804200300-7bf1ce1c7347
|
||||
github.com/ProtonMail/go-crypto v1.4.1
|
||||
@@ -74,7 +72,7 @@ require (
|
||||
github.com/meilisearch/meilisearch-go v0.36.3
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/microsoft/go-mssqldb v1.10.0
|
||||
github.com/microsoft/go-mssqldb v1.11.0
|
||||
github.com/minio/minio-go/v7 v7.3.0
|
||||
github.com/msteinert/pam/v2 v2.1.0
|
||||
github.com/niklasfasching/go-org v1.9.1
|
||||
@@ -120,7 +118,8 @@ require (
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
|
||||
github.com/DataDog/zstd v1.5.7 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/RoaringBitmap/roaring/v2 v2.18.2 // indirect
|
||||
@@ -276,13 +275,3 @@ ignore (
|
||||
./vendor
|
||||
./web_src
|
||||
)
|
||||
|
||||
// When doing "go get -u ./...", Golang will try to update all dependencies
|
||||
// But not all latest versions of dependencies are compatible with other packages or our codebase, so we need to pin some dependencies to specific versions
|
||||
// Need to regularly maintain this list to try to update them to latest versions, especially the TODO ones
|
||||
|
||||
replace github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0 // v1.21.0+ uses API version unsupported by Azurite in CI
|
||||
|
||||
replace github.com/Azure/azure-sdk-for-go/sdk/storage/azblob => github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 // v1.6.4+ uses API version unsupported by Azurite in CI
|
||||
|
||||
replace github.com/microsoft/go-mssqldb => github.com/microsoft/go-mssqldb v1.9.7 // downgraded with Azure SDK
|
||||
|
||||
@@ -30,24 +30,20 @@ github.com/42wim/httpsig v1.2.4 h1:mI5bH0nm4xn7K18fo1K3okNDRq8CCJ0KbBYWyA6r8lU=
|
||||
github.com/42wim/httpsig v1.2.4/go.mod h1:yKsYfSyTBEohkPik224QPFylmzEBtda/kjyIAJjh3ps=
|
||||
github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432 h1:3Fcz1QzlS7Jv4FT2KI3cHNSZL+KPN3dXxurn9f3YL/Y=
|
||||
github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432/go.mod h1:BLWe6Nol65Xxncvaw07yYMxiyk02We1lBrbRYsMYsjE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0 h1:ci6Yd6nysBRLEodoziB6ah1+YOzZbZk+NYneoA6q+6E=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0/go.mod h1:QyVsSSN64v5TGltphKLQ2sQxe4OBQg0J1eKRcVBnfgE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
|
||||
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
|
||||
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -475,8 +471,8 @@ github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
|
||||
github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/microsoft/go-mssqldb v1.9.7 h1:I+JEk79gYsc6bdVzDHFSSYE9dtNa7dxRwJ0WQbt6i8w=
|
||||
github.com/microsoft/go-mssqldb v1.9.7/go.mod h1:yYMPDufyoF2vVuVCUGtZARr06DKFIhMrluTcgWlXpr4=
|
||||
github.com/microsoft/go-mssqldb v1.11.0 h1:YbDqolEjGH9hBfvKzONTf5/dbl9RKXmizMJE93lVxNs=
|
||||
github.com/microsoft/go-mssqldb v1.11.0/go.mod h1:goQLDOPlMN/l1REhnNPElMoY/yX+fUWn1+7UoFJPH9Y=
|
||||
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
|
||||
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
|
||||
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
|
||||
|
||||
@@ -59,7 +59,6 @@ func (cfg *MinioStorageConfig) ToShadow() {
|
||||
}
|
||||
}
|
||||
|
||||
// MinioStorageConfig represents the configuration for a minio storage
|
||||
type AzureBlobStorageConfig struct {
|
||||
Endpoint string `ini:"AZURE_BLOB_ENDPOINT" json:",omitempty"`
|
||||
AccountName string `ini:"AZURE_BLOB_ACCOUNT_NAME" json:",omitempty"`
|
||||
|
||||
+321
-245
@@ -4,15 +4,26 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -20,50 +31,71 @@ import (
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var _ Object = &azureBlobObject{}
|
||||
const azureBlobAPIVersion = "2025-11-05" // must not exceed the Azurite version used in CI
|
||||
|
||||
type azureBlobError string
|
||||
|
||||
func (e azureBlobError) Error() string { return string(e) }
|
||||
|
||||
type azureBlobObject struct {
|
||||
blobClient *blob.Client
|
||||
ctx context.Context
|
||||
name string
|
||||
size int64
|
||||
modTime *time.Time
|
||||
offset int64
|
||||
storage *AzureBlobStorage
|
||||
blobURL *url.URL
|
||||
info *objectFileInfo
|
||||
etag string
|
||||
offset int64
|
||||
closed bool
|
||||
|
||||
respBody io.ReadCloser
|
||||
}
|
||||
|
||||
func (a *azureBlobObject) resetRespBody() {
|
||||
// close resp, the next attempt reopens at the current offset
|
||||
if a.respBody != nil {
|
||||
_ = a.respBody.Close()
|
||||
a.respBody = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (a *azureBlobObject) Read(p []byte) (int, error) {
|
||||
// TODO: improve the performance, we can implement another interface, maybe implement io.WriteTo
|
||||
if a.offset >= a.size {
|
||||
if a.closed {
|
||||
return 0, fs.ErrClosed
|
||||
}
|
||||
if a.offset >= a.info.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
count := min(int64(len(p)), a.size-a.offset)
|
||||
|
||||
res, err := a.blobClient.DownloadBuffer(a.ctx, p, &blob.DownloadBufferOptions{
|
||||
Range: blob.HTTPRange{
|
||||
Offset: a.offset,
|
||||
Count: count,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return 0, convertAzureBlobErr(err)
|
||||
var lastErr error
|
||||
for range 4 {
|
||||
if a.respBody == nil {
|
||||
reqHeader := http.Header{"X-Ms-Range": {fmt.Sprintf("bytes=%d-", a.offset)}, "If-Match": {a.etag}}
|
||||
_, body, err := a.storage.do(a.storage.ctx, http.MethodGet, a.blobURL, reqHeader, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
a.respBody = body
|
||||
}
|
||||
n, err := io.ReadFull(a.respBody, p[:min(int64(len(p)), a.info.size-a.offset)])
|
||||
a.offset += int64(n)
|
||||
if err != nil {
|
||||
a.resetRespBody()
|
||||
}
|
||||
if err == nil || n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
a.offset += res
|
||||
|
||||
return int(res), nil
|
||||
return 0, util.Iif(lastErr == io.EOF, io.ErrUnexpectedEOF, lastErr)
|
||||
}
|
||||
|
||||
func (a *azureBlobObject) Close() error {
|
||||
a.offset = 0
|
||||
return nil
|
||||
func (a *azureBlobObject) Close() (err error) {
|
||||
a.closed = true
|
||||
if a.respBody != nil {
|
||||
err = a.respBody.Close()
|
||||
a.respBody = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
|
||||
@@ -72,267 +104,311 @@ func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
|
||||
case io.SeekCurrent:
|
||||
offset += a.offset
|
||||
case io.SeekEnd:
|
||||
offset = a.size + offset
|
||||
offset = a.info.size + offset
|
||||
default:
|
||||
return 0, errors.New("Seek: invalid whence")
|
||||
return 0, errors.New("seek: invalid whence")
|
||||
}
|
||||
|
||||
if offset > a.size {
|
||||
return 0, errors.New("Seek: invalid offset")
|
||||
} else if offset < 0 {
|
||||
return 0, errors.New("Seek: invalid offset")
|
||||
if offset < 0 || offset > a.info.size {
|
||||
return 0, errors.New("seek: invalid offset")
|
||||
}
|
||||
a.resetRespBody()
|
||||
a.offset = offset
|
||||
return a.offset, nil
|
||||
}
|
||||
|
||||
func (a *azureBlobObject) Stat() (os.FileInfo, error) {
|
||||
return &azureBlobFileInfo{
|
||||
a.name,
|
||||
a.size,
|
||||
*a.modTime,
|
||||
}, nil
|
||||
return a.info, nil
|
||||
}
|
||||
|
||||
var _ ObjectStorage = &AzureBlobStorage{}
|
||||
|
||||
type AzureBlobStorage struct {
|
||||
cfg *setting.AzureBlobStorageConfig
|
||||
ctx context.Context
|
||||
credential *azblob.SharedKeyCredential
|
||||
client *azblob.Client
|
||||
cfg *setting.AzureBlobStorageConfig
|
||||
ctx context.Context
|
||||
client *http.Client
|
||||
endpoint *url.URL
|
||||
key []byte
|
||||
blockSize int
|
||||
concurrency int
|
||||
retryDelay time.Duration
|
||||
}
|
||||
|
||||
func convertAzureBlobErr(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return fs.ErrNotExist
|
||||
}
|
||||
var respErr *azcore.ResponseError
|
||||
if !errors.As(err, &respErr) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%s", respErr.ErrorCode)
|
||||
}
|
||||
|
||||
// NewAzureBlobStorage returns a azure blob storage
|
||||
func NewAzureBlobStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
|
||||
config := cfg.AzureBlobConfig
|
||||
|
||||
log.Info("Creating Azure Blob storage at %s:%s with base path %s", config.Endpoint, config.Container, config.BasePath)
|
||||
|
||||
cred, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey)
|
||||
key, err := base64.StdEncoding.DecodeString(config.AccountKey)
|
||||
if err != nil {
|
||||
return nil, convertAzureBlobErr(err)
|
||||
return nil, fmt.Errorf("invalid azure blob account key: %w", err)
|
||||
}
|
||||
client, err := azblob.NewClientWithSharedKeyCredential(config.Endpoint, cred, &azblob.ClientOptions{})
|
||||
endpoint, err := url.Parse(config.Endpoint)
|
||||
if err != nil {
|
||||
return nil, convertAzureBlobErr(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = client.CreateContainer(ctx, config.Container, &container.CreateOptions{})
|
||||
if err != nil {
|
||||
// Check to see if we already own this container (which happens if you run this twice)
|
||||
if !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
|
||||
return nil, convertMinioErr(err)
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
}
|
||||
a := &AzureBlobStorage{
|
||||
cfg: &config,
|
||||
ctx: ctx,
|
||||
client: &http.Client{Transport: transport},
|
||||
endpoint: endpoint,
|
||||
key: key,
|
||||
blockSize: 4 * 1024 * 1024,
|
||||
concurrency: 4,
|
||||
retryDelay: 200 * time.Millisecond,
|
||||
}
|
||||
_, _, err = a.do(ctx, http.MethodPut, a.url(config.Container, url.Values{"restype": {"container"}}), nil, nil)
|
||||
if err != nil && !errors.Is(err, azureBlobError("ContainerAlreadyExists")) {
|
||||
return nil, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) url(name string, query url.Values) *url.URL {
|
||||
u := *a.endpoint
|
||||
u.Path = strings.TrimSuffix(u.Path, "/") + "/" + name
|
||||
u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20") // Azure doesn't decode "+" as space
|
||||
return &u
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) blobName(p string) string {
|
||||
return a.cfg.Container + "/" + buildObjectStorePath(a.cfg.BasePath, p)
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) signString(s string) string {
|
||||
mac := hmac.New(sha256.New, a.key)
|
||||
_, _ = mac.Write([]byte(s))
|
||||
return base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// https://learn.microsoft.com/rest/api/storageservices/authorize-with-shared-key
|
||||
func (a *AzureBlobStorage) signRequest(req *http.Request) string {
|
||||
lines := []string{
|
||||
req.Method,
|
||||
req.Header.Get("Content-Encoding"),
|
||||
req.Header.Get("Content-Language"),
|
||||
util.Iif(req.ContentLength > 0, strconv.FormatInt(req.ContentLength, 10), ""),
|
||||
}
|
||||
for _, name := range []string{
|
||||
"Content-MD5", "Content-Type", "Date",
|
||||
"If-Modified-Since", "If-Match", "If-None-Match", "If-Unmodified-Since", "Range",
|
||||
} {
|
||||
lines = append(lines, req.Header.Get(name))
|
||||
}
|
||||
msHeaders := map[string]string{}
|
||||
for key, values := range req.Header {
|
||||
if key = strings.ToLower(key); strings.HasPrefix(key, "x-ms-") {
|
||||
msHeaders[key] = key + ":" + strings.Join(values, ",")
|
||||
}
|
||||
}
|
||||
|
||||
return &AzureBlobStorage{
|
||||
cfg: &config,
|
||||
ctx: ctx,
|
||||
credential: cred,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) buildAzureBlobPath(p string) string {
|
||||
return buildObjectStorePath(a.cfg.BasePath, p)
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) getObjectNameFromPath(path string) string {
|
||||
s := strings.Split(path, "/")
|
||||
return s[len(s)-1]
|
||||
}
|
||||
|
||||
// Open opens a file
|
||||
func (a *AzureBlobStorage) Open(path string) (Object, error) {
|
||||
blobClient := a.getBlobClient(path)
|
||||
res, err := blobClient.GetProperties(a.ctx, &blob.GetPropertiesOptions{})
|
||||
if err != nil {
|
||||
return nil, convertAzureBlobErr(err)
|
||||
for _, key := range slices.Sorted(maps.Keys(msHeaders)) {
|
||||
lines = append(lines, msHeaders[key])
|
||||
}
|
||||
return &azureBlobObject{
|
||||
ctx: a.ctx,
|
||||
blobClient: blobClient,
|
||||
name: a.getObjectNameFromPath(path),
|
||||
size: *res.ContentLength,
|
||||
modTime: res.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Save saves a file to azure blob storage
|
||||
func (a *AzureBlobStorage) Save(path string, r io.Reader, size int64) (int64, error) {
|
||||
rd := util.NewCountingReader(r)
|
||||
_, err := a.client.UploadStream(
|
||||
a.ctx,
|
||||
a.cfg.Container,
|
||||
a.buildAzureBlobPath(path),
|
||||
rd,
|
||||
// TODO: support set block size and concurrency
|
||||
&blockblob.UploadStreamOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, convertAzureBlobErr(err)
|
||||
lines = append(lines, "/"+a.cfg.AccountName+req.URL.EscapedPath()) // encoded as sent, not decoded
|
||||
query := req.URL.Query()
|
||||
for _, key := range slices.Sorted(maps.Keys(query)) {
|
||||
slices.Sort(query[key])
|
||||
lines = append(lines, strings.ToLower(key)+":"+strings.Join(query[key], ","))
|
||||
}
|
||||
return int64(rd.Count()), nil
|
||||
return a.signString(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
type azureBlobFileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) Name() string {
|
||||
return path.Base(a.name)
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) Size() int64 {
|
||||
return a.size
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) ModTime() time.Time {
|
||||
return a.modTime
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) IsDir() bool {
|
||||
return strings.HasSuffix(a.name, "/")
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) Mode() os.FileMode {
|
||||
return os.ModePerm
|
||||
}
|
||||
|
||||
func (a azureBlobFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns the stat information of the object
|
||||
func (a *AzureBlobStorage) Stat(path string) (os.FileInfo, error) {
|
||||
blobClient := a.getBlobClient(path)
|
||||
res, err := blobClient.GetProperties(a.ctx, &blob.GetPropertiesOptions{})
|
||||
if err != nil {
|
||||
return nil, convertAzureBlobErr(err)
|
||||
}
|
||||
s := strings.Split(path, "/")
|
||||
return &azureBlobFileInfo{
|
||||
s[len(s)-1],
|
||||
*res.ContentLength,
|
||||
*res.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Delete delete a file
|
||||
func (a *AzureBlobStorage) Delete(path string) error {
|
||||
blobClient := a.getBlobClient(path)
|
||||
_, err := blobClient.Delete(a.ctx, nil)
|
||||
return convertAzureBlobErr(err)
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) getSasURL(b *blob.Client, template sas.BlobSignatureValues) (string, error) {
|
||||
urlParts, err := blob.ParseURL(b.URL())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var t time.Time
|
||||
if urlParts.Snapshot == "" {
|
||||
t = time.Time{}
|
||||
} else {
|
||||
t, err = time.Parse(blob.SnapshotTimeFormat, urlParts.Snapshot)
|
||||
// only GET returns the body, the caller closes it
|
||||
func (a *AzureBlobStorage) do(ctx context.Context, method string, u *url.URL, header http.Header, body []byte) (http.Header, io.ReadCloser, error) {
|
||||
const maxDelay = 3 * time.Second
|
||||
for retry := 0; ; retry++ {
|
||||
req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
return nil, nil, err
|
||||
}
|
||||
maps.Copy(req.Header, header)
|
||||
req.Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
|
||||
req.Header.Set("x-ms-version", azureBlobAPIVersion)
|
||||
req.Header.Set("Authorization", "SharedKey "+a.cfg.AccountName+":"+a.signRequest(req))
|
||||
|
||||
resp, err := a.client.Do(req)
|
||||
if retry < 3 && (err != nil || slices.Contains([]int{408, 429, 500, 502, 503, 504}, resp.StatusCode)) {
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, nil, ctx.Err()
|
||||
case <-time.After(min(a.retryDelay<<retry, maxDelay)):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if resp.StatusCode < http.StatusBadRequest {
|
||||
if method == http.MethodGet {
|
||||
return resp.Header, resp.Body, nil
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return resp.Header, nil, nil
|
||||
}
|
||||
var errBody struct{ Message, AuthenticationErrorDetail string }
|
||||
_ = xml.NewDecoder(resp.Body).Decode(&errBody)
|
||||
_ = resp.Body.Close()
|
||||
code := cmp.Or(resp.Header.Get("x-ms-error-code"), resp.Status)
|
||||
if code == "BlobNotFound" {
|
||||
return nil, nil, fs.ErrNotExist
|
||||
}
|
||||
if errBody.Message == "" {
|
||||
return nil, nil, azureBlobError(code)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("%w: %s", azureBlobError(code), strings.TrimSpace(errBody.Message+"\n"+errBody.AuthenticationErrorDetail))
|
||||
}
|
||||
|
||||
template.ContainerName = urlParts.ContainerName
|
||||
template.BlobName = urlParts.BlobName
|
||||
template.SnapshotTime = t
|
||||
template.Version = sas.Version
|
||||
|
||||
qps, err := template.SignWithSharedKey(a.credential)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
endpoint := b.URL() + "?" + qps.Encode()
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqParams *ServeDirectOptions) (*url.URL, error) {
|
||||
blobClient := a.getBlobClient(storePath)
|
||||
|
||||
startTime := time.Now().UTC()
|
||||
|
||||
param := prepareServeDirectOptions(reqParams, name)
|
||||
|
||||
u, err := a.getSasURL(blobClient, sas.BlobSignatureValues{
|
||||
Permissions: (&sas.BlobPermissions{
|
||||
Read: method == http.MethodGet || method == http.MethodHead,
|
||||
Write: method == http.MethodPut,
|
||||
}).String(),
|
||||
StartTime: startTime,
|
||||
ExpiryTime: startTime.Add(5 * time.Minute),
|
||||
ContentDisposition: param.ContentDisposition,
|
||||
ContentType: param.ContentType,
|
||||
})
|
||||
func (a *AzureBlobStorage) Open(path string) (Object, error) {
|
||||
obj, err := a.stat(path)
|
||||
if err != nil {
|
||||
return nil, convertAzureBlobErr(err)
|
||||
return nil, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Save(path string, r io.Reader, _ int64) (int64, error) {
|
||||
name := a.blobName(path)
|
||||
block := make([]byte, a.blockSize)
|
||||
n, err := util.ReadAtMost(r, block)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n < a.blockSize {
|
||||
_, _, err := a.do(a.ctx, http.MethodPut, a.url(name, nil), http.Header{"X-Ms-Blob-Type": {"BlockBlob"}}, block[:n])
|
||||
return int64(n), err
|
||||
}
|
||||
|
||||
return url.Parse(u)
|
||||
g, ctx := errgroup.WithContext(a.ctx)
|
||||
g.SetLimit(a.concurrency)
|
||||
blockList := bytes.NewBufferString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
|
||||
idPrefix := rand.Text()
|
||||
var total int64
|
||||
for blockNum := 0; n > 0 && ctx.Err() == nil; blockNum++ {
|
||||
id := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "%s%038d", idPrefix, blockNum)) // 64 bytes like the old SDK, Azure rejects mixed ID lengths
|
||||
blockList.WriteString("<Latest>" + id + "</Latest>")
|
||||
total += int64(n)
|
||||
data := block[:n]
|
||||
g.Go(func() error {
|
||||
_, _, err := a.do(ctx, http.MethodPut, a.url(name, url.Values{"comp": {"block"}, "blockid": {id}}), nil, data)
|
||||
return err
|
||||
})
|
||||
if n < a.blockSize {
|
||||
break
|
||||
}
|
||||
block = make([]byte, a.blockSize)
|
||||
if n, err = util.ReadAtMost(r, block); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err = errors.Join(g.Wait(), err); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
blockList.WriteString("</BlockList>")
|
||||
_, _, err = a.do(a.ctx, http.MethodPut, a.url(name, url.Values{"comp": {"blocklist"}}), nil, blockList.Bytes())
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) stat(p string) (*azureBlobObject, error) {
|
||||
blobURL := a.url(a.blobName(p), nil)
|
||||
header, _, err := a.do(a.ctx, http.MethodHead, blobURL, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size, sizeErr := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
|
||||
modTime, timeErr := http.ParseTime(header.Get("Last-Modified"))
|
||||
return &azureBlobObject{storage: a, blobURL: blobURL, info: &objectFileInfo{path.Base(p), size, modTime}, etag: header.Get("ETag")}, errors.Join(sizeErr, timeErr)
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Stat(path string) (os.FileInfo, error) {
|
||||
obj, err := a.stat(path)
|
||||
if err != nil {
|
||||
return nil, err // not obj.info, a nil pointer would be a non-nil os.FileInfo
|
||||
}
|
||||
return obj.info, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Delete(path string) error {
|
||||
_, _, err := a.do(a.ctx, http.MethodDelete, a.url(a.blobName(path), nil), nil, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// https://learn.microsoft.com/rest/api/storageservices/create-service-sas
|
||||
func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqParams *ServeDirectOptions) (*url.URL, error) {
|
||||
permissions := util.Iif(method == http.MethodPut, "w", "r")
|
||||
param := prepareServeDirectOptions(reqParams, name)
|
||||
now := time.Now().UTC()
|
||||
start, expiry := now.Add(-15*time.Minute).Format(time.RFC3339), now.Add(5*time.Minute).Format(time.RFC3339) // SAS expiration policies require a start, backdated for clock skew
|
||||
canonicalName := "/blob/" + a.cfg.AccountName + "/" + a.blobName(storePath)
|
||||
signature := a.signString(strings.Join([]string{permissions, start, expiry, canonicalName, "", "", "", azureBlobAPIVersion, "b", "", "", "", param.ContentDisposition, "", "", param.ContentType}, "\n"))
|
||||
|
||||
query := url.Values{"sv": {azureBlobAPIVersion}, "st": {start}, "se": {expiry}, "sr": {"b"}, "sp": {permissions}, "sig": {signature}}
|
||||
if param.ContentDisposition != "" {
|
||||
query.Set("rscd", param.ContentDisposition)
|
||||
}
|
||||
if param.ContentType != "" {
|
||||
query.Set("rsct", param.ContentType)
|
||||
}
|
||||
return a.url(a.blobName(storePath), query), nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
|
||||
basePrefix := buildObjectStorePathPrefix(a.cfg.BasePath, "")
|
||||
dirPrefix := buildObjectStorePathPrefix(a.cfg.BasePath, dirName)
|
||||
pager := a.client.NewListBlobsFlatPager(a.cfg.Container, &container.ListBlobsFlatOptions{
|
||||
Prefix: &dirPrefix,
|
||||
})
|
||||
|
||||
callback := func(object *azureBlobObject, objPath string) error {
|
||||
defer object.Close()
|
||||
return fn(objPath, object)
|
||||
}
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(a.ctx)
|
||||
query := url.Values{"restype": {"container"}, "comp": {"list"}, "prefix": {buildObjectStorePathPrefix(a.cfg.BasePath, dirName)}}
|
||||
for {
|
||||
_, body, err := a.do(a.ctx, http.MethodGet, a.url(a.cfg.Container, query), nil, nil)
|
||||
if err != nil {
|
||||
return convertAzureBlobErr(err)
|
||||
return err
|
||||
}
|
||||
for _, azureObj := range resp.Segment.BlobItems {
|
||||
objPath := strings.TrimPrefix(*azureObj.Name, basePrefix)
|
||||
objWrap := &azureBlobObject{
|
||||
ctx: a.ctx,
|
||||
blobClient: a.getBlobClient(objPath),
|
||||
name: *azureObj.Name,
|
||||
size: *azureObj.Properties.ContentLength,
|
||||
modTime: azureObj.Properties.LastModified,
|
||||
var result struct {
|
||||
Blobs []struct {
|
||||
Name string `xml:"Name"`
|
||||
ContentLength int64 `xml:"Properties>Content-Length"`
|
||||
LastModified string `xml:"Properties>Last-Modified"`
|
||||
Etag string `xml:"Properties>Etag"`
|
||||
ResourceType string `xml:"Properties>ResourceType"`
|
||||
} `xml:"Blobs>Blob"`
|
||||
NextMarker string `xml:"NextMarker"`
|
||||
}
|
||||
err = xml.NewDecoder(body).Decode(&result)
|
||||
_ = body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, blob := range result.Blobs {
|
||||
if blob.ResourceType == "directory" { // listed by hierarchical namespace accounts
|
||||
continue
|
||||
}
|
||||
if err := callback(objWrap, objPath); err != nil {
|
||||
return convertAzureBlobErr(err)
|
||||
modTime, err := http.ParseTime(blob.LastModified)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
object := &azureBlobObject{
|
||||
storage: a,
|
||||
blobURL: a.url(a.cfg.Container+"/"+blob.Name, nil),
|
||||
info: &objectFileInfo{path.Base(blob.Name), blob.ContentLength, modTime},
|
||||
etag: blob.Etag,
|
||||
}
|
||||
err = fn(strings.TrimPrefix(blob.Name, basePrefix), object)
|
||||
_ = object.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if result.NextMarker == "" {
|
||||
return nil
|
||||
}
|
||||
query.Set("marker", result.NextMarker)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) getBlobClient(path string) *blob.Client {
|
||||
return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path))
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -5,72 +5,121 @@ package storage
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/test"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func prepareAzureStorageConfig(t *testing.T, basePath ...string) *setting.Storage {
|
||||
type azureBlobFaultTransport struct {
|
||||
failed, truncated atomic.Bool
|
||||
}
|
||||
|
||||
func (t *azureBlobFaultTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if t.failed.CompareAndSwap(false, true) {
|
||||
return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: http.NoBody}, nil
|
||||
}
|
||||
resp, err := http.DefaultTransport.RoundTrip(req)
|
||||
if err == nil && req.Method == http.MethodGet && t.truncated.CompareAndSwap(false, true) {
|
||||
resp.Body = struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{io.MultiReader(io.LimitReader(resp.Body, 2), iotest.ErrReader(io.ErrUnexpectedEOF)), resp.Body}
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func newAzureBlobTestStorage(t *testing.T, basePath string) *AzureBlobStorage {
|
||||
endpoint := test.ExternalServiceHTTP(t, "TEST_AZURESTORAGE_ENDPOINT", "http://devstoreaccount1.azurite.local:10000")
|
||||
return &setting.Storage{
|
||||
objStore, err := NewStorage(setting.AzureBlobStorageType, &setting.Storage{
|
||||
AzureBlobConfig: setting.AzureBlobStorageConfig{
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#ip-style-url
|
||||
Endpoint: endpoint,
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#well-known-storage-account-and-key
|
||||
Endpoint: endpoint,
|
||||
AccountName: "devstoreaccount1",
|
||||
AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
|
||||
Container: "test-container",
|
||||
BasePath: util.OptionalArg(basePath),
|
||||
BasePath: basePath,
|
||||
},
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
s, ok := objStore.(*AzureBlobStorage)
|
||||
require.True(t, ok)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestAzureBlobStorage(t *testing.T) {
|
||||
t.Run("NoBasePath", func(t *testing.T) {
|
||||
config := prepareAzureStorageConfig(t)
|
||||
objStore, err := NewStorage(setting.AzureBlobStorageType, config)
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
t.Run("WithBasePath", func(t *testing.T) {
|
||||
config := prepareAzureStorageConfig(t, "test-base-path")
|
||||
objStore, err := NewStorage(setting.AzureBlobStorageType, config)
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_azureBlobObject(t *testing.T) {
|
||||
s, err := NewStorage(setting.AzureBlobStorageType, prepareAzureStorageConfig(t))
|
||||
require.NoError(t, err)
|
||||
t.Run("General", func(t *testing.T) { testStorageGeneral(t, newAzureBlobTestStorage(t, "")) })
|
||||
t.Run("GeneralWithBasePath", func(t *testing.T) { testStorageGeneral(t, newAzureBlobTestStorage(t, "test-base-path")) })
|
||||
|
||||
s := newAzureBlobTestStorage(t, "")
|
||||
s.blockSize, s.concurrency, s.retryDelay = 4, 2, 0
|
||||
transport := &azureBlobFaultTransport{}
|
||||
s.client.Transport = transport
|
||||
data := "Q2xTckt6Y1hDOWh0"
|
||||
_, err = s.Save("test.txt", strings.NewReader(data), int64(len(data)))
|
||||
assert.NoError(t, err)
|
||||
obj, err := s.Open("test.txt")
|
||||
assert.NoError(t, err)
|
||||
offset, err := obj.Seek(2, io.SeekStart)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 2, offset)
|
||||
buf1 := make([]byte, 3)
|
||||
read, err := obj.Read(buf1)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, read)
|
||||
assert.Equal(t, data[2:5], string(buf1))
|
||||
offset, err = obj.Seek(-5, io.SeekEnd)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, len(data)-5, offset)
|
||||
buf2 := make([]byte, 4)
|
||||
read, err = obj.Read(buf2)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 4, read)
|
||||
assert.Equal(t, data[11:15], string(buf2))
|
||||
assert.NoError(t, obj.Close())
|
||||
assert.NoError(t, s.Delete("test.txt"))
|
||||
|
||||
t.Run("SaveBlocksWithRetryAndRejectTruncatedInput", func(t *testing.T) {
|
||||
written, err := s.Save("test.txt", strings.NewReader(data), -1)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, len(data), written)
|
||||
assert.True(t, transport.failed.Load())
|
||||
_, err = s.Save("truncated.txt", io.MultiReader(strings.NewReader(data), iotest.ErrReader(io.ErrUnexpectedEOF)), -1)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
})
|
||||
|
||||
t.Run("ReadResumesSeeksAndFailsOnChangedBlob", func(t *testing.T) {
|
||||
obj, err := s.Open("test.txt")
|
||||
require.NoError(t, err)
|
||||
defer obj.Close()
|
||||
buf := make([]byte, 4)
|
||||
_, err = io.ReadFull(obj, buf)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, data[:4], string(buf))
|
||||
assert.True(t, transport.truncated.Load())
|
||||
_, err = obj.Seek(-5, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
_, err = io.ReadFull(obj, buf)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, data[11:15], string(buf))
|
||||
|
||||
_, err = s.Save("test.txt", strings.NewReader("changed"), -1)
|
||||
require.NoError(t, err)
|
||||
_, err = obj.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
_, err = io.ReadAll(obj)
|
||||
assert.ErrorIs(t, err, azureBlobError("ConditionNotMet"))
|
||||
})
|
||||
|
||||
t.Run("ServeDirectURLAllowsPut", func(t *testing.T) {
|
||||
u, err := s.ServeDirectURL("direct.txt", "direct.txt", http.MethodPut, nil)
|
||||
require.NoError(t, err)
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, u.String(), strings.NewReader("direct"))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-ms-blob-type", "BlockBlob")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
obj, err := s.Open("direct.txt")
|
||||
require.NoError(t, err)
|
||||
defer obj.Close()
|
||||
content, err := io.ReadAll(obj)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "direct", string(content))
|
||||
})
|
||||
|
||||
t.Run("StatAfterDeleteReturnsNilInfo", func(t *testing.T) {
|
||||
assert.NoError(t, s.Delete("test.txt"))
|
||||
assert.NoError(t, s.Delete("direct.txt"))
|
||||
info, err := s.Stat("test.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||
assert.Equal(t, os.FileInfo(nil), info)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func (m *minioObject) Stat() (os.FileInfo, error) {
|
||||
return nil, convertMinioErr(err)
|
||||
}
|
||||
|
||||
return &minioFileInfo{oi}, nil
|
||||
return &objectFileInfo{path.Base(oi.Key), oi.Size, oi.LastModified}, nil
|
||||
}
|
||||
|
||||
// minio reports a missing key on the first Read, ReadAt or Seek rather than on Open, so all
|
||||
@@ -234,46 +234,18 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
|
||||
return uploadInfo.Size, nil
|
||||
}
|
||||
|
||||
type minioFileInfo struct {
|
||||
minio.ObjectInfo
|
||||
}
|
||||
|
||||
func (m minioFileInfo) Name() string {
|
||||
return path.Base(m.ObjectInfo.Key)
|
||||
}
|
||||
|
||||
func (m minioFileInfo) Size() int64 {
|
||||
return m.ObjectInfo.Size
|
||||
}
|
||||
|
||||
func (m minioFileInfo) ModTime() time.Time {
|
||||
return m.LastModified
|
||||
}
|
||||
|
||||
func (m minioFileInfo) IsDir() bool {
|
||||
return strings.HasSuffix(m.ObjectInfo.Key, "/")
|
||||
}
|
||||
|
||||
func (m minioFileInfo) Mode() os.FileMode {
|
||||
return os.ModePerm
|
||||
}
|
||||
|
||||
func (m minioFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns the stat information of the object
|
||||
func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
|
||||
func (m *MinioStorage) Stat(p string) (os.FileInfo, error) {
|
||||
info, err := m.client.StatObject(
|
||||
m.ctx,
|
||||
m.bucket,
|
||||
m.buildMinioPath(path),
|
||||
m.buildMinioPath(p),
|
||||
minio.StatObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, convertMinioErr(err)
|
||||
}
|
||||
return &minioFileInfo{info}, nil
|
||||
return &objectFileInfo{path.Base(info.Key), info.Size, info.LastModified}, nil
|
||||
}
|
||||
|
||||
// Delete delete a file
|
||||
|
||||
@@ -10,13 +10,12 @@ import (
|
||||
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/test"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func prepareMinioStorageConfig(t *testing.T, basePath ...string) *setting.Storage {
|
||||
func prepareMinioStorageConfig(t *testing.T, basePath string) *setting.Storage {
|
||||
return &setting.Storage{
|
||||
MinioConfig: setting.MinioStorageConfig{
|
||||
Endpoint: test.ExternalServiceHTTP(t, "TEST_MINIO_ENDPOINT", "minio:9000"),
|
||||
@@ -24,28 +23,21 @@ func prepareMinioStorageConfig(t *testing.T, basePath ...string) *setting.Storag
|
||||
SecretAccessKey: "12345678",
|
||||
Bucket: "gitea",
|
||||
Location: "us-east-1",
|
||||
BasePath: util.OptionalArg(basePath),
|
||||
BasePath: basePath,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinioStorage(t *testing.T) {
|
||||
t.Run("NoBasePath", func(t *testing.T) {
|
||||
config := prepareMinioStorageConfig(t)
|
||||
objStore, err := NewStorage(setting.MinioStorageType, config)
|
||||
for _, basePath := range []string{"", "test-base-path"} {
|
||||
objStore, err := NewStorage(setting.MinioStorageType, prepareMinioStorageConfig(t, basePath))
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
t.Run("WithBasePath", func(t *testing.T) {
|
||||
config := prepareMinioStorageConfig(t, "test-base-path")
|
||||
objStore, err := NewStorage(setting.MinioStorageType, config)
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageBadRequest(t *testing.T) {
|
||||
cfg := prepareMinioStorageConfig(t)
|
||||
cfg := prepareMinioStorageConfig(t, "")
|
||||
cfg.MinioConfig.SecretAccessKey = "invalid-secret"
|
||||
_, err := NewStorage(setting.MinioStorageType, cfg)
|
||||
assert.ErrorContains(t, err, "ObjectStorage.BucketExists: endpoint="+cfg.MinioConfig.Endpoint)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/httplib"
|
||||
"gitea.dev/modules/log"
|
||||
@@ -42,6 +43,19 @@ type Object interface {
|
||||
Stat() (os.FileInfo, error)
|
||||
}
|
||||
|
||||
type objectFileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (o *objectFileInfo) Name() string { return o.name } // base name, no dir path
|
||||
func (o *objectFileInfo) Size() int64 { return o.size }
|
||||
func (o *objectFileInfo) ModTime() time.Time { return o.modTime }
|
||||
func (o *objectFileInfo) IsDir() bool { return false } // object storage doesn't support dir
|
||||
func (o *objectFileInfo) Mode() os.FileMode { return os.ModePerm }
|
||||
func (o *objectFileInfo) Sys() any { return nil }
|
||||
|
||||
// ServeDirectOptions customizes HTTP headers for a generated signed URL.
|
||||
type ServeDirectOptions struct {
|
||||
// Overrides the automatically detected MIME type.
|
||||
|
||||
@@ -6,6 +6,7 @@ package storage
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -47,7 +48,7 @@ func TestObjectStoragePath(t *testing.T) {
|
||||
assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
|
||||
}
|
||||
|
||||
func testStorageIterator(t *testing.T, objStore ObjectStorage) {
|
||||
func testStorageAccess(t *testing.T, objStore ObjectStorage) {
|
||||
testFiles := [][]string{
|
||||
{"a/1.txt", "a1"},
|
||||
{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
|
||||
@@ -57,39 +58,64 @@ func testStorageIterator(t *testing.T, objStore ObjectStorage) {
|
||||
{"b/3.txt", "b3"},
|
||||
{"b/x 4.txt", "bx4"},
|
||||
}
|
||||
for _, f := range testFiles {
|
||||
_, err := objStore.Save(f[0], strings.NewReader(f[1]), -1)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
defer func() {
|
||||
for _, f := range testFiles {
|
||||
_ = objStore.Delete(f[0])
|
||||
}
|
||||
}()
|
||||
|
||||
expectedList := map[string][]string{
|
||||
"a": {"a/1.txt"},
|
||||
"a/": {"a/1.txt"},
|
||||
"/a/": {"a/1.txt"},
|
||||
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
||||
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
".": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
"a/b/../../a": {"a/1.txt"},
|
||||
}
|
||||
for dir, expected := range expectedList {
|
||||
count := 0
|
||||
err := objStore.IterateObjects(dir, func(path string, f Object) error {
|
||||
content, err := io.ReadAll(f)
|
||||
t.Run("ReadWrite", func(t *testing.T) {
|
||||
for _, it := range testFiles {
|
||||
fp, content := it[0], it[1]
|
||||
_, err := objStore.Save(fp, strings.NewReader(content), -1)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, content)
|
||||
assert.Contains(t, expected, path)
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, expected, count)
|
||||
}
|
||||
|
||||
stat, err := objStore.Stat(fp)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, path.Base(fp), stat.Name())
|
||||
|
||||
f, err := objStore.Open(fp)
|
||||
assert.NoError(t, err)
|
||||
stat, err = f.Stat()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, path.Base(fp), stat.Name())
|
||||
|
||||
contentBytes, err := io.ReadAll(f)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, content, string(contentBytes))
|
||||
_ = f.Close()
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Iterator", func(t *testing.T) {
|
||||
expectedList := map[string][]string{
|
||||
"a": {"a/1.txt"},
|
||||
"a/": {"a/1.txt"},
|
||||
"/a/": {"a/1.txt"},
|
||||
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
||||
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
".": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||
"a/b/../../a": {"a/1.txt"},
|
||||
}
|
||||
for dir, expected := range expectedList {
|
||||
count := 0
|
||||
err := objStore.IterateObjects(dir, func(p string, f Object) error {
|
||||
content, err := io.ReadAll(f)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, content)
|
||||
assert.Contains(t, expected, p)
|
||||
count++
|
||||
|
||||
stat, err := f.Stat()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, path.Base(p), stat.Name())
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, expected, count)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorage) {
|
||||
@@ -139,7 +165,7 @@ func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorag
|
||||
}
|
||||
|
||||
func testStorageGeneral(t *testing.T, objStore ObjectStorage) {
|
||||
t.Run("StorageIterator", func(t *testing.T) { testStorageIterator(t, objStore) })
|
||||
t.Run("StorageAccess", func(t *testing.T) { testStorageAccess(t, objStore) })
|
||||
|
||||
if _, ok := objStore.(*LocalStorage); ok {
|
||||
t.Skipf("Skipping tests for local storage")
|
||||
|
||||
+10
-31
@@ -17,17 +17,18 @@ func (NopCloser) Close() error { return nil }
|
||||
|
||||
// ReadAtMost reads at most len(buf) bytes from r into buf.
|
||||
// It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes.
|
||||
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
|
||||
// If EOF occurs while reading, err will be nil.
|
||||
func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
|
||||
n, err = io.ReadFull(r, buf)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
for n < len(buf) && err == nil {
|
||||
var read int
|
||||
read, err = r.Read(buf[n:])
|
||||
n += read
|
||||
}
|
||||
return n, err
|
||||
return n, Iif(err == io.EOF, nil, err)
|
||||
}
|
||||
|
||||
// ReadWithLimit reads at most "limit" bytes from r into buf.
|
||||
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
|
||||
// If EOF occurs while reading, err will be nil.
|
||||
func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
|
||||
return readWithLimit(r, 4*1024, n)
|
||||
}
|
||||
@@ -49,12 +50,11 @@ func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
|
||||
if res.Len()+batch > limit {
|
||||
bufTmp = bufFix[:limit-res.Len()]
|
||||
}
|
||||
n, err := io.ReadFull(r, bufTmp)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
eof = true
|
||||
} else if err != nil {
|
||||
n, err := ReadAtMost(r, bufTmp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
eof = n != len(bufTmp)
|
||||
if _, err = res.Write(bufTmp[:n]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -82,24 +82,3 @@ func IsEmptyReader(r io.Reader) (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type CountingReader struct {
|
||||
io.Reader
|
||||
n int
|
||||
}
|
||||
|
||||
var _ io.Reader = &CountingReader{}
|
||||
|
||||
func (w *CountingReader) Count() int {
|
||||
return w.n
|
||||
}
|
||||
|
||||
func (w *CountingReader) Read(p []byte) (int, error) {
|
||||
n, err := w.Reader.Read(p)
|
||||
w.n += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
func NewCountingReader(rd io.Reader) *CountingReader {
|
||||
return &CountingReader{Reader: rd}
|
||||
}
|
||||
|
||||
@@ -86,18 +86,6 @@
|
||||
"matchPackageNames": ["typescript"],
|
||||
"allowedVersions": "^6", // typescript-eslint is not yet compatible with typescript 7
|
||||
},
|
||||
{
|
||||
"matchPackageNames": ["github.com/Azure/azure-sdk-for-go/sdk/azcore"],
|
||||
"allowedVersions": "<1.21.0", // v1.21.0+ uses API version unsupported by Azurite in CI
|
||||
},
|
||||
{
|
||||
"matchPackageNames": ["github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"],
|
||||
"allowedVersions": "<1.6.4", // v1.6.4+ uses API version unsupported by Azurite in CI
|
||||
},
|
||||
{
|
||||
"matchPackageNames": ["github.com/microsoft/go-mssqldb"],
|
||||
"allowedVersions": "<=1.9.7", // downgraded with Azure SDK
|
||||
},
|
||||
{
|
||||
"matchPackageNames": ["github.com/yuin/goldmark"],
|
||||
"allowedVersions": "<2.0.0", // v2 is a rewrite, the markdown renderer and extensions need a migration
|
||||
|
||||
Reference in New Issue
Block a user