Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
alist-org
GitHub Repository: alist-org/alist
Path: blob/main/drivers/azure_blob/driver.go
1987 views
1
package azure_blob
2
3
import (
4
"context"
5
"fmt"
6
"io"
7
"path"
8
"regexp"
9
"strings"
10
"time"
11
12
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
13
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
14
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
15
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
16
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
17
"github.com/alist-org/alist/v3/internal/driver"
18
"github.com/alist-org/alist/v3/internal/model"
19
)
20
// Azure Blob Storage based on the blob APIs
21
// Link: https://learn.microsoft.com/rest/api/storageservices/blob-service-rest-api
22
type AzureBlob struct {
23
model.Storage
24
Addition
25
client *azblob.Client
26
containerClient *container.Client
27
config driver.Config
28
}
29
30
// Config returns the driver configuration.
31
func (d *AzureBlob) Config() driver.Config {
32
return d.config
33
}
34
35
// GetAddition returns additional settings specific to Azure Blob Storage.
36
func (d *AzureBlob) GetAddition() driver.Additional {
37
return &d.Addition
38
}
39
40
// Init initializes the Azure Blob Storage client using shared key authentication.
41
func (d *AzureBlob) Init(ctx context.Context) error {
42
// Validate the endpoint URL
43
accountName := extractAccountName(d.Addition.Endpoint)
44
if !regexp.MustCompile(`^[a-z0-9]+$`).MatchString(accountName) {
45
return fmt.Errorf("invalid storage account name: must be chars of lowercase letters or numbers only")
46
}
47
48
credential, err := azblob.NewSharedKeyCredential(accountName, d.Addition.AccessKey)
49
if err != nil {
50
return fmt.Errorf("failed to create credential: %w", err)
51
}
52
53
// Check if Endpoint is just account name
54
endpoint := d.Addition.Endpoint
55
if accountName == endpoint {
56
endpoint = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName)
57
}
58
// Initialize Azure Blob client with retry policy
59
client, err := azblob.NewClientWithSharedKeyCredential(endpoint, credential,
60
&azblob.ClientOptions{ClientOptions: azcore.ClientOptions{
61
Retry: policy.RetryOptions{
62
MaxRetries: MaxRetries,
63
RetryDelay: RetryDelay,
64
},
65
}})
66
if err != nil {
67
return fmt.Errorf("failed to create client: %w", err)
68
}
69
d.client = client
70
71
// Ensure container exists or create it
72
containerName := strings.Trim(d.Addition.ContainerName, "/ \\")
73
if containerName == "" {
74
return fmt.Errorf("container name cannot be empty")
75
}
76
return d.createContainerIfNotExists(ctx, containerName)
77
}
78
79
// Drop releases resources associated with the Azure Blob client.
80
func (d *AzureBlob) Drop(ctx context.Context) error {
81
d.client = nil
82
return nil
83
}
84
85
// List retrieves blobs and directories under the specified path.
86
func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
87
prefix := ensureTrailingSlash(dir.GetPath())
88
89
pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
90
Prefix: &prefix,
91
})
92
93
var objs []model.Obj
94
for pager.More() {
95
page, err := pager.NextPage(ctx)
96
if err != nil {
97
return nil, fmt.Errorf("failed to list blobs: %w", err)
98
}
99
100
// Process directories
101
for _, blobPrefix := range page.Segment.BlobPrefixes {
102
objs = append(objs, &model.Object{
103
Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")),
104
Path: *blobPrefix.Name,
105
Modified: *blobPrefix.Properties.LastModified,
106
Ctime: *blobPrefix.Properties.CreationTime,
107
IsFolder: true,
108
})
109
}
110
111
// Process files
112
for _, blob := range page.Segment.BlobItems {
113
if strings.HasSuffix(*blob.Name, "/") {
114
continue
115
}
116
objs = append(objs, &model.Object{
117
Name: path.Base(*blob.Name),
118
Path: *blob.Name,
119
Size: *blob.Properties.ContentLength,
120
Modified: *blob.Properties.LastModified,
121
Ctime: *blob.Properties.CreationTime,
122
IsFolder: false,
123
})
124
}
125
}
126
return objs, nil
127
}
128
129
// Link generates a temporary SAS URL for accessing a blob.
130
func (d *AzureBlob) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
131
blobClient := d.containerClient.NewBlobClient(file.GetPath())
132
expireDuration := time.Hour * time.Duration(d.SignURLExpire)
133
134
sasURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil)
135
if err != nil {
136
return nil, fmt.Errorf("failed to generate SAS URL: %w", err)
137
}
138
return &model.Link{URL: sasURL}, nil
139
}
140
141
// MakeDir creates a virtual directory by uploading an empty blob as a marker.
142
func (d *AzureBlob) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
143
dirPath := path.Join(parentDir.GetPath(), dirName)
144
if err := d.mkDir(ctx, dirPath); err != nil {
145
return nil, fmt.Errorf("failed to create directory marker: %w", err)
146
}
147
148
return &model.Object{
149
Path: dirPath,
150
Name: dirName,
151
IsFolder: true,
152
}, nil
153
}
154
155
// Move relocates an object (file or directory) to a new directory.
156
func (d *AzureBlob) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
157
srcPath := srcObj.GetPath()
158
dstPath := path.Join(dstDir.GetPath(), srcObj.GetName())
159
160
if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil {
161
return nil, fmt.Errorf("move operation failed: %w", err)
162
}
163
164
return &model.Object{
165
Path: dstPath,
166
Name: srcObj.GetName(),
167
Modified: time.Now(),
168
IsFolder: srcObj.IsDir(),
169
Size: srcObj.GetSize(),
170
}, nil
171
}
172
173
// Rename changes the name of an existing object.
174
func (d *AzureBlob) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
175
srcPath := srcObj.GetPath()
176
dstPath := path.Join(path.Dir(srcPath), newName)
177
178
if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil {
179
return nil, fmt.Errorf("rename operation failed: %w", err)
180
}
181
182
return &model.Object{
183
Path: dstPath,
184
Name: newName,
185
Modified: time.Now(),
186
IsFolder: srcObj.IsDir(),
187
Size: srcObj.GetSize(),
188
}, nil
189
}
190
191
// Copy duplicates an object (file or directory) to a specified destination directory.
192
func (d *AzureBlob) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
193
dstPath := path.Join(dstDir.GetPath(), srcObj.GetName())
194
195
// Handle directory copying using flat listing
196
if srcObj.IsDir() {
197
srcPrefix := srcObj.GetPath()
198
srcPrefix = ensureTrailingSlash(srcPrefix)
199
200
// Get all blobs under the source directory
201
blobs, err := d.flattenListBlobs(ctx, srcPrefix)
202
if err != nil {
203
return nil, fmt.Errorf("failed to list source directory contents: %w", err)
204
}
205
206
// Process each blob - copy to destination
207
for _, blob := range blobs {
208
// Skip the directory marker itself
209
if *blob.Name == srcPrefix {
210
continue
211
}
212
213
// Calculate relative path from source
214
relPath := strings.TrimPrefix(*blob.Name, srcPrefix)
215
itemDstPath := path.Join(dstPath, relPath)
216
217
if strings.HasSuffix(itemDstPath, "/") || (blob.Metadata["hdi_isfolder"] != nil && *blob.Metadata["hdi_isfolder"] == "true") {
218
// Create directory marker at destination
219
err := d.mkDir(ctx, itemDstPath)
220
if err != nil {
221
return nil, fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err)
222
}
223
} else {
224
// Copy the blob
225
if err := d.copyFile(ctx, *blob.Name, itemDstPath); err != nil {
226
return nil, fmt.Errorf("failed to copy %s: %w", *blob.Name, err)
227
}
228
}
229
230
}
231
232
// Create directory marker at destination if needed
233
if len(blobs) == 0 {
234
err := d.mkDir(ctx, dstPath)
235
if err != nil {
236
return nil, fmt.Errorf("failed to create directory [%s]: %w", dstPath, err)
237
}
238
}
239
240
return &model.Object{
241
Path: dstPath,
242
Name: srcObj.GetName(),
243
Modified: time.Now(),
244
IsFolder: true,
245
}, nil
246
}
247
248
// Copy a single file
249
if err := d.copyFile(ctx, srcObj.GetPath(), dstPath); err != nil {
250
return nil, fmt.Errorf("failed to copy blob: %w", err)
251
}
252
return &model.Object{
253
Path: dstPath,
254
Name: srcObj.GetName(),
255
Size: srcObj.GetSize(),
256
Modified: time.Now(),
257
IsFolder: false,
258
}, nil
259
}
260
261
// Remove deletes a specified blob or recursively deletes a directory and its contents.
262
func (d *AzureBlob) Remove(ctx context.Context, obj model.Obj) error {
263
path := obj.GetPath()
264
265
// Handle recursive directory deletion
266
if obj.IsDir() {
267
return d.deleteFolder(ctx, path)
268
}
269
270
// Delete single file
271
return d.deleteFile(ctx, path, false)
272
}
273
274
// Put uploads a file stream to Azure Blob Storage with progress tracking.
275
func (d *AzureBlob) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
276
blobPath := path.Join(dstDir.GetPath(), stream.GetName())
277
blobClient := d.containerClient.NewBlockBlobClient(blobPath)
278
279
// Determine optimal upload options based on file size
280
options := optimizedUploadOptions(stream.GetSize())
281
282
// Track upload progress
283
progressTracker := &progressTracker{
284
total: stream.GetSize(),
285
updateProgress: up,
286
}
287
288
// Wrap stream to handle context cancellation and progress tracking
289
limitedStream := driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, progressTracker))
290
291
// Upload the stream to Azure Blob Storage
292
_, err := blobClient.UploadStream(ctx, limitedStream, options)
293
if err != nil {
294
return nil, fmt.Errorf("failed to upload file: %w", err)
295
}
296
297
return &model.Object{
298
Path: blobPath,
299
Name: stream.GetName(),
300
Size: stream.GetSize(),
301
Modified: time.Now(),
302
IsFolder: false,
303
}, nil
304
}
305
306
// The following methods related to archive handling are not implemented yet.
307
// func (d *AzureBlob) GetArchiveMeta(...) {...}
308
// func (d *AzureBlob) ListArchive(...) {...}
309
// func (d *AzureBlob) Extract(...) {...}
310
// func (d *AzureBlob) ArchiveDecompress(...) {...}
311
312
// Ensure AzureBlob implements the driver.Driver interface.
313
var _ driver.Driver = (*AzureBlob)(nil)
314
315